From 9fac6f8342096d924adb1351e443325c992cf579 Mon Sep 17 00:00:00 2001 From: Codevena Date: Fri, 7 Aug 2026 12:31:40 +0200 Subject: [PATCH 01/55] bench: measure opencode harness overhead per tool-surface variant (31.01 -> 22.86 credits) --- bench/results/qwen-overhead/tool-surface.json | 66 +++++++++++++++++++ 1 file changed, 66 insertions(+) create mode 100644 bench/results/qwen-overhead/tool-surface.json diff --git a/bench/results/qwen-overhead/tool-surface.json b/bench/results/qwen-overhead/tool-surface.json new file mode 100644 index 0000000..492a19a --- /dev/null +++ b/bench/results/qwen-overhead/tool-surface.json @@ -0,0 +1,66 @@ +{ + "measuredAt": "2026-08-07", + "coefficient": 1.21, + "coefficientUnit": "credits per 1000 tokens", + "model": "alibaba-token-plan/qwen3.8-max", + "opencodeVersion": "1.18.10", + "prompt": "Antworte nur mit: OK", + "source": "opencode session DB (~/.local/share/opencode/opencode.db), via scripts/measure-opencode-tokens.ts", + "variants": [ + { + "name": "baseline", + "args": [], + "usage": { + "total": 25629, + "input": 23544, + "output": 15, + "reasoning": 22, + "cacheRead": 2048, + "cacheWrite": 0 + }, + "credits": 31.01, + "note": "Reused from the Phase 0a measurement rather than re-run, to avoid spending ~30 credits on a number already in the DB. Two deviations from the other variants, both stated rather than hidden: the prompt was 'Antworte nur mit: OK und deinem Modellnamen' (a few tokens longer out of ~24K), and the cache was warm (2048 cacheRead), which inflates `total` relative to the cold variants. The input-only comparison below is the controlled one." + }, + { + "name": "pure", + "args": ["--pure"], + "usage": { + "total": 23262, + "input": 23237, + "output": 6, + "reasoning": 19, + "cacheRead": 0, + "cacheWrite": 0 + }, + "credits": 28.15 + }, + { + "name": "pure+rg-reviewer", + "args": ["--pure", "--agent", "rg-reviewer"], + "usage": { + "total": 18895, + "input": 17847, + "output": 5, + "reasoning": 19, + "cacheRead": 1024, + "cacheWrite": 0 + }, + "credits": 22.86 + } + ], + "findings": { + "cheapestVariant": "pure+rg-reviewer", + "cheapestCredits": 22.86, + "reductionFromBaseline": "26.3% by total, 24.2% by uncached input (23544 -> 17847)", + "pluginsAreNotTheCost": "--pure saved 307 input tokens against the baseline (23544 -> 23237). The 8 installed skills and the single @opencode-ai/plugin dependency are not the driver.", + "toolSchemasAreTheCost": "The reduced-tool agent saved 5390 input tokens against --pure (23237 -> 17847). The tool schemas are where the ~24K system prompt lives.", + "stopCondition": "NOT evaluated here by design (plan Task 2 Step 7). 22.86 credits is above both the 13-credit at-risk line and the 20-credit stop line, so Task 3's caching lever now has to carry the difference. Per the plan, Task 3 Step 7 is the single evaluation site." + }, + "sideFinding": { + "id": "opencode-permission-flag", + "severity": "independent of this plan", + "claim": "src/providers/opencode.ts:97 and :242 both pass `--dangerously-skip-permissions`, which does not exist in opencode 1.18.10 (the documented flag is `--auto`).", + "evidence": "`opencode run --help` lists only `--auto`; `opencode run --definitely-not-a-real-flag --help` exits 0, so unknown flags are silently ignored rather than rejected.", + "consequence": "Both the reviewer path and the complete()/curator path run without permission auto-approval. A plausible but UNPROVEN contributor to the curator hang recorded in spec §8 (a 150s no-output hang observed 2026-08-07). Fix separately from this plan." + } +} From ef54ed0e43b5d42131ed6e258f100e58405ab560 Mon Sep 17 00:00:00 2001 From: Codevena Date: Fri, 7 Aug 2026 12:52:21 +0200 Subject: [PATCH 02/55] feat(bench): --provider-model pins a reviewer's upstream model into provenance --- src/bench/runner.ts | 14 +++ src/cli/commands/bench.ts | 33 ++++++- src/cli/index.ts | 15 +++- tests/unit/bench-provider-model.test.ts | 114 ++++++++++++++++++++++++ 4 files changed, 174 insertions(+), 2 deletions(-) create mode 100644 tests/unit/bench-provider-model.test.ts diff --git a/src/bench/runner.ts b/src/bench/runner.ts index 58ae8de..3c02e2a 100644 --- a/src/bench/runner.ts +++ b/src/bench/runner.ts @@ -97,6 +97,11 @@ export interface BenchConfigOptions { /** Exact critic model and upstream route recorded by benchmark provenance. */ criticModel?: string; criticOpenrouterProvider?: OpenRouterProviderRouting; + /** Pin a provider's upstream model for this run. Recorded by provenance via + * buildRoster, which reads providers..model. Without this, a provider whose + * model is the "default" sentinel resolves against the user's own CLI config — + * an unversioned benchmark input. */ + providerModels?: Partial>; /** Hard provider-side output ceiling for OpenRouter review/critic requests. */ maxOutputTokens?: number; } @@ -131,6 +136,15 @@ export function buildBenchConfig(opts: BenchConfigOptions = {}): ReviewgateConfi if (pc) pc.enabled = true; } } + // Applied AFTER the panel loop so a pinned model survives the enable pass, and + // independently of `providers` so a critic-only or curator-only provider can be + // pinned too. + if (opts.providerModels) { + for (const [provider, model] of Object.entries(opts.providerModels)) { + const pc = base.providers[provider as ProviderId]; + if (pc) pc.model = model; + } + } const s = opts.suppressors; if (s) { if (s.critic !== undefined) { diff --git a/src/cli/commands/bench.ts b/src/cli/commands/bench.ts index 97c2386..dc92e39 100644 --- a/src/cli/commands/bench.ts +++ b/src/cli/commands/bench.ts @@ -70,6 +70,33 @@ const KNOWN_PROVIDERS: ReadonlySet = new Set([ "ollama", ]); +/** Parse `--provider-model opencode=alibaba-token-plan/qwen3.8-max,ollama=glm-5.2:cloud`. + * Splits on the FIRST `=` only — model ids legitimately contain slashes, colons + * and occasionally `=`. Validated against KNOWN_PROVIDERS rather than a second + * hand-maintained list, so a new provider cannot be accepted here while being + * rejected two functions down. */ +export function parseProviderModels(raw: string): Partial> { + const out: Partial> = {}; + const pairs = raw + .split(",") + .map((s) => s.trim()) + .filter(Boolean); + for (const pair of pairs) { + const eq = pair.indexOf("="); + if (eq <= 0) { + throw new Error(`--provider-model expects =, got "${pair}"`); + } + const provider = pair.slice(0, eq).trim(); + const model = pair.slice(eq + 1).trim(); + if (!model) throw new Error(`--provider-model: empty model for "${provider}"`); + if (!KNOWN_PROVIDERS.has(provider)) { + throw new Error(`--provider-model: unknown provider "${provider}"`); + } + out[provider as ProviderId] = model; + } + return out; +} + export interface BenchRunInput { repoRoot: string; corpus: string; @@ -88,6 +115,9 @@ export interface BenchRunInput { ablationLabels?: string[]; criticModel?: string; criticOpenrouterProvider?: OpenRouterProviderRouting; + /** Pin a reviewer's upstream model, so provenance records what actually ran + * instead of a "default" sentinel that resolves outside the repo. */ + providerModels?: Partial>; /** Benchmark-only physical critic completion limit; runtime default remains 1. */ criticMaxAttempts?: number; /** Benchmark-only physical reviewer invocation limit per configured reviewer/case. */ @@ -476,7 +506,7 @@ async function preregistrationDigest( return { digest: sha256File(path), tracked: tracked.status === 0 }; } -async function buildRoster( +export async function buildRoster( config: ReviewgateConfig, adapters: Partial>, ): Promise> { @@ -578,6 +608,7 @@ async function runBenchRunInternal(input: BenchRunInput): Promise { + it("parses a single provider=model pair", () => { + expect(parseProviderModels(`opencode=${QWEN}`)).toEqual({ opencode: QWEN }); + }); + + it("parses several comma-separated pairs", () => { + expect(parseProviderModels(`opencode=${QWEN},ollama=glm-5.2:cloud`)).toEqual({ + opencode: QWEN, + ollama: "glm-5.2:cloud", + }); + }); + + it("keeps '=' inside the model id (provider/model:tag forms)", () => { + expect(parseProviderModels("openrouter=deepseek/deepseek-v4-flash=x")).toEqual({ + openrouter: "deepseek/deepseek-v4-flash=x", + }); + }); + + it("rejects an unknown provider", () => { + expect(() => parseProviderModels("qwen=whatever")).toThrow(/unknown provider "qwen"/); + }); + + it("rejects a pair without '='", () => { + expect(() => parseProviderModels("opencode")).toThrow(/expects =/); + }); + + it("rejects an empty model", () => { + expect(() => parseProviderModels("opencode=")).toThrow(/empty model/); + }); + + it("returns an empty object for an empty string", () => { + expect(parseProviderModels("")).toEqual({}); + }); +}); + +describe("buildBenchConfig providerModels", () => { + // The guard's two numbers, stated up front: WITHOUT the mechanism the model is + // "default"; WITH it, the pinned id. Both differ, so this test is not vacuous. + it("leaves the model at the 'default' sentinel when no override is given", () => { + const cfg = buildBenchConfig({ providers: ["opencode"] }); + expect(cfg.providers.opencode?.model).toBe("default"); + }); + + it("pins the provider model when an override is given", () => { + const cfg = buildBenchConfig({ + providers: ["opencode"], + providerModels: { opencode: QWEN }, + }); + expect(cfg.providers.opencode?.model).toBe(QWEN); + }); + + // Two numbers: the shipped default for ollama is "glm-5.2:cloud" + // (src/config/defaults.ts:63), so the override MUST be a different value or the + // test proves nothing. WITHOUT the mechanism: "glm-5.2:cloud". WITH it: + // "qwen3-coder:480b-cloud". + it("pins a provider that is not in the reviewer panel", () => { + expect(buildBenchConfig({ providers: ["opencode"] }).providers.ollama?.model).toBe( + "glm-5.2:cloud", + ); + const cfg = buildBenchConfig({ + providers: ["opencode"], + providerModels: { ollama: "qwen3-coder:480b-cloud" }, + }); + expect(cfg.providers.ollama?.model).toBe("qwen3-coder:480b-cloud"); + }); + + // `suppressors: { critic: "openrouter" }` is REQUIRED: defaultConfig.phases.critic + // is null, and buildBenchConfig applies criticModel only inside its + // `if (base.phases.critic)` branch. Without it, cfg.phases.critic?.model is + // undefined and this test fails — which is correct existing behaviour, NOT a bug + // to "fix" by making criticModel unconditional. + it("does not disturb the critic model override", () => { + const cfg = buildBenchConfig({ + providers: ["opencode"], + providerModels: { opencode: QWEN }, + suppressors: { critic: "openrouter" }, + criticModel: "deepseek/deepseek-v4-flash", + }); + expect(cfg.providers.opencode?.model).toBe(QWEN); + expect(cfg.phases.critic?.model).toBe("deepseek/deepseek-v4-flash"); + }); +}); + +describe("buildRoster provenance", () => { + // The spec states the guard in terms of PROVENANCE, not config. Every test above + // asserts cfg.providers..model — the config layer. This one closes the hop to + // what actually gets written into the results file, via the `roster.push` in + // buildRoster (`model: providerCfg?.model ?? "unknown"`). + // Empty adapters: preflight is skipped, cli_version falls back to "unknown", and the + // model still resolves — which is the only field under test here. + // WITHOUT the override: "default". WITH it: the pinned id. + it("records the pinned model in the provenance roster", async () => { + const plain = await buildRoster(buildBenchConfig({ providers: ["opencode"] }), {}); + expect(plain[0]?.model).toBe("default"); + + const pinned = await buildRoster( + buildBenchConfig({ providers: ["opencode"], providerModels: { opencode: QWEN } }), + {}, + ); + expect(pinned[0]?.model).toBe(QWEN); + }); +}); From 979bfea5d9ad0c1412bdb9b0c8e9edee56079f40 Mon Sep 17 00:00:00 2001 From: Codevena Date: Fri, 7 Aug 2026 12:59:16 +0200 Subject: [PATCH 03/55] =?UTF-8?q?bench:=20cache=20fill,=20credit=20discoun?= =?UTF-8?q?t=20and=20TTL=20survival=20=E2=80=94=209.17=20credits/call,=20s?= =?UTF-8?q?top=20condition=20clear?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- bench/results/qwen-overhead/caching.json | 93 ++++++++++++++++++++++++ 1 file changed, 93 insertions(+) create mode 100644 bench/results/qwen-overhead/caching.json diff --git a/bench/results/qwen-overhead/caching.json b/bench/results/qwen-overhead/caching.json new file mode 100644 index 0000000..cabd281 --- /dev/null +++ b/bench/results/qwen-overhead/caching.json @@ -0,0 +1,93 @@ +{ + "measuredAt": "2026-08-07", + "variant": "pure+rg-reviewer", + "model": "alibaba-token-plan/qwen3.8-max", + "prompt": "Antworte nur mit: OK", + "consoleCreditsBefore": { "percentUsed": 4.45, "credits": 111.25 }, + "consoleCreditsAfter": { "percentUsed": 5.55, "credits": 138.75 }, + "consoleDeltaCredits": 27.5, + "consoleDeltaCoversCalls": 3, + "calls": [ + { + "n": 1, + "gapSeconds": 0, + "usage": { + "total": 18980, + "input": 7616, + "output": 5, + "reasoning": 95, + "cacheRead": 11264, + "cacheWrite": 0 + }, + "cacheFraction": 0.597 + }, + { + "n": 2, + "gapSeconds": "<60", + "usage": { + "total": 18903, + "input": 2487, + "output": 6, + "reasoning": 26, + "cacheRead": 16384, + "cacheWrite": 0 + }, + "cacheFraction": 0.868 + }, + { + "n": 3, + "gapSeconds": "<60", + "usage": { + "total": 18891, + "input": 4531, + "output": 5, + "reasoning": 19, + "cacheRead": 14336, + "cacheWrite": 0 + }, + "cacheFraction": 0.76 + }, + { + "n": 4, + "gapSeconds": 1451, + "note": "TTL test — 24 minutes after call 3, ~5x the documented 5-minute cache TTL. NOT covered by the console delta above.", + "usage": { + "total": 18907, + "input": 2491, + "output": 6, + "reasoning": 26, + "cacheRead": 16384, + "cacheWrite": 0 + }, + "cacheFraction": 0.868 + } + ], + "verdicts": { + "fills": true, + "discounted": true, + "survivesTtl": true + }, + "notes": { + "statedCriterionNotMet": "The plan's criterion was 'call 3 cacheFraction >= 0.8'. Call 3 measured 0.760 and therefore FAILS it by the letter. The criterion was mis-specified: it assumed monotonic warming, while the measurement shows a block-quantised plateau fluctuating between 0.597 and 0.868 (every cacheRead is a multiple of 1024). Recorded as-is rather than rounded up. The economic question is settled by the credit delta below, which is measured rather than inferred from the fraction.", + "discountEvidence": "Flat-coefficient prediction for the three calls was 68.70 credits (56,774 tokens x 1.21/1K). The console delta was 27.5 — 40% of that, i.e. a 2.5x saving. Solving for the cache-read rate with uncached tokens at 1.21/1K gives ~0.22 credits/1K, a ratio of ~1/5.3 against uncached. Alibaba's list pricing puts cache read at 1/8 of input ($0.25 vs $2.00); the measured ratio is the same order but not identical, which is expected since the credit ledger is a different mechanism from list pricing.", + "ttlEvidence": "Call 4 ran 24 minutes after call 3 and returned the HIGHEST cache fraction of all four (0.868). The documented 5-minute TTL does not bound this plan's cache in practice — either reads refresh it or the effective TTL is longer.", + "outputMixCaveat": "All four calls produced 5-6 output tokens. A real review is reasoning-heavy and output weighs ~3x input, so the per-case figure will be higher than the per-call figure below. Task 5 is the only measurement that resolves this." + }, + "stopCondition": { + "effectivePerCall": 9.17, + "verdict": "clear", + "basis": "discounted: true -> console delta 27.5 credits / 3 calls = 9.17. Divisor is 3, not 4: the Step 4 console reading precedes Step 5's TTL call, which is therefore not covered by the delta. Band: < 13 = clear.", + "chain": { + "baselineDefaultAgent": 31.01, + "plusReducedToolSet": 22.86, + "plusWarmCache": 9.17, + "totalReduction": "3.4x" + }, + "extrapolation": { + "note": "At 9.17 credits/call against a 2,500-credit weekly window. The second figure in each pair adds a realistic 1.5K reasoning-heavy output (+5.4 credits/case) and is the honest planning number until Task 5 measures it.", + "bench30x1": "275 credits (11% of window) / ~438 credits (17.5%) with realistic output", + "bench30x3Authoritative": "825 credits (33%) / ~1,314 credits (53%) with realistic output", + "wasBeforeThisTask": "30x1 = 900 credits (36%), 30x3 = 2,700 credits (108%, did not fit)" + } + } +} From e7c25e18253d47b3d0322ac227efa1729f967195 Mon Sep 17 00:00:00 2001 From: Codevena Date: Fri, 7 Aug 2026 13:06:27 +0200 Subject: [PATCH 04/55] bench: per-case Qwen cost at tuned invocation + Phase 2 go / Phase 3 no-go --- bench/results/qwen-overhead/DECISION.md | 98 ++++++++++++++ bench/results/qwen-overhead/smoke.json | 162 ++++++++++++++++++++++++ src/providers/opencode.ts | 14 +- 3 files changed, 273 insertions(+), 1 deletion(-) create mode 100644 bench/results/qwen-overhead/DECISION.md create mode 100644 bench/results/qwen-overhead/smoke.json diff --git a/bench/results/qwen-overhead/DECISION.md b/bench/results/qwen-overhead/DECISION.md new file mode 100644 index 0000000..f3dd135 --- /dev/null +++ b/bench/results/qwen-overhead/DECISION.md @@ -0,0 +1,98 @@ +# Qwen3.8-Max cost decision — Phase 0 go/no-go + +- **Date:** 2026-08-07 +- **Plan:** `docs/superpowers/plans/2026-08-07-qwen-overhead-and-provider-model.md` Task 5 +- **Spec:** `docs/superpowers/specs/2026-08-07-qwen-reviewer-measurement-design.md` +- **Verdict:** **Phase 2 GO. Phase 3 (authoritative) NO-GO at the Lite tier.** + +## What was measured + +One `reviewgate bench run` over a 2-case corpus (1 clean + 1 seeded), reviewer = +`opencode`, model pinned via the new `--provider-model` flag. Result: +`2/2 cases scored → precision 1, recall 1, clean-FP 0`, exit 0, 75 seconds. + +**N=2. That is a smoke test of the pipeline, not evidence about Qwen's review +quality.** The acceptance bar in spec §6 is unaffected and unanswered. + +Provenance confirms the flag works end to end — the whole reason Task 4 exists: + +```json +{ "id": "opencode", "cli_version": "1.18.10", + "model": "alibaba-token-plan/qwen3.8-max", "persona": "security" } +``` + +Not `"default"`. A future run is attributable to a specific model. + +## The cost, and the correction it forces + +Four LLM calls fell inside the bench window (13:00:40–13:01:55), attributed by +timestamp from opencode's session DB: + +| time | uncached | cacheRead | credits | +| --- | --- | --- | --- | +| 13:00:44 | 21,323 | 2,048 | 26.25 (cold cache) | +| 13:01:15 | 10,262 | 12,288 | 15.12 | +| 13:01:26 | 543 | 22,144 | 5.53 | +| 13:01:30 | 1,334 | 22,528 | 6.57 | +| | | **total** | **53.47 for 2 cases** | + +**≈ 26.7 credits per case.** A case costs roughly **two** LLM calls, not one, and +the first call of a run pays a cold cache. + +This supersedes every earlier extrapolation, exactly as the plan said it would: + +| stage | credits/unit | 30 × 1 | 30 × 3 | +| --- | --- | --- | --- | +| baseline, default agent (Task 2) | 31.0 /call | 900 (36 %) | 2,700 (108 %) | +| + reduced tool set (Task 2) | 22.9 /call | 686 (27 %) | 2,058 (82 %) | +| + warm cache (Task 3) | 9.2 /call | 275 (11 %) | 825 (33 %) | +| **real case (Task 5)** | **26.7 /case** | **800 (32 %)** | **2,400 (96 %)** | + +The Task 3 figure was a per-**call** number measured on a 5-output-token prompt. +The plan predicted a realistic output mix would add +3.6 to +7.3 credits. It added +**+17.5**, and the two-calls-per-case structure was not anticipated at all. + +## Decision against the stop condition + +Spec §5 Phase 0b: stop if per-case cost cannot be brought under **20 credits**. +Measured: **26.7**. The condition is **tripped**. + +But the consequence is narrower than "stop everything", because the two runs have +very different costs: + +- **Phase 2 (exploratory, 30 × 1): GO.** 800 credits ≈ 32 % of the 2,500-credit + weekly window. It fits, and it is the only way to answer the actual question in + spec §6 — whether Qwen finds a seeded bug that GLM-5.2 and claude-code both miss. +- **Phase 3 (authoritative, 30 × 3): NO-GO at Lite.** 2,400 credits ≈ 96 % of the + window. Running it consumes essentially the entire week and leaves nothing for + day-to-day gate traffic. + +**Recommended next step: run Phase 2.** It is affordable, it answers the quality +question, and it produces the definitive per-case cost from 30 real cases instead +of the 2-case extrapolation above — which is the number Phase 3 needs. + +## Caveats, stated rather than buried + +- **Not cross-checked against the console.** The credit figures convert DB tokens + at 1.21/1K uncached and 0.22/1K cached. That model was validated once + (4.45 % → 5.55 %, predicted 5.42 %, actual 5.55 %), but *this* extrapolation was + not. A single console reading would confirm or refute it: the predicted total + after the TTL call plus the four bench calls is **≈ 7.95 %**. +- **Cold-start amortisation is unquantified.** Only the first case of a run pays + the cold cache. If steady state is the last two calls (12.10 credits/case), 30 + cases would be ~380 credits (15 %) and the authoritative run ~1,140 (46 %), + which would change the Phase 3 verdict. That is an extrapolation from **two** + data points and is not treated as a result here. Phase 2 settles it. +- **The escape hatches in spec §5a are probably closed.** Extra Bundles and a plan + tier change are both *orders*, and the account carries a + `RISK.RISK_CONTROL_REJECTION` block on orders (Task 1b). Unverified — it needs a + purchase attempt — but do not plan on buying headroom. +- **`--auto` vs `--dangerously-skip-permissions`.** `src/providers/opencode.ts:97` + now passes `--auto`; the old flag does not exist in opencode 1.18.10 and was + silently ignored. **`:242` (the `complete()` / critic / curator path) still passes + the dead flag** and was deliberately left alone — fixing it changes curator + runtime behaviour and belongs in its own commit with its own gate. +- **The adapter change at `:97` is measurement scaffolding**, not a shipped default. + It pins `--pure --agent rg-reviewer`, which depends on a user-global file + (`~/.config/opencode/agent/rg-reviewer.md`) that no other machine has. Revert it + or make it configurable before this ships. diff --git a/bench/results/qwen-overhead/smoke.json b/bench/results/qwen-overhead/smoke.json new file mode 100644 index 0000000..a25a340 --- /dev/null +++ b/bench/results/qwen-overhead/smoke.json @@ -0,0 +1,162 @@ +{ + "schema": "reviewgate.bench.result.v1", + "provenance": { + "reviewgate_version": "0.1.0-alpha.15", + "corpus_commit": "979bfea5d9ad0c1412bdb9b0c8e9edee56079f40", + "corpus_dirty": true, + "providers": [ + { + "id": "opencode", + "cli_version": "1.18.10", + "model": "alibaba-token-plan/qwen3.8-max", + "persona": "security" + } + ], + "config_hash": "47885e462593a94983c8fffe0a8b6c237bf3504e752a5a27bfafc00562f9c923", + "window": 5, + "repeat": 1, + "include_advisory": false, + "temperature": null, + "stores": "per-case-fresh", + "cache": "cold", + "file_context": "full", + "phases": { + "critic": false, + "reputation": true, + "fp_ledger": false, + "confidence_floor": 0.6, + "scope_to_diff": true, + "ablations": [] + }, + "host_os": "darwin-arm64", + "timestamp": "2026-08-07T11:01:55.522Z", + "case_count": { + "seeded": 1, + "clean": 1 + }, + "case_run_count": { + "seeded": 1, + "clean": 1, + "total": 2 + }, + "critic": null, + "integrity": { + "source_commit": "979bfea5d9ad0c1412bdb9b0c8e9edee56079f40", + "repository_dirty": true, + "runner_sha256": "e0c90ec15d33363e6b70713d56bc3b2c7585c17f40a0fe0f8fd9305901d4e233", + "runner_kind": "source-runtime", + "preregistration_sha256": null, + "authoritative_requested": false, + "max_provider_calls": null, + "provider_calls_used": 2, + "max_output_tokens": null, + "reviewer_max_attempts": 1 + } + }, + "cases": [ + { + "id": "clean-clamp-ts", + "kind": "clean", + "status": "scored", + "content_hash": "7bad03dbcf5fc9f543955cfece96db2d712ad5558c83b5430a064118b5cc455a", + "counts": { + "tp": 0, + "fp": 0, + "fn": 0, + "neutral": 0 + }, + "panel_ok": 1, + "panel_configured": 1, + "file_context": "full", + "repeat": 1, + "latency_ms": 31295, + "error": null + }, + { + "id": "sql-injection-ts", + "kind": "seeded-bug", + "status": "scored", + "content_hash": "2be1d6de9a4d3057f64fd00b26eb42ce12ee59075929f4937c7c638dcc7c6236", + "counts": { + "tp": 1, + "fp": 0, + "fn": 0, + "neutral": 0 + }, + "panel_ok": 1, + "panel_configured": 1, + "file_context": "full", + "repeat": 1, + "latency_ms": 42536, + "error": null + } + ], + "providers": [ + { + "provider": "opencode", + "coverage": { + "num": 2, + "den": 2, + "value": 1, + "ci_lo": 0.34238022750665303, + "ci_hi": 1 + }, + "precision": { + "num": 1, + "den": 3, + "value": 0.3333333333333333, + "ci_lo": 0.06149194472039621, + "ci_hi": 0.7923403991979522 + }, + "recall": { + "num": 1, + "den": 1, + "value": 1, + "ci_lo": 0.20654931437723745, + "ci_hi": 1 + }, + "authoritative": true + } + ], + "cost": [ + { + "provider": "opencode", + "calls": 2, + "cache_hits": 0, + "tokens_in": null, + "tokens_out": null, + "billed_usd": null, + "oauth_quota_calls": 2 + } + ], + "critic": null, + "aggregate": { + "precision": { + "num": 1, + "den": 1, + "value": 1, + "ci_lo": 0.20654931437723745, + "ci_hi": 1 + }, + "recall": { + "num": 1, + "den": 1, + "value": 1, + "ci_lo": 0.20654931437723745, + "ci_hi": 1 + }, + "clean_fp_rate": { + "num": 0, + "den": 1, + "value": 0, + "ci_lo": 0, + "ci_hi": 0.7934506856227626 + } + }, + "stability": null, + "verdict": { + "authoritative": true, + "gate_exit_code": 0, + "reasons": [] + } +} diff --git a/src/providers/opencode.ts b/src/providers/opencode.ts index d3f4943..7ffd2db 100644 --- a/src/providers/opencode.ts +++ b/src/providers/opencode.ts @@ -94,7 +94,19 @@ export class OpenCodeAdapter implements ProviderAdapter { const stdoutFile = join(run, "out.txt"); const stderrFile = join(run, "err.log"); - const args = ["run", "--dangerously-skip-permissions", "--format", "default"]; + // MEASUREMENT SCAFFOLDING (2026-08-07, docs/superpowers/plans/2026-08-07-qwen-overhead-and-provider-model.md + // Task 5) — NOT a shipped default. `--pure --agent rg-reviewer` is the cheapest + // variant measured in bench/results/qwen-overhead/tool-surface.json: it cuts the + // opencode system prompt from ~23.5K to ~17.8K input tokens by dropping the + // write/edit tool schemas a reviewer never needs. Revert before release unless + // the bench result justifies keeping it. + // + // `--auto` replaces `--dangerously-skip-permissions`, which does NOT exist in + // opencode 1.18.10 — verified against `opencode run --help`, and opencode exits 0 + // on unknown flags instead of rejecting them, so the old flag was silently + // ignored on every call. The complete() path at the bottom of this file still + // passes the dead flag; fix that separately, it changes curator behaviour. + const args = ["run", "--auto", "--pure", "--agent", "rg-reviewer", "--format", "default"]; // Only force a model with -m for a REAL provider/model id. The sentinel // "default" (or empty) means "use opencode's own configured default model" // — which is how opencode is meant to be driven here (e.g. a MiniMax Token From 384df2a216a6541cd555bbf99e281f9a2618c46b Mon Sep 17 00:00:00 2001 From: Codevena Date: Fri, 7 Aug 2026 13:08:12 +0200 Subject: [PATCH 05/55] =?UTF-8?q?spec:=20rig=20stale-report=20defect=20?= =?UTF-8?q?=E2=80=94=20ownership=20by=20run=5Fid?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 31 of 36 recorded pilot turns open with the previous turn's final pending.json, byte-identical; 13 turns count findings they did not produce, 9 of them produce none of their own. The archiver's docstring says it captures every version that APPEARS while a turn runs; it was implemented as every version that EXISTS. The harvester then folds those into the turn's totals, corrupting the M2 denominator and slope, M3 recall, M4 escape rate, suppression totals and criticRuns attribution. Rule: a report belongs to the turn whose audit delta contains its run_id (verified 1:1 across all 34 recorded gate runs). Works retroactively on the pilots; no rebuild needed for that half. --- .../2026-08-07-rig-stale-report-design.md | 209 ++++++++++++++++++ 1 file changed, 209 insertions(+) create mode 100644 docs/superpowers/specs/2026-08-07-rig-stale-report-design.md diff --git a/docs/superpowers/specs/2026-08-07-rig-stale-report-design.md b/docs/superpowers/specs/2026-08-07-rig-stale-report-design.md new file mode 100644 index 0000000..a02962f --- /dev/null +++ b/docs/superpowers/specs/2026-08-07-rig-stale-report-design.md @@ -0,0 +1,209 @@ +# Rig stale-report defect — design + +_Written 2026-08-07. Status: design approved, not yet implemented._ + +## The defect + +`src/rig/driver.ts` starts a fresh report archiver for every turn (`:338`), and that archiver +captures whatever `pending.json` sits in the live `.reviewgate/` — including the version the +*previous* turn left behind. Its docstring says it archives "every version of `pending.{json,md}` +that **appears** WHILE a turn runs" (`:201`); it was implemented as every version that **exists** +while a turn runs. The first poll fires 250 ms into the turn, long before the turn's own gate has +written anything, so the predecessor's final report is archived as this turn's `1-pending.json`. + +`src/rig/harvest.ts` then counts it. `collectTurnFindings` (`:141`) reads every `*-pending.json` +in the turn's directory and folds their findings into that turn's totals. + +### This is not an edge case — measured, not inferred + +Across the three recorded pilots (36 turns): + +| | pilot-01 | pilot-02 | pilot-03 | total | +|---|---|---|---|---| +| turns opening with the previous turn's final report | 11/12 | 11/12 | 9/12 | **31/36** | +| archived reports owned by the turn | 19 | 14 | 14 | **47** | +| archived reports inherited from an earlier turn | 11 | 11 | 9 | **31** | +| archived reports owned by no turn at all | 0 | 0 | 0 | **0** | + +Every one of the 31 inherited reports is byte-identical to its predecessor's final archived report. + +Turns whose finding count is inflated by findings they did not produce: + +- pilot-01: turns 2, 3, 5, 8, 10 — of which 3, 8 and 10 have **zero** own findings +- pilot-02: turns 3, 5, 10, 12 — **all four** have zero own findings +- pilot-03: turns 4, 5, 8, 11 — of which 5 and 8 have zero own findings + +**13 of 36 turns count findings they did not earn; 9 of those produced none of their own.** +The sharpest instance is pilot-03 turn 5: its audit delta is empty (the gate produced no +`run.complete` at all), yet it currently reports 3 findings — every one of them turn 4's. + +### What it corrupts + +- **Cross-turn double counting.** `collectTurnFindings` dedupes by signature *within* a turn + precisely so a finding surviving three iterations is not counted three times — the comment at + `:129-135` says counting it repeatedly "would inflate the M2 denominator and M6". That dedup is + per-turn, so the same finding is counted once in turn N-1 and again in turn N, which is the exact + failure the dedup exists to prevent. +- **M2** — `fpBurden = rejectedAsFp / findingsTotal` (`:450`) takes a polluted denominator, and each + polluted point enters the OLS slope (`:549-551`). +- **M3 recall** — `caught` is computed from `blockingTexts` (`:454`), which include inherited findings. +- **M4 escape rate** — `flaggedLater` scans the same texts across turns (`:527`). +- **Suppression totals** (`:557`) double-count an inherited report's suppressions. +- **`criticRuns`** — an inherited report's critic object is attributed to the wrong turn. + +### Why it has looked covered + +The existing test `"a turn where the gate never ran is a warning, not a silent zero"` +(`tests/unit/rig-harvest.test.ts:609`) builds its dead turn with `reports: []` — it exercises the +case where nothing was inherited. The driver's `gateReviewed` flag does detect an unreviewed turn +(`driver.ts:365`, warned at `:382`), but **`harvest.ts` never reads it** — the flag exists and is +not consulted. The `iterations === 0` warning (`harvest.ts:420-424`) says such a turn is "EXCLUDED +from the M1/cost-per-turn samples", which is true and creates the false impression that dead turns +are handled; findings, recall, escape and suppression are not excluded. + +## The rule + +A report belongs to the turn whose audit delta contains its `run_id`. + +| Report's `run_id` | Meaning | Treatment | +|---|---|---| +| in this turn's `runDelta.added` | **own** — this turn's gate produced it | counted, as today | +| in an earlier turn's audit set | **inherited** — already counted where produced | dropped, one warning per turn | +| in no turn's audit set | **orphan** — pruned audit or foreign snapshot | dropped, one loud warning per report | + +### Why `run_id` alone, not `(run_id, iter)` + +A gate run lives inside one Stop hook and therefore inside one turn. Verified across all 34 +recorded gate runs in the three pilots: **no `run_id` appears in more than one turn's audit delta.** + +Keying on the pair would be strictly worse: if a gate writes `pending.json` for iteration 3 and then +dies before appending `run.complete`, the pair-key calls that report an orphan and drops real data, +while the `run_id` key correctly keeps it. `criticRuns` retains its internal `run_id:iter` keying — +that answers a different question (invocation identity *within* a turn) and is unaffected. + +### Why this discriminator and not content hashing + +`harvest.ts:149-156` already rejects content hashing for exactly this class of question, in favour +of `run_id:iter`, because hashing "would silently collapse two genuinely distinct invocations that +happened to report equal counts". A cross-turn byte-hash dedup would re-introduce the rejected +approach, and it cannot distinguish "same report inherited" from "different run, identical bytes". + +## Components + +**`src/rig/harvest.ts` — the guard.** `collectTurnFindings` gains the owned-`run_id` set as a +parameter. `harvestTurn` already computes `runDelta` at `:405`, three lines before it calls +`collectTurnFindings` at `:413`, so the data is in hand and no new plumbing crosses a module +boundary. This half works retroactively on the recorded pilots and needs no rebuild. + +Three consequences fall out rather than needing separate handling: + +- `criticRuns` is repaired for free, since filtering upstream stops an inherited report's critic + object reaching the wrong turn. +- A turn with `iterations === 0` drops to zero findings, because its audit delta is empty so nothing + can be owned. +- `reportsRead` counts only owned reports, keeping the existing "the gate ran but NO `pending.json` + was archived" warning (`:431`) truthful instead of masked by an inherited file. + +**`src/rig/driver.ts` — the hygiene fix.** `startReportArchiver` seeds its `seen` set with the +hashes of `pending.json` and `pending.md` as they exist *before* the agent starts, so a version +unchanged since the previous turn is never archived. This makes the archiver match its own +docstring. It is not the guard — it reaches future runs only, and only after a rebuild. + +Nothing is lost by skipping: all 31 inherited reports are byte-identical to a report the previous +turn's archiver already captured, because that archiver's final sweep (`:245`) records the file's +end-of-turn state, which is exactly what the next turn inherits. + +## Failure handling + +The rule removes findings from a turn, which makes it a suppressor, and a suppressor must fail safe. + +- **Inherited → one warning per turn**, stating the count and the owning turn, and that the findings + are not lost but counted where they were produced. Per-report warnings would bury the signal. +- **Orphan → one loud warning per report**, and the report is dropped. This is the branch with no + real-data coverage (0 occurrences in 36 turns), so it gets the noisiest treatment. Dropping rather + than keeping follows the rig's "missing data is not zero" stance: a report that cannot be + attributed to a turn must not be silently attributed to *this* one. +- **Neither is fatal.** This is a deliberate judgement call against the nearest precedent: + `harvestTurn` *does* throw when the audit log shrinks (`:407-411`). The closer precedent is the + unreadable-report policy (`:174-179`, guarded by the test at `rig-harvest.test.ts:637`), whose + rationale — losing a whole expensive run's numbers to one unreadable file is worse — applies here. + A shrinking audit log invalidates every per-turn delta in the chain; one unattributable report + does not. + +Two behaviours stated explicitly rather than left to be discovered: + +- **A snapshot with no audit tree turns every report into an orphan**, so the turn reports zero + findings with loud warnings. This is consistent — such a turn already has `iterations === 0` and + is already excluded from M1 — but it is a real behaviour change for any legacy run lacking an + audit tree. There are none among the three pilots. +- **A report whose `run.complete` lands after its own turn's snapshot** would be an orphan in turn N + and owned by turn N+1. `awaitQuiescent` waits for `gate.lock` release before snapshotting, so this + should not arise; if it ever does, attributing the report to the turn whose audit actually contains + it is still the defensible answer. + +## Tests + +**Fixture rework first, because it gates everything.** `auditLine` emits +`run_id: "session-"` (`rig-harvest.test.ts:93`) but `pendingReport` hardcodes +`run_id: "session-x"` for every report (`:144`). Under the new rule every existing fixture report +becomes an orphan and every existing finding-count assertion breaks. `pendingReport` must take the +turn index and emit the matching `run_id`, with a per-report override so a test can deliberately +construct an inherited or orphan report. + +Each test carries its two numbers, so a vacuous test is caught on paper before it is written: + +| Test | Without the fix | With the fix | +|---|---|---| +| an inherited report is not counted again in the turn that merely saw it | 2 findings | 1 | +| a turn the gate never reviewed reports nothing, not its predecessor's findings | 3 | 0 | +| a report owned by no turn is dropped and warned about | 1, no warning | 0, warning | +| `criticRuns` is not attributed to a turn that only inherited the report | 1 critic run | 0 | +| `reportsRead` counts only owned reports, so the unmeasured-turn warning fires | no warning | warning | +| driver: a `pending.json` unchanged since before the turn is not archived | 1 file | 0 | +| driver: a `pending.json` that changes during the turn is still archived | 1 file | 1 file | + +The last row has identical numbers on both sides by design. It is not vacuous but an +over-suppression guard: a driver fix that skipped on filename, or on "a file existed", rather than +on content hash would redden exactly there and nowhere else. + +Every test is mutation-checked in a **copy** of the repo and seen red once before being believed. + +## The correction deliverable + +The pre-fix baseline is already captured, so the delta cannot be back-fitted: + +| | pilot-01 | pilot-02 | pilot-03 | +|---|---|---|---| +| recall | 0.60 (3/5) | 0.33 (1/3) | 1.00 (2/2) | +| escape rate | 0.20 (1/5) | 0.67 (2/3) | 0.00 (0/2) | +| M2 slope | 0.0239/turn (n=10) | 0.0000/turn (n=9) | 0.0014/turn (n=9) | +| iterations median | 1 over 12 reviewed | 1 over 12 reviewed | 1 over 10 reviewed | +| cost | $0.0236 | $0.0125 | $0.0136 | + +After the fix is green, re-harvest all three pilots offline (`bun run dev rig harvest` — no binary, +no agent quota) and write `docs/dev/2026-08-07-rig-stale-report-correction.md` with before/after per +pilot and per metric, plus a correction to any write-up quoting the superseded numbers. Because +`rig/results/` is gitignored, the table belongs in the document — the numbers are otherwise +reproducible only on this machine. + +**No deltas are predicted here, deliberately.** What is established is that finding counts, the M2 +denominator and slope, suppression totals and `criticRuns` attribution are wrong on 13 of 36 turns. +Whether recall or escape rate move is an open question the re-harvest answers. The one seeded turn +checked by hand — pilot-01 turn 2 — was caught by a `path-traversal-readtemplate` finding in its +**own** report (run `01KZ8C82`); its two inherited findings are an INFO `generic-interface-coverage` +and a WARN `no-type-constraints`, neither matching the seed tags. So at least one plausible recall +inflation is ruled out, and the rest is unknown until measured. + +A re-harvest is deterministic offline recomputation, so it needs no preregistration in the sense the +pilot runs did. The guard against tuning the rule until the numbers look better is that the rule is +fixed **in this document, before** the deltas are computed. + +## Out of scope, deliberately + +- **`bun run build`.** The driver fix reaches future runs only after a rebuild, which stays a + separate, deliberately-taken step with its own sha notation. The installed binary stays + `sha256:fc9b8c18…` for this work, and a second session committing live into this checkout stays + undisturbed. +- Reading `manifest.turns[].gateReviewed` in the harvester. The `run_id` rule subsumes it: a turn + the gate never reviewed has an empty audit delta and therefore owns nothing. Adding a second, + weaker signal would give two sources of truth for one question. From 0f8b6cf2b6c3871178b63dd022a5258643948f2b Mon Sep 17 00:00:00 2001 From: Codevena Date: Fri, 7 Aug 2026 13:08:12 +0200 Subject: [PATCH 06/55] bench: Per-Case-Kosten console-direkt gemessen (28.2), Token-Modell laeuft 9% zu niedrig --- bench/results/qwen-overhead/DECISION.md | 44 +++++++++++++++++-------- 1 file changed, 31 insertions(+), 13 deletions(-) diff --git a/bench/results/qwen-overhead/DECISION.md b/bench/results/qwen-overhead/DECISION.md index f3dd135..b8fb4bf 100644 --- a/bench/results/qwen-overhead/DECISION.md +++ b/bench/results/qwen-overhead/DECISION.md @@ -36,8 +36,21 @@ timestamp from opencode's session DB: | 13:01:30 | 1,334 | 22,528 | 6.57 | | | | **total** | **53.47 for 2 cases** | -**≈ 26.7 credits per case.** A case costs roughly **two** LLM calls, not one, and -the first call of a run pays a cold cache. +**Superseded by a direct console measurement.** The table above converts DB tokens +through a model; the console measures credits directly, and it is the better +number. Reading before the TTL call: **5.55 %** (138.75 credits). Reading after the +bench run: **8.17 %** (204.25 credits) — a delta of **65.5 credits** covering the +TTL call plus the four bench calls. Subtracting the TTL call at the measured +average for comparable calls (~9.2) leaves **≈ 56.3 credits for 2 cases**: + +> **≈ 28.2 credits per case** (direct), against ≈ 26.7 from the token model. + +The token model runs ~9 % low (predicted 7.95 %, actual 8.17 %), so every +DB-derived figure in this document is a slight underestimate. The direction of +every conclusion below is unchanged, and if anything reinforced. + +A case costs roughly **two** LLM calls, not one, and the first call of a run pays +a cold cache. This supersedes every earlier extrapolation, exactly as the plan said it would: @@ -46,7 +59,8 @@ This supersedes every earlier extrapolation, exactly as the plan said it would: | baseline, default agent (Task 2) | 31.0 /call | 900 (36 %) | 2,700 (108 %) | | + reduced tool set (Task 2) | 22.9 /call | 686 (27 %) | 2,058 (82 %) | | + warm cache (Task 3) | 9.2 /call | 275 (11 %) | 825 (33 %) | -| **real case (Task 5)** | **26.7 /case** | **800 (32 %)** | **2,400 (96 %)** | +| real case, token model (Task 5) | 26.7 /case | 800 (32 %) | 2,400 (96 %) | +| **real case, console-measured (Task 5)** | **28.2 /case** | **846 (34 %)** | **2,538 (102 %)** | The Task 3 figure was a per-**call** number measured on a 5-output-token prompt. The plan predicted a realistic output mix would add +3.6 to +7.3 credits. It added @@ -55,17 +69,18 @@ The plan predicted a realistic output mix would add +3.6 to +7.3 credits. It add ## Decision against the stop condition Spec §5 Phase 0b: stop if per-case cost cannot be brought under **20 credits**. -Measured: **26.7**. The condition is **tripped**. +Measured: **28.2** (console-direct). The condition is **tripped**. But the consequence is narrower than "stop everything", because the two runs have very different costs: -- **Phase 2 (exploratory, 30 × 1): GO.** 800 credits ≈ 32 % of the 2,500-credit +- **Phase 2 (exploratory, 30 × 1): GO.** ≈ 846 credits ≈ 34 % of the 2,500-credit weekly window. It fits, and it is the only way to answer the actual question in spec §6 — whether Qwen finds a seeded bug that GLM-5.2 and claude-code both miss. -- **Phase 3 (authoritative, 30 × 3): NO-GO at Lite.** 2,400 credits ≈ 96 % of the - window. Running it consumes essentially the entire week and leaves nothing for - day-to-day gate traffic. + Note the window already stands at 8.17 % used, so budget ~42 % after it. +- **Phase 3 (authoritative, 30 × 3): NO-GO at Lite.** ≈ 2,538 credits ≈ **102 %** + of the window — it does not fit at all, let alone leave room for day-to-day gate + traffic. **Recommended next step: run Phase 2.** It is affordable, it answers the quality question, and it produces the definitive per-case cost from 30 real cases instead @@ -73,11 +88,14 @@ of the 2-case extrapolation above — which is the number Phase 3 needs. ## Caveats, stated rather than buried -- **Not cross-checked against the console.** The credit figures convert DB tokens - at 1.21/1K uncached and 0.22/1K cached. That model was validated once - (4.45 % → 5.55 %, predicted 5.42 %, actual 5.55 %), but *this* extrapolation was - not. A single console reading would confirm or refute it: the predicted total - after the TTL call plus the four bench calls is **≈ 7.95 %**. +- **Cross-checked against the console — and the token model lost.** Predicted + 7.95 %, actual **8.17 %**: the model (1.21/1K uncached, 0.22/1K cached) runs ~9 % + low. Attempts to re-fit the two coefficients across three calibration points do + not converge — the fits disagree wildly (uncached 1.25–1.71/1K, cached + 0.05–0.43/1K) because the console displays only two decimals (±0.125 credits) and + the sample sizes are small. **Do not trust a fitted coefficient; read the + console.** The per-case figure used above is console-direct for exactly that + reason. - **Cold-start amortisation is unquantified.** Only the first case of a run pays the cold cache. If steady state is the last two calls (12.10 credits/case), 30 cases would be ~380 credits (15 %) and the authoritative run ~1,140 (46 %), From f36abf1d9a9714123b6981a85f78a1f039e2d023 Mon Sep 17 00:00:00 2001 From: Codevena Date: Fri, 7 Aug 2026 13:12:56 +0200 Subject: [PATCH 07/55] =?UTF-8?q?plan:=20rig=20stale-report=20fix=20?= =?UTF-8?q?=E2=80=94=20run=5Fid=20ownership,=20driver=20seeding,=20pilot?= =?UTF-8?q?=20re-harvest?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../plans/2026-08-07-rig-stale-report-fix.md | 599 ++++++++++++++++++ 1 file changed, 599 insertions(+) create mode 100644 docs/superpowers/plans/2026-08-07-rig-stale-report-fix.md diff --git a/docs/superpowers/plans/2026-08-07-rig-stale-report-fix.md b/docs/superpowers/plans/2026-08-07-rig-stale-report-fix.md new file mode 100644 index 0000000..54acaf6 --- /dev/null +++ b/docs/superpowers/plans/2026-08-07-rig-stale-report-fix.md @@ -0,0 +1,599 @@ +# Rig stale-report fix — Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Stop the rig attributing one turn's `pending.json` to the next turn, which currently +inflates 13 of 36 recorded pilot turns with findings they never produced. + +**Architecture:** A report belongs to the turn whose audit delta contains its `run_id`. The +harvester enforces this (works retroactively on already-recorded runs, no rebuild); the driver +additionally stops archiving a report that was already on disk before the turn began (forward-only +hygiene). Design and evidence: `docs/superpowers/specs/2026-08-07-rig-stale-report-design.md`. + +**Tech Stack:** Bun, TypeScript, Biome, `bun test`. + +## Global Constraints + +- **Bun only.** `bun test`, never jest/vitest. Single test: `bun test tests/unit/foo.test.ts`. +- **Before calling anything done:** `bunx tsc --noEmit` **and** `bun run lint`, both clean. +- **Never pipe `bun test` through `tail`** — a red test's identity is lost. Redirect to a file. +- **Never run `bun run build`.** It re-pins the binary and deploys machine-wide via the + `~/.local/bin/reviewgate` symlink. Explicitly out of scope; the binary stays `sha256:fc9b8c18…`. +- **A SECOND SESSION commits to this checkout live.** Never `git add -A`. Stage explicit paths and + check `git log` before assuming a commit is yours. +- **Never commit `.reviewgate/` state** (it is live gate state, not source). +- Ownership key is `run_id` **alone**, never `(run_id, iter)` — a gate that dies before appending + `run.complete` would otherwise have its real report dropped as an orphan. + +--- + +## File Structure + +| File | Responsibility | Change | +|---|---|---| +| `src/rig/harvest.ts` | the guard: decides which archived reports a turn owns | modify `collectTurnFindings` + its call site in `harvestTurn` | +| `src/rig/driver.ts` | hygiene: stops writing the misleading artifact | modify `startReportArchiver` | +| `tests/unit/rig-harvest.test.ts` | fixture + 5 ownership guards | modify `pendingReport`/`FxTurn`/`buildFixture`, add 5 tests | +| `tests/unit/rig-driver.test.ts` | 2 archiver guards | add 2 tests | +| `docs/dev/2026-08-07-rig-stale-report-correction.md` | before/after numbers | create | + +--- + +## Task 1: Align fixture reports with their turn's audit run_id + +Pure refactor — no production code changes, every existing test stays green. It must land first: +`auditLine` emits `run_id: "session-"` (`tests/unit/rig-harvest.test.ts:93`) while +`pendingReport` hardcodes `run_id: "session-x"` (`:144`). Under Task 2's rule every existing fixture +report would become an orphan and every finding-count assertion would break for the wrong reason. + +**Files:** +- Modify: `tests/unit/rig-harvest.test.ts:141-146` (`pendingReport`), the `FxTurn` interface + (around `:55-74`), and the `buildFixture` report loop (`:253-258`) + +**Interfaces:** +- Consumes: nothing +- Produces: `pendingReport(findings: FxFinding[], iter: number, critic: FxCritic | undefined, runId: string): string` + and a new optional `FxTurn.reportRunIds?: (string | undefined)[]`, which Task 2's tests use to + construct inherited and orphan reports. + +- [ ] **Step 1: Add `reportRunIds` to the `FxTurn` interface** + +Insert directly after the `reportIters?: number[];` field: + +```ts + /** + * parallel to `reports`: the `run_id` that version carries. Defaults to this turn's own + * (`session-`), i.e. a report the turn's own gate produced. Set it to an EARLIER + * turn's id to model the archiver catching a leftover `pending.json`, or to an id no audit + * event carries to model a report that cannot be attributed to any turn. + */ + reportRunIds?: (string | undefined)[]; +``` + +- [ ] **Step 2: Make `pendingReport` take the run_id** + +Replace the signature and the `run_id` line: + +```ts +function pendingReport( + findings: FxFinding[], + iter: number, + critic: FxCritic | undefined, + runId: string, +): string { + return JSON.stringify({ + schema: "reviewgate.pending.v1", + run_id: runId, +``` + +Leave the rest of the object untouched. + +- [ ] **Step 3: Pass this turn's run_id at the call site** + +In `buildFixture`, replace the `writeFileSync(join(reportDir, ...))` call: + +```ts + for (const [n, findings] of reports.entries()) { + writeFileSync( + join(reportDir, `${n + 1}-pending.json`), + pendingReport( + findings, + turn.reportIters?.[n] ?? n + 1, + turn.critics?.[n], + turn.reportRunIds?.[n] ?? `session-${index}`, + ), + ); + } +``` + +- [ ] **Step 4: Run the harvest suite — everything must still pass** + +Run: `bun test tests/unit/rig-harvest.test.ts > /tmp/t1.txt 2>&1; tail -5 /tmp/t1.txt` +Expected: 0 fail. This task changes no behaviour — `criticRuns` is keyed `run_id:iter` within a +single turn's Map, so renaming the run_id consistently cannot change any grouping. + +- [ ] **Step 5: Commit** + +```bash +git add tests/unit/rig-harvest.test.ts +git commit -m "test(rig): fixture reports carry their own turn's run_id" +``` + +--- + +## Task 2: Ownership by run_id in the harvester + +**Files:** +- Modify: `src/rig/harvest.ts:141-211` (`collectTurnFindings`) and `:413-417` (its call site) +- Test: `tests/unit/rig-harvest.test.ts` + +**Interfaces:** +- Consumes: `FxTurn.reportRunIds` from Task 1 +- Produces: `collectTurnFindings(snapshotDir: string, turnIndex: number, warnings: string[], ownedRunIds: Set, knownRunIds: Set)` — same return shape as before + +- [ ] **Step 1: Write all five failing tests** + +Append inside the existing `describe("rig harvest", ...)` block: + +```ts + test("an inherited report is not counted again in the turn that merely saw it", () => { + const fx = buildFixture([ + { seeded: null, iterations: [{ warn: 1 }], reports: [[{ signature: "s1" }]] }, + { + seeded: null, + iterations: [{ warn: 1 }], + // The archiver's first poll caught turn 1's leftover, then turn 2's own report. + reports: [[{ signature: "s1" }], [{ signature: "s2" }]], + reportRunIds: ["session-1", undefined], + reportIters: [1, 1], + }, + ]); + const result = harvest(fx.manifestPath, fx.scriptPath); + + expect(result.turns[1]?.findingsTotal).toBe(1); + expect(result.turns[1]?.findings[0]?.signature).toBe("s2"); + expect(result.warnings.some((w) => w.includes("turn 2") && /EARLIER turn/.test(w))).toBe(true); + }); + + test("a turn the gate never reviewed reports nothing, not its predecessor's findings", () => { + const fx = buildFixture([ + { + seeded: null, + iterations: [{ warn: 3 }], + reports: [[{ signature: "a" }, { signature: "b" }, { signature: "c" }]], + }, + // The agent died; the gate never ran. Only turn 1's leftover was on disk to archive. + { + seeded: null, + iterations: [], + reports: [[{ signature: "a" }, { signature: "b" }, { signature: "c" }]], + reportRunIds: ["session-1"], + }, + ]); + const result = harvest(fx.manifestPath, fx.scriptPath); + + expect(result.turns[1]?.iterations).toBe(0); + expect(result.turns[1]?.findingsTotal).toBe(0); + }); + + test("a report owned by NO turn is dropped and warned about, not charged to this turn", () => { + const fx = buildFixture([ + { + seeded: null, + iterations: [{ warn: 1 }], + reports: [[{ signature: "own" }], [{ signature: "ghost" }]], + reportRunIds: [undefined, "session-99"], + reportIters: [1, 1], + }, + ]); + const result = harvest(fx.manifestPath, fx.scriptPath); + + expect(result.turns[0]?.findingsTotal).toBe(1); + expect(result.warnings.some((w) => /NO turn's audit events/.test(w))).toBe(true); + }); + + test("criticRuns is not attributed to a turn that only INHERITED the report", () => { + const critic = { provider: "ollama", status: "ran" as const, verdicts: 2, demoted: 1 }; + const fx = buildFixture([ + { + seeded: null, + iterations: [{ warn: 1 }], + reports: [[{ signature: "s1" }]], + critics: [critic], + }, + { + seeded: null, + iterations: [{ warn: 0 }], + reports: [[{ signature: "s1" }], []], + critics: [critic, undefined], + reportRunIds: ["session-1", undefined], + reportIters: [1, 1], + }, + ]); + const result = harvest(fx.manifestPath, fx.scriptPath); + + expect(result.turns[0]?.criticRuns?.length).toBe(1); + expect(result.turns[1]?.criticRuns ?? []).toEqual([]); + }); + + test("reportsRead counts only OWNED reports, so the unmeasured-turn warning still fires", () => { + const fx = buildFixture([ + { seeded: null, iterations: [{ warn: 1 }], reports: [[{ signature: "s1" }]] }, + // The gate DID run, but the archiver caught only turn 1's leftover — none of turn 2's own. + { + seeded: null, + iterations: [{ warn: 1 }], + reports: [[{ signature: "s1" }]], + reportRunIds: ["session-1"], + }, + ]); + const result = harvest(fx.manifestPath, fx.scriptPath); + + expect(result.turns[1]?.iterations).toBe(1); + expect(result.turns[1]?.findingsTotal).toBe(0); + expect( + result.warnings.some((w) => w.includes("turn 2") && /NO pending\.json was archived/.test(w)), + ).toBe(true); + }); +``` + +- [ ] **Step 2: Run them and record that each is RED** + +Run: `bun test tests/unit/rig-harvest.test.ts > /tmp/t2-red.txt 2>&1; grep -c "(fail)" /tmp/t2-red.txt` +Expected: **5 failures.** This run IS the mutation evidence for all five — each is red in the +absence of the mechanism it guards. Keep `/tmp/t2-red.txt`; the expected pre-fix values are +2 findings, 3 findings, 2 findings, 1 criticRun, and a missing warning respectively. + +- [ ] **Step 3: Add the ownership filter to `collectTurnFindings`** + +Change the signature: + +```ts +function collectTurnFindings( + snapshotDir: string, + turnIndex: number, + warnings: string[], + ownedRunIds: Set, + knownRunIds: Set, +): { findings: Finding[]; panel: PanelSlot[]; reportsRead: number; criticRuns: CriticInfo[] } { +``` + +Declare the two tallies next to `let reportsRead = 0;`: + +```ts + let inheritedCount = 0; + const orphanNames: string[] = []; +``` + +Insert the filter immediately after the `PendingReportSchema.safeParse` guard and **before** +`reportsRead++`: + +```ts + // OWNERSHIP. The archiver captures whatever `pending.json` is on disk, which on 31 of 36 + // recorded pilot turns was the PREVIOUS turn's leftover — counting it here would count one + // finding once in the turn that produced it and again in the turn that merely saw it, the + // very double-count the per-turn signature dedup above exists to prevent. A gate run lives + // inside one Stop hook and therefore one turn, so `run_id` alone identifies the owner + // (verified 1:1 across all 34 recorded gate runs). Keyed on run_id and NOT on (run_id, iter): + // a gate that writes a report and then dies before appending `run.complete` would otherwise + // have its real report discarded as unattributable. + const runId = parsed.data.run_id; + if (!ownedRunIds.has(runId)) { + if (knownRunIds.has(runId)) inheritedCount++; + else orphanNames.push(name); + continue; + } + reportsRead++; +``` + +(Delete the now-duplicated original `reportsRead++` line.) + +Add the warnings immediately before the `return {` at the end of the function: + +```ts + // One line per TURN, not per report: naming each of eleven inherited files would bury the + // signal. Nothing is lost — each is counted in the turn whose gate produced it. + if (inheritedCount > 0) { + warnings.push( + `turn ${turnIndex}: ${inheritedCount} archived report(s) carry a run_id produced by an EARLIER turn — the gate did not write them during this turn. They are EXCLUDED here and counted where they were produced, so one finding is not counted twice across turns.`, + ); + } + // One line per REPORT, and loud: unlike an inherited report, an orphan is not counted anywhere, + // so this is real data loss rather than a correction. + for (const orphan of orphanNames) { + warnings.push( + `turn ${turnIndex}: archived report ${orphan} carries a run_id that appears in NO turn's audit events and was EXCLUDED — it cannot be attributed to any turn (pruned audit day-partition, or a snapshot from a different run). This turn's findings may be UNDERSTATED.`, + ); + } +``` + +- [ ] **Step 4: Pass the two sets in at the call site** + +In `harvestTurn`, directly after the `runDelta`/`decisionDelta` block and before the +`collectTurnFindings` call: + +```ts + // `window.runs` is cumulative for this snapshot, so it carries every earlier turn's runs too — + // which is exactly what distinguishes an INHERITED report (owned by an earlier turn) from an + // ORPHAN (owned by none). + const ownedRunIds = new Set(runDelta.added.map((r) => r.run_id)); + const knownRunIds = new Set(window.runs.map((r) => r.run_id)); +``` + +Then change the call itself: + +```ts + const { findings, panel, reportsRead, criticRuns } = collectTurnFindings( + snapshotDir, + index, + warnings, + ownedRunIds, + knownRunIds, + ); +``` + +- [ ] **Step 5: Run the full harvest suite** + +Run: `bun test tests/unit/rig-harvest.test.ts > /tmp/t2-green.txt 2>&1; tail -5 /tmp/t2-green.txt` +Expected: 0 fail — the five new tests pass and no pre-existing test regressed. + +- [ ] **Step 6: Static gates** + +Run: `bunx tsc --noEmit && bun run lint` +Expected: both clean. + +- [ ] **Step 7: Commit** + +```bash +git add src/rig/harvest.ts tests/unit/rig-harvest.test.ts +git commit -m "fix(rig): a report belongs to the turn whose audit delta owns its run_id" +``` + +--- + +## Task 3: Stop the driver archiving a pre-existing report + +**Files:** +- Modify: `src/rig/driver.ts:214-247` (`startReportArchiver`) +- Test: `tests/unit/rig-driver.test.ts` (insert after the existing archiver tests, around `:313`) + +**Interfaces:** +- Consumes: nothing from Tasks 1–2 (independent; the harvester is the actual guard) +- Produces: no signature change — `startReportArchiver(repoRoot: string, destDir: string): () => void` + +- [ ] **Step 1: Write both failing tests** + +Insert inside the same `describe("rig driver", ...)` block that contains `gateLikeWriter`: + +```ts + test("does NOT archive a pending.json left behind by the PREVIOUS turn", async () => { + // The archiver promises every version that APPEARS while the turn runs. A file already on + // disk when the turn starts did not appear during it — on 31 of 36 recorded pilot turns this + // leftover was archived as that turn's report #1. + const { root, scriptPath } = sandbox(1); + const pending = join(root, ".reviewgate", "pending.json"); + writeFileSync(pending, '{"verdict":"FAIL","findings":[{"rule_id":"stale-from-last-turn"}]}'); + const manifest = await runDriver({ + scriptPath, + outDir: join(root, "out"), + repoRoot: root, + agentCmd: appendingAgent(root), // touches agent.log only; pending.json is never rewritten + maxTurns: 1, + }); + const reportsDir = join(manifest.turns[0]?.snapshotDir ?? "", "reports"); + const archived = existsSync(reportsDir) + ? readdirSync(reportsDir).filter((f) => f.endsWith("pending.json")) + : []; + expect(archived).toEqual([]); + }, 20_000); + + test("still archives a pending.json that CHANGES during the turn, even if one existed before", async () => { + // The over-suppression guard. A fix that skipped on filename, or on "a file was already + // there", rather than on CONTENT HASH would swallow this turn's real report. + const { root, scriptPath } = sandbox(1); + const pending = join(root, ".reviewgate", "pending.json"); + writeFileSync(pending, '{"verdict":"FAIL","findings":[{"rule_id":"stale-from-last-turn"}]}'); + const manifest = await runDriver({ + scriptPath, + outDir: join(root, "out"), + repoRoot: root, + agentCmd: gateLikeWriter([ + { file: pending, body: '{"verdict":"FAIL","findings":[{"rule_id":"fresh-this-turn"}]}' }, + ]), + maxTurns: 1, + }); + const reportsDir = join(manifest.turns[0]?.snapshotDir ?? "", "reports"); + const archived = readdirSync(reportsDir) + .filter((f) => f.endsWith("pending.json")) + .map((f) => readFileSync(join(reportsDir, f), "utf8")); + expect(archived.some((c) => c.includes("fresh-this-turn"))).toBe(true); + expect(archived.some((c) => c.includes("stale-from-last-turn"))).toBe(false); + }, 20_000); +``` + +- [ ] **Step 2: Run them and record which is red** + +Run: `bun test tests/unit/rig-driver.test.ts > /tmp/t3-red.txt 2>&1; grep -c "(fail)" /tmp/t3-red.txt` +Expected: **exactly 1 failure** — the first test (1 archived file instead of 0). The second test +passes already **by design**: it is an over-suppression guard, so its job is to be green on both +sides. Its mutation check is Step 5, which mutates the *fix* rather than removing it. + +- [ ] **Step 3: Seed the archiver with the pre-turn state** + +In `startReportArchiver`, insert immediately after `const seen = new Set();`: + +```ts + // Seed with the state on disk BEFORE the agent runs. The docstring promises every version that + // APPEARS while the turn runs; without this seed the first poll (250ms in, long before this + // turn's gate has written anything) captures the PREVIOUS turn's leftover pending.json as this + // turn's report #1. Nothing is lost: the previous turn's own final sweep already archived those + // exact bytes. Hashed, not merely name-checked — a report REWRITTEN during this turn must still + // be archived. + for (const name of ["pending.json", "pending.md"]) { + const src = join(reviewgateDir(repoRoot), name); + if (!existsSync(src)) continue; + try { + seen.add(`${name}:${createHash("sha256").update(readFileSync(src, "utf8")).digest("hex")}`); + } catch { + /* unreadable this instant → it is simply not seeded, and a later tick captures it */ + } + } +``` + +`createHash`, `existsSync`, `readFileSync`, `join` and `reviewgateDir` are already imported in this +file — verify rather than re-adding them. + +- [ ] **Step 4: Run the driver suite** + +Run: `bun test tests/unit/rig-driver.test.ts > /tmp/t3-green.txt 2>&1; tail -5 /tmp/t3-green.txt` +Expected: 0 fail. + +- [ ] **Step 5: Mutation-check the over-suppression guard in a COPY** + +The second test never went red in Step 2, so it is unproven until deliberately broken. Do this in a +copy, never in the real repo: + +```bash +cp -R /Users/markus/Developer/reviewgate /tmp/rg-mutation-check +cd /tmp/rg-mutation-check +``` + +In the copy, replace the seeding loop's `seen.add(...)` line with a name-only variant that ignores +content — `seen.add(name)` — then: + +```bash +bun test tests/unit/rig-driver.test.ts > /tmp/t3-mutant.txt 2>&1; grep -c "(fail)" /tmp/t3-mutant.txt +``` + +Expected: the over-suppression test goes **red** (the fresh report is swallowed). If it stays green +the test is vacuous and must be rewritten before proceeding. + +Then discard the copy and confirm the real repo is untouched: + +```bash +rm -rf /tmp/rg-mutation-check +cd /Users/markus/Developer/reviewgate && git status --short +``` + +- [ ] **Step 6: Static gates** + +Run: `bunx tsc --noEmit && bun run lint` +Expected: both clean. + +- [ ] **Step 7: Commit** + +```bash +git add src/rig/driver.ts tests/unit/rig-driver.test.ts +git commit -m "fix(rig): archive only reports that appear DURING the turn" +``` + +--- + +## Task 4: Re-harvest the pilots and publish the correction + +**Files:** +- Create: `docs/dev/2026-08-07-rig-stale-report-correction.md` + +**Interfaces:** +- Consumes: the corrected harvester from Task 2 +- Produces: the correction document; no code + +- [ ] **Step 1: Run the full suite once, clean** + +Run: `bun test > /tmp/full-suite.txt 2>&1; tail -5 /tmp/full-suite.txt` +Expected: 0 fail. (Baseline before this work was 3191 pass / 12 skip / 0 fail, plus the 7 new tests +and whatever the parallel session has added — do not treat the absolute count as the assertion, only +`0 fail`.) + +- [ ] **Step 2: Re-harvest all three pilots offline** + +No binary and no agent quota is involved — this runs from source. + +```bash +mkdir -p /tmp/rig-after +for p in pilot-01 pilot-02 pilot-03; do + bun run dev rig harvest \ + --manifest rig/results/$p/manifest.json \ + --script rig/scripts/$p.json \ + --out /tmp/rig-after/$p.json > /tmp/rig-after/$p.stdout 2> /tmp/rig-after/$p.stderr + echo "=== $p"; cat /tmp/rig-after/$p.stdout +done +``` + +- [ ] **Step 3: Check the warnings actually fired** + +Run: `grep -c "EARLIER turn" /tmp/rig-after/*.stderr` +Expected: non-zero for all three — pilot-01 and pilot-02 have 11 inherited reports each and +pilot-03 has 9, spread over the turns listed in the spec. A zero here means the filter never +engaged and the numbers below are not the corrected ones. + +- [ ] **Step 4: Write the correction document** + +Create `docs/dev/2026-08-07-rig-stale-report-correction.md` containing: + +1. A one-paragraph statement of the defect and its scope (31/36 turns inherited a report; 13/36 + counted findings they did not produce; 9 of those produced none of their own). +2. This before-table, which was captured **pre-fix** so the delta cannot be back-fitted: + + | | pilot-01 | pilot-02 | pilot-03 | + |---|---|---|---| + | recall | 0.60 (3/5) | 0.33 (1/3) | 1.00 (2/2) | + | escape rate | 0.20 (1/5) | 0.67 (2/3) | 0.00 (0/2) | + | M2 slope | 0.0239/turn (n=10) | 0.0000/turn (n=9) | 0.0014/turn (n=9) | + | iterations median | 1 over 12 reviewed | 1 over 12 reviewed | 1 over 10 reviewed | + | cost | $0.0236 | $0.0125 | $0.0136 | + +3. The matching after-table, read from Step 2's output. +4. A per-metric statement of what moved and what did not. State plainly if a metric did not move. +5. A note that `rig/results/` is gitignored, so these numbers are reproducible only on this machine, + and that the correction rests on the corrected harvester at the commit from Task 2. + +Do **not** assert that any specific metric moved until Step 2's output is in hand. + +- [ ] **Step 5: Correct any write-up quoting a superseded number** + +Run: `grep -rln "0\.60\|0\.33\|0\.0239" docs/dev/ docs/superpowers/ 2>/dev/null` + +For each hit, check whether the number is a rig metric from these pilots. If it is, add a dated +correction line pointing at the new document rather than silently editing the old figure — a +superseded number that vanishes without trace is how a corrected record becomes untrustworthy. + +- [ ] **Step 6: Commit** + +```bash +git add docs/dev/2026-08-07-rig-stale-report-correction.md +git commit -m "docs: correct the pilot metrics invalidated by the rig stale-report defect" +``` + +(Add any write-up files touched in Step 5 to the same `git add`, by explicit path.) + +--- + +## Post-implementation gate + +Per the repo's Definition of Done, two independent reviewers, each returning `VERDICT: PASS`: + +- **Slot A — MUST execute.** Codex is at quota until **2026-08-08 11:07Z**; until then this slot is a + Claude reviewer subagent with repo read *and* run access. Instruct it to re-run the numbers in the + correction document and to mutation-check the new tests in a copy. +- **Slot B — the second, independent voice.** `agy` (Gemini). Note it has been observed to fail + 0-byte: a missing findings file is an OPEN slot, not a pass. Check the log size **and** the + findings-file mtime against the round's start time. + +A non-executing PASS is one voice, never corroboration. + +## Self-review notes + +- **Spec coverage:** ownership rule → Task 2; `run_id`-not-pair rationale → Task 2 Step 3 comment; + driver hygiene → Task 3; inherited/orphan warning split → Task 2 Step 3; non-fatal policy → + inherited/orphan both use `warnings.push`, never `throw`; fixture rework → Task 1; all seven + tests → Tasks 2–3; correction deliverable → Task 4; rebuild excluded → Global Constraints; + `gateReviewed` deliberately not consulted → subsumed by the empty-audit-delta case, exercised by + Task 2's dead-turn test. +- **Type consistency:** `collectTurnFindings` gains `ownedRunIds`/`knownRunIds` in Task 2 Step 3 and + is called with exactly those two names in Step 4. `pendingReport`'s fourth parameter `runId` + (Task 1 Step 2) is supplied at the single call site (Step 3). `FxTurn.reportRunIds` is declared in + Task 1 Step 1 and used in Task 2 Step 1. +- **Known limitation, stated rather than hidden:** the driver fix (Task 3) changes nothing for any + future `rig run` until someone rebuilds the binary. That is deliberately out of scope here and + must be carried into the handoff as an open item. From 734f5eb0739ba461d72a07c4886748dad84d903c Mon Sep 17 00:00:00 2001 From: Codevena Date: Fri, 7 Aug 2026 13:15:54 +0200 Subject: [PATCH 08/55] =?UTF-8?q?plan:=20gate=20round=201=20=E2=80=94=20fi?= =?UTF-8?q?xture=20turns=20need=20a=20gate=20iteration;=20both=20driver=20?= =?UTF-8?q?tests=20are=20red=20pre-fix?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../plans/2026-08-07-rig-stale-report-fix.md | 49 +++++++++++++++---- 1 file changed, 40 insertions(+), 9 deletions(-) diff --git a/docs/superpowers/plans/2026-08-07-rig-stale-report-fix.md b/docs/superpowers/plans/2026-08-07-rig-stale-report-fix.md index 54acaf6..843082d 100644 --- a/docs/superpowers/plans/2026-08-07-rig-stale-report-fix.md +++ b/docs/superpowers/plans/2026-08-07-rig-stale-report-fix.md @@ -54,7 +54,8 @@ report would become an orphan and every finding-count assertion would break for - Consumes: nothing - Produces: `pendingReport(findings: FxFinding[], iter: number, critic: FxCritic | undefined, runId: string): string` and a new optional `FxTurn.reportRunIds?: (string | undefined)[]`, which Task 2's tests use to - construct inherited and orphan reports. + construct inherited and orphan reports. Also leaves every fixture turn that carries `reports` + with at least one `iterations` entry (Step 4), which Task 2's rule depends on. - [ ] **Step 1: Add `reportRunIds` to the `FxTurn` interface** @@ -106,13 +107,35 @@ In `buildFixture`, replace the `writeFileSync(join(reportDir, ...))` call: } ``` -- [ ] **Step 4: Run the harvest suite — everything must still pass** +- [ ] **Step 4: Give every fixture turn that carries reports at least one gate iteration** + +**This step is load-bearing — without it Task 2 breaks three existing tests.** Four fixture turns +declare `reports` but no `iterations`, so they emit no `run.complete` and their audit delta is +empty. Under Task 2's rule their reports would be owned by no turn, become orphans, and be dropped +— collapsing `criticRuns` to `[]`. A report cannot exist without a gate run that wrote it, so these +fixtures were modelling an impossible state; make them model reality: + +- `"criticRuns records that the critic ran, which the demotion count cannot show"` (`:360-371`) — + add `iterations: [{}],` to **both** turn objects. +- `"criticRuns dedupes one invocation repeated across archived report versions"` (`:386-398`) — + add `iterations: [{}],` (one iteration; its `reportIters` is `[1, 1]`). +- `"criticRuns keeps two distinct iterations that reported identical counts"` (`:409-418`) — + add `iterations: [{}, {}],` (two iterations; its `reportIters` is `[1, 2]`). + +These tests assert only on `criticRuns` and `suppressed.critic`, never on `iterations` or the +warning list, so adding the iterations cannot change what they check. + +Do **not** add iterations to `"a turn where the gate never ran is a warning, not a silent zero"` +(`:609-623`) or to any other fixture with `iterations: []` — those model a dead turn deliberately. + +- [ ] **Step 5: Run the harvest suite — everything must still pass** Run: `bun test tests/unit/rig-harvest.test.ts > /tmp/t1.txt 2>&1; tail -5 /tmp/t1.txt` -Expected: 0 fail. This task changes no behaviour — `criticRuns` is keyed `run_id:iter` within a -single turn's Map, so renaming the run_id consistently cannot change any grouping. +Expected: 0 fail. Task 1 changes no production behaviour — `criticRuns` is keyed `run_id:iter` +within a single turn's Map, so renaming the run_id consistently cannot change any grouping, and the +added iterations are not asserted on by the affected tests. -- [ ] **Step 5: Commit** +- [ ] **Step 6: Commit** ```bash git add tests/unit/rig-harvest.test.ts @@ -411,12 +434,20 @@ Insert inside the same `describe("rig driver", ...)` block that contains `gateLi }, 20_000); ``` -- [ ] **Step 2: Run them and record which is red** +- [ ] **Step 2: Run them and record that both are red** Run: `bun test tests/unit/rig-driver.test.ts > /tmp/t3-red.txt 2>&1; grep -c "(fail)" /tmp/t3-red.txt` -Expected: **exactly 1 failure** — the first test (1 archived file instead of 0). The second test -passes already **by design**: it is an over-suppression guard, so its job is to be green on both -sides. Its mutation check is Step 5, which mutates the *fix* rather than removing it. +Expected: **2 failures.** Both tests are red pre-fix, for different reasons, and the distinction +matters when judging whether they have teeth: + +- Test 1 fails on the archived-file count (1 instead of 0). +- Test 2 fails on its **second** assertion only: pre-fix the archiver captures the stale report + *and* the fresh one, so `stale-from-last-turn` is present when the test requires it absent. Its + first assertion (`fresh-this-turn` must be archived) is green pre-fix and stays green post-fix — + that half is the over-suppression guard, and it is the half Step 5's mutation check proves. + +So each assertion in Test 2 is covered by a different observation: the stale-exclusion half by this +pre-fix red, the fresh-retention half by the Step 5 mutant. Neither half is vacuous. - [ ] **Step 3: Seed the archiver with the pre-turn state** From 9882d6a5ae3591c03b0ab0fbb6bcd3d8d3511d1d Mon Sep 17 00:00:00 2001 From: Codevena Date: Fri, 7 Aug 2026 15:43:15 +0200 Subject: [PATCH 09/55] =?UTF-8?q?docs:=20handoff=20=E2=80=94=20rig=20stale?= =?UTF-8?q?-report=20diagnosed,=20specced=20and=20planned;=20nothing=20imp?= =?UTF-8?q?lemented?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- NEXT_SESSION.md | 196 ++++++++++++++++++++++++++---------------------- 1 file changed, 108 insertions(+), 88 deletions(-) diff --git a/NEXT_SESSION.md b/NEXT_SESSION.md index b38411a..4883b82 100644 --- a/NEXT_SESSION.md +++ b/NEXT_SESSION.md @@ -1,134 +1,154 @@ # Reviewgate — Next-Session Handoff -_Last updated: 2026-08-07, after the Slice B revert was implemented, gated and pushed. +_Last updated: 2026-08-07, after the rig stale-report defect was diagnosed, specced and planned. Supersedes all earlier content._ ## One-line state -**All three slice questions are now closed: Slice A shipped, Slice C declined on measurement, -Slice B implemented as a REVERT and pushed (`27c29f7`). Nothing is half-done — the next session -picks a new task rather than continuing one.** +**The rig stale-report defect is fully diagnosed and measured, the design and the implementation +plan are written and committed — but NOT ONE LINE OF THE FIX IS IMPLEMENTED. The next session +implements `docs/superpowers/plans/2026-08-07-rig-stale-report-fix.md`, Task 1 first.** ## Verified state (checked with commands at handoff time) | | | |---|---| -| my commits this session | **`27c29f7`** (the revert), `5543549` (this handoff), `76a758c` (Trailhead stamp) | -| pushed? | **YES, all of them** — verified on `origin/master` after the final push | -| also on master | **`b3032b3`** — *not mine.* A **second session** is committing to this checkout live; its commit rode along in my push, with Markus's explicit go-ahead | -| Trailhead stamp | `5543549`. `verify-map.js` → **MAP OK**, 70/70 paths | -| working tree | `.reviewgate/lore/approvals.jsonl` modified + two untracked `measure-opencode-tokens` files — **all foreign**, leave them alone | -| suite | **3191 pass / 12 skip / 0 fail**, exit 0 — run at the reviewed tree of `27c29f7` | -| `tsc` / `lint` | clean. `rig/` checked SEPARATELY (it is outside `tsconfig.include`) — 0 errors in the changed file | +| my commits this session | **`384df2a`** (spec), **`f36abf1`** (plan), **`734f5eb`** (plan corrections) — docs only, zero code | +| pushed? | **NO.** `master` is **8 ahead** of `origin/master` — but only 3 are mine | +| the other 5 | `0f8b6cf`, `e7c25e1`, `979bfea`, `ef54ed0`, `9fac6f8` — **a SECOND SESSION's bench/Qwen work**, still live | +| working tree | only `.reviewgate/lore/approvals.jsonl` (foreign gate state) — leave it alone | +| suite | **3209 pass / 12 skip / 0 fail**, exit 0 — run at `734f5eb` (was 3191; the parallel session added 18) | | build | **deliberately NOT run.** Installed binary still `sha256:fc9b8c18…` | -| Trailhead | stamped to `b3032b3`; 3 GEÄNDERT rows were all mine and re-checked, entry points unchanged, 0 FEHLT, `CLAUDE.md` at exactly 80/80 lines | +| Trailhead stamp | **left at `5543549` ON PURPOSE** — see Traps | ⚠ **A SECOND SESSION IS COMMITTING TO THIS CHECKOUT.** Never `git add -A`; stage explicit paths and check `git log` before assuming a commit is yours. A `git worktree` remains the standing fix. ## What got done — and how it was verified -The four tasks of `docs/superpowers/plans/2026-08-07-slice-b-revert.md`, all of them: +**Nothing was implemented. What exists is a diagnosis backed by measurement, plus a gated plan.** -1. `isBlockingSecurity` deleted; `src/core/aggregator.ts:621` is the CRITICAL-only check again. -2. The two floor tests inverted (WARN/`"keep"` → INFO/`"likely_fp"`). -3. Three boundary guards confirmed green with assertions untouched. -4. Design spec carries a dated REVERTED banner; original rationale preserved. +The handoff that suggested this task described it as "a dead turn inherits the previous turn's +`pending.json`". That was an order of magnitude too small. Measured against the recorded pilots: -**Evidence, not adjectives:** +| | pilot-01 | pilot-02 | pilot-03 | total | +|---|---|---|---|---| +| turns opening with the previous turn's final report | 11/12 | 11/12 | 9/12 | **31/36** | +| reports owned by the turn | 19 | 14 | 14 | **47** | +| reports inherited from an earlier turn | 11 | 11 | 9 | **31** | +| reports owned by **no** turn (orphans) | 0 | 0 | 0 | **0** | -- **Mutation check in a COPY** — restoring `isBlockingSecurity` reddens **exactly** the two inverted - tests, the other 16 stay green. Copy discarded, `git diff` confirmed the original untouched. -- **The new abort path was itself mutation-checked** — failure-only code that would otherwise ship - untested. Floor restored → replay exits 1, prints all 3 activations with both diagnostic flags - `false`, and the `die()` guidance then correctly points at `aggregator.ts:621`. -- **Replay 3 → 0 activations** with signature-match unchanged at **15/19** — the 0 comes from the - revert, not from a broken instrument. -- Suite unchanged at 3191/12/0, exactly as the plan predicted. +**13 of 36 turns count findings they did not earn; 9 of those produced none of their own.** +Sharpest case: pilot-03 turn 5 has an EMPTY audit delta and still reports 3 findings, all turn 4's. -**Post-implementation gate: 3 rounds** (Slot A = executing Claude subagent, Slot B = agy). Round 1 -FAIL/2 WARN, round 2 FAIL/2 WARN, round 3 PASS/PASS. +Root cause: `driver.ts:201` promises to archive every version that **appears** while a turn runs; +it was implemented as every version that **exists**. The first poll fires 250 ms in, while the +predecessor's `pending.json` is still on disk. -## THE NEXT TASK — pick one; none is a continuation +**Evidence, not adjectives — every number above came from a command run against +`rig/results/pilot-0{1,2,3}/`, not from reading code.** Also executed and confirmed: -Nothing is left mid-flight. The strongest candidate, and why: +- `run_id` maps **1:1 to a turn** across all **34** recorded gate runs — none spans two turns. + This is what makes the ownership rule sound and retroactive. +- Every one of the 31 inherited reports is **byte-identical** to its predecessor's final report, + so dropping it loses nothing. +- The **pre-fix baseline** for all three pilots is captured (table below) so the correction delta + cannot be back-fitted. +- `createHash`/`existsSync`/`readFileSync`/`join`/`reviewgateDir` are already imported in + `driver.ts:6-20`; `window.runs`/`runDelta` are in scope at the harvest insertion point. -**The rig stale-report defect** — a dead turn inherits the previous turn's `pending.json`, so a turn -that produced nothing looks like it produced the previous turn's findings. It silently corrupts any -metric read from `turns/*/reports/`, which is the exact failure class this rig has already been -burned by twice. It is next because every future measurement rests on it, and because it **cannot -ride along inside a pilot** — it needs a rebuild, so it must be its own task with its own -preregistration. Entry point: `src/rig/driver.ts` (turn loop) plus `src/rig/harvest.ts`. +**Pre-fix baseline — capture this again only if you distrust it; do not overwrite it:** -Alternatives, all still open and all smaller: +| | pilot-01 | pilot-02 | pilot-03 | +|---|---|---|---| +| recall | 0.60 (3/5) | 0.33 (1/3) | 1.00 (2/2) | +| escape rate | 0.20 (1/5) | 0.67 (2/3) | 0.00 (0/2) | +| M2 slope | 0.0239/turn (n=10) | 0.0000/turn (n=9) | 0.0014/turn (n=9) | +| iterations median | 1 over 12 reviewed | 1 over 12 reviewed | 1 over 10 reviewed | +| cost | $0.0236 | $0.0125 | $0.0136 | -1. **`isFloorActivation` is not floor-exclusive** — documented in a comment this session, **not** - guarded by a test. See the trap below. A test would be cheap. -2. **Two `~/Developer` fixes**, diagnosed, still not applied: stale repo-local hooks in - `~/Developer/.claude/settings.json`; a 15.07. `control-plane.json` that makes `~/Developer` count - as an armed checkout. -3. **Four repos armed without ever being `init`ed** (`barrierefrei`, `fatemehdaily`, `viergewinnt`, - `youtubeQuiz`) — a policy call, not a code task. -4. **Sandboxes to reap:** `/private/tmp/rig-pilot01-NZHKOT`, `/private/tmp/rig-pilot02-kzYEoV`, - `/private/tmp/rig-pilot03-a3doEy`, and `dist/reviewgate.prev`. Neither replay depends on them. +### Plan gate: ONE round, and only half a gate + +- **agy (Slot B): PASS**, 0 CRITICAL / 0 WARN / 1 INFO. Findings file verified fresh (mtime + 11:14:35Z against a round start of 11:13:26Z), log 2767 bytes. Its INFO was **correct** and is + fixed in `734f5eb`. +- **Slot A (executing): STILL OPEN.** agy's log shows a single `readFile` — it reviewed by reading, + not by executing, despite being told to run the code. +- **The proof that this matters:** I found a plan-breaking defect agy missed while it asserted "the + rule produces deterministic, safe outcomes in all cases" — four fixture turns declare `reports` + but no `iterations`, so under the new rule their reports become orphans and **three existing + `criticRuns` tests collapse to `[]`**. That is now Task 1 Step 4. + +## THE NEXT TASK + +**Implement the plan, Task 1 → Task 4, in order.** +`docs/superpowers/plans/2026-08-07-rig-stale-report-fix.md` + +Why it is next: every future rig measurement rests on the harvester being right, and the corpus is +currently wrong in a way that is invisible from the reports themselves. The harvest half works +**retroactively and needs no rebuild**, so the three recorded pilots become usable again rather than +being written off. + +Entry points: `src/rig/harvest.ts:141` (`collectTurnFindings`) and `:413` (its call site); +`src/rig/driver.ts:214` (`startReportArchiver`). + +**Task 1 must land first** — it is a test-only refactor, and without its Step 4 Task 2 reddens three +existing tests for the wrong reason. ## Traps — NEW this session -- **`isFloorActivation` (`rig/scripts/critic-floor-replay.ts`) is NOT floor-exclusive, and its 0 is - a property of the CORPUS, not a theorem about the code.** A CRITICAL **correctness** singleton - kept by the surviving CRITICAL exemption, then clamped CRITICAL→WARN by the reputation pass - (`aggregator.ts:903-914`), reproduces the same marker. Not security — `touchesSecurity` returns - early at `:883`. It cannot fire in the replay only because that script's `aggregate()` call site - passes **no** reputation inputs and the pass is gated on `repUnreliable.size > 0` (`:876-877`). - **If that call site ever gains reputation inputs, the tripwire will false-alarm.** Documented in - the script's header; NOT covered by a test. -- **The flag-based diagnosis in that script rests on an unguarded invariant:** every CRITICAL→WARN - transition inside `aggregate()` stamps `demoted_from_critical` (`:165`, `:854`, `:911`, `:1061`), - and the two non-reputation paths early-return on security/correctness (`:838`, `:1051`). - Re-check that list after ANY change to the demote passes, or the guidance points at the wrong line. -- **A "VERIFIED BY EXECUTION" stamp does not protect the sentence it is attached to.** This session - executed the *existence* of that second producer and then invented a *cause* for it — and marked - the invented cause as execution-verified. The reviewer caught it. Execute the claim you are - actually writing down, not a neighbouring one. -- **A vendor-diverse PASS is not independent confirmation when the second slot cannot run the code.** - agy passed all three rounds and found nothing; in round 2 it explicitly confirmed the false causal - claim as "accurately described against the aggregator implementation". Every substantive finding - came from the executing slot. Treat a non-executing PASS as one voice, never as corroboration. -- **Failure-only code is untested code.** The new abort branch had never run once during - development. Mutate deliberately to make it run before believing its output. +- **`run_id` alone is the ownership key, never `(run_id, iter)`.** A gate that writes + `pending.json` for iteration 3 and dies before appending `run.complete` would have its REAL report + dropped as an orphan under a pair key. Verified 1:1 across 34 gate runs. +- **Four fixture turns model an impossible state** (`reports` with no `iterations`): the three + `criticRuns` tests at `rig-harvest.test.ts:360`, `:384`, `:407`. They need a gate iteration added. + Do NOT add one to `:609` ("a turn where the gate never ran") — that one is deliberately dead. +- **The trailhead stamp was deliberately NOT moved.** All 4 GEÄNDERT rows (`tests/unit/`, + `src/cli/commands/bench.ts`, `src/bench/runner.ts`, `src/cli/commands/`) are the PARALLEL + session's bench work, which this session never looked at. Stamping HEAD would claim a verification + that did not happen. 0 FEHLT, 66/70 still valid, `CLAUDE.md` at exactly 80/80 lines. +- **The gate escalated on findings that are not mine and cannot be honestly dispositioned.** + `F-002`/`F-003` on `src/providers/opencode.ts` are the parallel session's code, but the ownership + snapshot marked them `session_attributable: true` (their edits landed inside my baseline window), + so `out-of-scope` and `out-of-session` both fail closed. They remain **open and escalated** — + see `.reviewgate/ESCALATION.md`. That escalation also rests on a **quota-degraded panel** (codex + capped until 2026-08-08 11:07Z); the file itself says to re-run after the reset before treating + the findings as final. +- **`harvest.ts` never reads `manifest.turns[].gateReviewed`** — the flag exists, is written by + the driver, and is consulted by nothing. The plan subsumes it rather than adding a second signal. +- **An `iterations === 0` warning that says "EXCLUDED from the M1/cost-per-turn samples" is true and + misleading** — findings, recall, escape and suppression were never excluded. ## Traps — still standing -- **Never run `bun run build` casually** — it re-pins the binary AND deploys machine-wide via the - `~/.local/bin/reviewgate` symlink. Build → record sha → preregister → run. +- **Never run `bun run build` casually** — re-pins the binary AND deploys machine-wide via the + `~/.local/bin/reviewgate` symlink. Build → record sha → preregister → run. **Task 3's driver fix + reaches no real `rig run` until someone rebuilds; that is deliberately out of scope.** - **Never pipe `bun test` through `tail`** — a red test's identity is lost. Redirect to a file. -- **`bun run lint`/`tsc` do NOT cover `rig/scripts/`.** Check it explicitly: `bunx biome check` plus - a `tsc --noEmit` with an include that reaches `rig/` (needs `typeRoots` pointing at - `node_modules` — `bun-types` is not under `@types/`). -- **`agy` fails 0-byte intermittently.** A reviewer with no findings file is an OPEN slot, not a - pass — check log size AND findings-file **mtime against the round's start time**. +- **`bun run lint`/`tsc` do NOT cover `rig/scripts/`.** Check it explicitly (needs `typeRoots` + pointing at `node_modules` — `bun-types` is not under `@types/`). +- **`agy` fails 0-byte intermittently, and reviews shallowly even when it does not.** A missing + findings file is an OPEN slot; so, arguably, is a PASS whose log shows no execution. - **Codex quota resets 2026-08-08 11:07Z.** Until then the executing slot is agy or a Claude subagent; that is the normal configuration, not a degraded one. - **A rate over `reports/*-pending.json` is a rate over SURVIVORS.** Use `cassette.jsonl`. -- **`rig/results/` is gitignored** — every number in the write-ups is reproducible only on this - machine. -- **Never reimplement a shipped helper in a rig script** — import it. `seedLanded` got the landing - semantics wrong that way and the whole discriminator hung off it. +- **`rig/results/` is gitignored** — every number here is reproducible only on this machine. +- **Never reimplement a shipped helper in a rig script** — import it. - **`applySymbolSignatures` runs BEFORE `validateFindingFacts`** (`orchestrator.ts:2219`, `:2226`). - Reviewgate's decision protocol assumes fix-and-decide within ONE turn; an agent that delegates a fix to a background worker structurally cannot. Still unaddressed. -## Open Trailhead note (carried forward, still unresolved) +## Open Trailhead note (carried forward) -`CLAUDE.md`'s Mess-Rig row points at `src/rig/driver.ts`, not at the offline replays under -`rig/scripts/` — which are now load-bearing (this session's revert check *is* one of them). -`CLAUDE.md` sits at exactly 80/80 lines, so this can only be a **swap**, not an addition. Deliberately -left as-is: it is a judgement call about which entry point serves a cold reader better. +`CLAUDE.md`'s Mess-Rig row points at `src/rig/driver.ts` rather than the offline replays under +`rig/scripts/`. After this session's work the row is arguably *more* correct than before — the next +task's entry points are `src/rig/driver.ts` and `src/rig/harvest.ts`. Left as-is; `CLAUDE.md` is at +exactly 80/80 lines, so any change is a swap, not an addition. ## Read-first order 1. This file. -2. `docs/dev/2026-08-07-slice-b-critic-floor-counterfactual.md` — the evidence behind the revert. -3. `rig/scripts/critic-floor-replay.ts` — read its HEADER before running it; it explains what the 0 - does and does not prove. -4. `docs/superpowers/specs/2026-08-05-true-positive-hole-design.md` §Slice B — the REVERTED banner. +2. `docs/superpowers/plans/2026-08-07-rig-stale-report-fix.md` — the plan to execute. +3. `docs/superpowers/specs/2026-08-07-rig-stale-report-design.md` — why the rule is what it is, + especially §"Why `run_id` alone" and §"Failure handling". +4. `.reviewgate/ESCALATION.md` — the open, not-mine findings, before ending your first turn. From cef7022edf87da805ea67d3cc111f54ed4459e70 Mon Sep 17 00:00:00 2001 From: Codevena Date: Fri, 7 Aug 2026 16:59:05 +0200 Subject: [PATCH 10/55] =?UTF-8?q?docs:=20Korrektur=20=E2=80=94=20Risk-Cont?= =?UTF-8?q?rol=20sperrt=20nur=20Kaeufe;=20Tarifwechsel=20offen,=20Phase=20?= =?UTF-8?q?3=20bei=20Standard=20machbar=20(25%)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- bench/results/qwen-overhead/DECISION.md | 28 +++++++++++++++---- ...-08-07-qwen-reviewer-measurement-design.md | 10 +++++-- 2 files changed, 30 insertions(+), 8 deletions(-) diff --git a/bench/results/qwen-overhead/DECISION.md b/bench/results/qwen-overhead/DECISION.md index b8fb4bf..a747367 100644 --- a/bench/results/qwen-overhead/DECISION.md +++ b/bench/results/qwen-overhead/DECISION.md @@ -78,9 +78,10 @@ very different costs: weekly window. It fits, and it is the only way to answer the actual question in spec §6 — whether Qwen finds a seeded bug that GLM-5.2 and claude-code both miss. Note the window already stands at 8.17 % used, so budget ~42 % after it. -- **Phase 3 (authoritative, 30 × 3): NO-GO at Lite.** ≈ 2,538 credits ≈ **102 %** - of the window — it does not fit at all, let alone leave room for day-to-day gate - traffic. +- **Phase 3 (authoritative, 30 × 3): NO-GO at Lite, GO at Standard.** ≈ 2,538 + credits ≈ **102 %** of the Lite window — it does not fit at all. On Standard + (10,000/window, $18) the same run is **25.4 %**. See the corrected escape-hatch + note below: a tier change is not blocked by the risk control. **Recommended next step: run Phase 2.** It is affordable, it answers the quality question, and it produces the definitive per-case cost from 30 real cases instead @@ -101,10 +102,25 @@ of the 2-case extrapolation above — which is the number Phase 3 needs. cases would be ~380 credits (15 %) and the authoritative run ~1,140 (46 %), which would change the Phase 3 verdict. That is an extrapolation from **two** data points and is not treated as a result here. Phase 2 settles it. -- **The escape hatches in spec §5a are probably closed.** Extra Bundles and a plan +- ~~**The escape hatches in spec §5a are probably closed.** Extra Bundles and a plan tier change are both *orders*, and the account carries a - `RISK.RISK_CONTROL_REJECTION` block on orders (Task 1b). Unverified — it needs a - purchase attempt — but do not plan on buying headroom. + `RISK.RISK_CONTROL_REJECTION` block on orders.~~ + **KORREKTUR (Markus, 2026-08-07): renewal and plan changes are NOT affected by + the risk-control block.** Only pay-as-you-go activation and Extra Usage Packs are. + A **Standard tier change is therefore available**, and it changes the Phase 3 + verdict: + + | | Lite (2,500/window) | **Standard (10,000/window)** | + | --- | --- | --- | + | bench 30 × 1 (846 cr) | 34 % | **8.5 %** | + | bench 30 × 3 (2,538 cr) | **102 % — does not fit** | **25.4 % — fits** | + | real reviews per week | ~88 (12/day) | **~355 (50/day)** | + | concurrent agents | 1–2 | 3–4 | + + **Phase 3 is NO-GO at Lite but GO at Standard**, for $12/month more. The 3–4 + concurrent agents also fit a three-voice panel for the first time. This does not + reach "gate on every turn across 19 repos" (that needs ~10× more), but it covers + a primary repo or a bounded panel role. - **`--auto` vs `--dangerously-skip-permissions`.** `src/providers/opencode.ts:97` now passes `--auto`; the old flag does not exist in opencode 1.18.10 and was silently ignored. **`:242` (the `complete()` / critic / curator path) still passes diff --git a/docs/superpowers/specs/2026-08-07-qwen-reviewer-measurement-design.md b/docs/superpowers/specs/2026-08-07-qwen-reviewer-measurement-design.md index 5deebb8..879e029 100644 --- a/docs/superpowers/specs/2026-08-07-qwen-reviewer-measurement-design.md +++ b/docs/superpowers/specs/2026-08-07-qwen-reviewer-measurement-design.md @@ -195,8 +195,14 @@ an improvisation. **Five are listed; four are live.** The pay-per-token DashScope route — which an earlier draft of this section recommended — was measured on 2026-08-07 and is **blocked**, so do not route the stop branch there. The live four are: -bare-completion adapter, shrink-the-scope, Extra Bundles, and a plan-tier change; -the last two are supplements to one of the first two, never substitutes. +bare-completion adapter, shrink-the-scope, Extra Bundles, and a plan-tier change. + +**Scope of the risk-control block (corrected 2026-08-07 by Markus):** it suspends +**purchases only** — pay-as-you-go activation and Extra Usage Packs. **Renewal and +plan-tier changes are NOT affected.** An earlier draft speculated otherwise. A +Standard upgrade is therefore a live option, and it is the one that resurrects the +authoritative run: 2,538 credits is 102 % of a Lite window but **25.4 % of a +Standard one**, with 3–4 concurrent agents instead of 1–2. **Correction (2026-08-07):** an earlier draft of this section treated "direct API" and "non-executing reviewer" as one option. They are **two independent axes** and From 56564ea55d2b52eaf827027977704e0acae765f4 Mon Sep 17 00:00:00 2001 From: Codevena Date: Sat, 8 Aug 2026 01:32:16 +0200 Subject: [PATCH 11/55] docs: Trailhead-Stempel neu gesetzt (bench.ts/runner.ts/tests geprueft, Einstiegspunkte unveraendert) --- CLAUDE.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 5b149eb..089e435 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -1,6 +1,6 @@ --- -verified_commit: 5543549 -verified_at: 2026-08-07 +verified_commit: cef7022 +verified_at: 2026-08-08 --- # reviewgate — Trailhead From 63a779fc282575f5b3c85d74317959c7466e339b Mon Sep 17 00:00:00 2001 From: Codevena Date: Sat, 8 Aug 2026 01:36:47 +0200 Subject: [PATCH 12/55] fix(opencode): Messgeruest aus dem Reviewer-Pfad entfernt (Gate-Finding F-002), --auto bleibt --- src/providers/opencode.ts | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/src/providers/opencode.ts b/src/providers/opencode.ts index 7ffd2db..4d1a665 100644 --- a/src/providers/opencode.ts +++ b/src/providers/opencode.ts @@ -94,19 +94,18 @@ export class OpenCodeAdapter implements ProviderAdapter { const stdoutFile = join(run, "out.txt"); const stderrFile = join(run, "err.log"); - // MEASUREMENT SCAFFOLDING (2026-08-07, docs/superpowers/plans/2026-08-07-qwen-overhead-and-provider-model.md - // Task 5) — NOT a shipped default. `--pure --agent rg-reviewer` is the cheapest - // variant measured in bench/results/qwen-overhead/tool-surface.json: it cuts the - // opencode system prompt from ~23.5K to ~17.8K input tokens by dropping the - // write/edit tool schemas a reviewer never needs. Revert before release unless - // the bench result justifies keeping it. - // // `--auto` replaces `--dangerously-skip-permissions`, which does NOT exist in // opencode 1.18.10 — verified against `opencode run --help`, and opencode exits 0 // on unknown flags instead of rejecting them, so the old flag was silently // ignored on every call. The complete() path at the bottom of this file still // passes the dead flag; fix that separately, it changes curator behaviour. - const args = ["run", "--auto", "--pure", "--agent", "rg-reviewer", "--format", "default"]; + // + // The `--pure --agent rg-reviewer` measurement scaffolding from 2026-08-07 was + // REVERTED here (gate finding F-002): it depended on a user-global + // ~/.config/opencode/agent/rg-reviewer.md that no other machine has. Its result is + // recorded in bench/results/qwen-overhead/tool-surface.json (~23.5K -> ~17.8K input + // tokens); make it a config option before reintroducing it. + const args = ["run", "--auto", "--format", "default"]; // Only force a model with -m for a REAL provider/model id. The sentinel // "default" (or empty) means "use opencode's own configured default model" // — which is how opencode is meant to be driven here (e.g. a MiniMax Token From fb6e89cc967bff0f97001d936de6d311bd51fd33 Mon Sep 17 00:00:00 2001 From: Codevena Date: Sat, 8 Aug 2026 01:38:08 +0200 Subject: [PATCH 13/55] =?UTF-8?q?docs:=20handoff=20=E2=80=94=20Qwen-Strang?= =?UTF-8?q?=20abgeschlossen,=20Stand=20der=20Parallel-Session=20korrigiert?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- NEXT_SESSION.md | 101 ++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 93 insertions(+), 8 deletions(-) diff --git a/NEXT_SESSION.md b/NEXT_SESSION.md index 4883b82..7dda53f 100644 --- a/NEXT_SESSION.md +++ b/NEXT_SESSION.md @@ -14,12 +14,12 @@ implements `docs/superpowers/plans/2026-08-07-rig-stale-report-fix.md`, Task 1 f | | | |---|---| | my commits this session | **`384df2a`** (spec), **`f36abf1`** (plan), **`734f5eb`** (plan corrections) — docs only, zero code | -| pushed? | **NO.** `master` is **8 ahead** of `origin/master` — but only 3 are mine | -| the other 5 | `0f8b6cf`, `e7c25e1`, `979bfea`, `ef54ed0`, `9fac6f8` — **a SECOND SESSION's bench/Qwen work**, still live | +| pushed? | **NO.** ~~`master` is 8 ahead~~ → **12 ahead** of `origin/master` as of 2026-08-08 01:4x — 3 are mine, **9 are the Qwen session's** | +| ~~the other 5~~ the other 9 | `9fac6f8`, `ef54ed0`, `979bfea`, `e7c25e1`, `0f8b6cf`, `cef7022`, `56564ea`, `63a779f` (+ `b3032b3`, `9473973` already pushed) — **the SECOND SESSION's bench/Qwen work, now CLOSED OUT.** See "Second work stream" below | | working tree | only `.reviewgate/lore/approvals.jsonl` (foreign gate state) — leave it alone | -| suite | **3209 pass / 12 skip / 0 fail**, exit 0 — run at `734f5eb` (was 3191; the parallel session added 18) | +| suite | **3209 pass / 12 skip / 0 fail**, exit 0 — re-run at `56564ea` by the Qwen session (139 s). ⚠ One earlier full run showed `1 fail` that did **not** reproduce in four subsequent runs; the name was never captured. Only hint: `cassette: prompt drift for codex-security` appeared solely in the failing run. Treat as flaky-unknown, not as green-by-proof | | build | **deliberately NOT run.** Installed binary still `sha256:fc9b8c18…` | -| Trailhead stamp | **left at `5543549` ON PURPOSE** — see Traps | +| ~~Trailhead stamp left at `5543549` ON PURPOSE~~ | → **now `cef7022`, stamped 2026-08-08 (`56564ea`).** This is the completion of that decision, not an override: the 4 GEÄNDERT rows were the Qwen session's own work, and that session verified them (`bench.ts`/`runner.ts` entry points still correct, 0 FEHLT, 80/80 lines). The session that could honestly stamp them did | ⚠ **A SECOND SESSION IS COMMITTING TO THIS CHECKOUT.** Never `git add -A`; stage explicit paths and check `git log` before assuming a commit is yours. A `git worktree` remains the standing fix. @@ -110,10 +110,11 @@ existing tests for the wrong reason. - **The gate escalated on findings that are not mine and cannot be honestly dispositioned.** `F-002`/`F-003` on `src/providers/opencode.ts` are the parallel session's code, but the ownership snapshot marked them `session_attributable: true` (their edits landed inside my baseline window), - so `out-of-scope` and `out-of-session` both fail closed. They remain **open and escalated** — - see `.reviewgate/ESCALATION.md`. That escalation also rests on a **quota-degraded panel** (codex - capped until 2026-08-08 11:07Z); the file itself says to re-run after the reset before treating - the findings as final. + so `out-of-scope` and `out-of-session` both fail closed. ~~They remain **open and escalated**~~ → + **RESOLVED 2026-08-08 by the session that owns that code:** `F-002` **fixed** (the measurement + scaffolding was removed from the reviewer path, `63a779f`), `F-003` **rejected with a reason** and + carried as a named next task. Both decisions are in `.reviewgate/decisions/1.jsonl`. The + quota-degraded-panel caveat still applies to any *re-run* (codex capped until 2026-08-08 11:07Z). - **`harvest.ts` never reads `manifest.turns[].gateReviewed`** — the flag exists, is written by the driver, and is consulted by nothing. The plan subsumes it rather than adding a second signal. - **An `iterations === 0` warning that says "EXCLUDED from the M1/cost-per-turn samples" is true and @@ -152,3 +153,87 @@ exactly 80/80 lines, so any change is a swap, not an addition. 3. `docs/superpowers/specs/2026-08-07-rig-stale-report-design.md` — why the rule is what it is, especially §"Why `run_id` alone" and §"Failure handling". 4. `.reviewgate/ESCALATION.md` — the open, not-mine findings, before ending your first turn. + +--- + +# Second work stream — Qwen3.8-Max as a measured reviewer (session of 2026-08-07/08) + +_Independent of the rig stale-report task above. Both are live in this checkout._ + +## One-line state + +**The cost question is answered and the tooling is built and committed; the *quality* question is +untouched. Phase 2 (the 30-case exploratory bench) is the next step — but run it on a Standard tier, +not on Lite.** + +## What got done — and how it was verified + +| | | +|---|---| +| `scripts/measure-opencode-tokens.ts` (`9473973`, **pushed**) | Token oracle over opencode's SQLite session DB. 6/6 green; mutation seen red (coefficient 1.21 → 2.42 ⇒ 4 pass / 2 fail, measured value `118.68406`) | +| `bench --provider-model` (`ef54ed0`) | Pins a reviewer's upstream model into provenance. 12/12 green; mutation reddened **exactly the 4 predicted cases**, the sentinel test stayed green. Verified end-to-end: a real `bench run` wrote `"model": "alibaba-token-plan/qwen3.8-max"`, not `"default"` | +| Overhead + caching measurements (`9fac6f8`, `979bfea`, `e7c25e1`, `0f8b6cf`) | Artifacts under `bench/results/qwen-overhead/`. Every credit number is **console-read**, not modelled | +| Risk-control scope correction (`cef7022`) | Markus: the block suspends **purchases only**; renewal and tier changes are unaffected | +| Trailhead stamp (`56564ea`) | `verify-map.js`: 0 FEHLT, 4 GEÄNDERT (all this stream's), entry points re-checked | +| F-002 fix (`63a779f`) | Measurement scaffolding removed from the live reviewer path | + +**The numbers, all console-verified:** + +| | credits/call | | | +|---|---|---|---| +| baseline, default agent | 31.01 | 30 × 1 | 30 × 3 | +| + reduced tool set (`--agent`) | 22.86 | | | +| + warm cache | 9.17 | | | +| **real case** (2 calls/case, 1st cold) | **28.2 /case** | **846 cr** | **2,538 cr** | +| …as % of a **Lite** window (2,500) | | 34 % | **102 % — does not fit** | +| …as % of a **Standard** window (10,000) | | 8.5 % | **25.4 % — fits** | + +Smoke run: `2/2 cases scored → precision 1, recall 1, clean-FP 0`. **N=2 — that is a pipeline test, +not evidence about review quality. The acceptance bar in the spec is untouched.** + +## THE NEXT TASK — and why + +**Decide the tier before spending anything.** Phase 2 costs 34 % of a Lite window but 8.5 % of a +Standard one, and Phase 3 is impossible on Lite and routine on Standard. Running Phase 2 on Lite is +the expensive ordering: it burns a third of the week to answer a question whose follow-up you then +cannot afford. $12/month decides this, and the risk-control block does **not** stand in the way. + +Once the tier is settled, Phase 2 is `reviewgate bench run --corpus bench/cases --providers +opencode,ollama,claude-code --provider-model opencode=alibaba-token-plan/qwen3.8-max`. The bar is +preregistered in the spec §6: Qwen earns a slot if it finds **≥1 seeded bug that GLM-5.2 and +claude-code both miss**, at a clean-FP rate no worse than GLM-5.2's. + +**Second task, small and independent:** finding **F-003** (rejected, carried forward) — +`src/providers/opencode.ts` `complete()` still passes `--dangerously-skip-permissions`, which does +not exist in opencode 1.18.10. Own commit, own gate: it changes curator runtime behaviour. + +## Traps — NEW from this stream + +- **`--dangerously-skip-permissions` does not exist in opencode 1.18.10.** The flag is `--auto`, and + opencode **exits 0 on unknown flags** instead of rejecting them, so the dead flag was silently + ignored on every call for an unknown span of time. Fixed at `:97`, still live at `:242`. +- **Credits are read, never computed.** Fitting the uncached/cached coefficients across three + calibration points does **not converge** (1.25–1.71 and 0.05–0.43 per 1K). The console shows two + decimals (±0.125 credits). The token model ran **9 % low** on its one real test. Any credit figure + in a future doc must cite a console delta. +- **A bench case costs ~2 LLM calls, not 1**, and the first call of a run pays a cold cache. Any + per-*call* figure understates the per-*case* cost by ~3×. +- **`bench.ts` rejects a corpus with zero clean cases** (exit 4). A "just run one case" smoke test is + invalid; take one clean + one seeded. +- **The reduced-tool `--agent rg-reviewer` win (23.5K → 17.8K input tokens) is real but not shipped.** + It depended on `~/.config/opencode/agent/rg-reviewer.md`, which exists only on Markus's machine. + Make it a config option before reintroducing it — do not hard-code it in the adapter again. +- **`phases.brain.curator` points at `opencode` with `model: "minimax-m2"`, whose plan has expired.** + The call **hangs** instead of erroring — killed after 150 s. Independent of everything above. +- **Pay-per-token DashScope is blocked**, account-wide (`AccessDenied.Unpurchased` on all 5 models + tested), root cause `RISK.RISK_CONTROL_REJECTION`. KYC would unblock it but costs a passport scan + and a month of bank transactions — and buys only pay-as-you-go and Extra Bundles, **neither of + which this work needs**. Do not treat it as a prerequisite. + +## Read-first order for this stream + +1. `bench/results/qwen-overhead/DECISION.md` — the go/no-go and every caveat. +2. `docs/superpowers/specs/2026-08-07-qwen-reviewer-measurement-design.md` §5a (the live options) and + §6 (the preregistered acceptance bar). +3. `docs/superpowers/plans/2026-08-07-qwen-overhead-and-provider-model.md` — the three findings + mappings at the end are the record of what three gate rounds actually caught. From a1197dcd8b92f60501801230139e1fbaefc59170 Mon Sep 17 00:00:00 2001 From: Codevena Date: Sun, 9 Aug 2026 23:49:34 +0200 Subject: [PATCH 14/55] docs: design policy accountability trace and replay --- ...8-09-policy-accountability-trace-design.md | 546 ++++++++++++++++++ 1 file changed, 546 insertions(+) create mode 100644 docs/superpowers/specs/2026-08-09-policy-accountability-trace-design.md diff --git a/docs/superpowers/specs/2026-08-09-policy-accountability-trace-design.md b/docs/superpowers/specs/2026-08-09-policy-accountability-trace-design.md new file mode 100644 index 0000000..0b138ba --- /dev/null +++ b/docs/superpowers/specs/2026-08-09-policy-accountability-trace-design.md @@ -0,0 +1,546 @@ +# Policy Accountability & Pruning — Slice 1: Policy Trace & Replay + +_Written 2026-08-09. Status: design approved in dialogue; written spec awaiting Markus review._ + +## Context + +Reviewgate's review policy grew from real incidents rather than speculative feature work. That +history justifies the existence of tests, the cassette, the benchmark and the rig, but it also left +outcome-changing policy distributed across pre-aggregation helpers, `src/core/orchestrator.ts` and +`src/core/aggregator.ts`. + +The current checkout has 204 TypeScript source files and 44,880 source lines. Two files — +`src/core/orchestrator.ts` (3,259 lines) and `src/core/loop-driver.ts` (3,003 lines) — hold 13.95% of +the source. The larger problem is not their line count by itself: a final finding can be affected by +fact validation, grounding, critic judgement, three scopes, several history layers, confidence, +reputation and category-specific severity calibration. The precedence and protection rules are +encoded in execution order plus comments such as “G0 mirror”, “hard suppressor” and “runs after +reputation”. + +Existing observability is partial: + +- `src/rig/ablate.ts` names only critic, reputation, FP-ledger and lore. Lore is not a demoter, and + the rig explicitly treats several unmodelled structural demoters as unknowable interactions. +- `bench matrix` exposes critic, confidence, reputation and scope-to-diff. Fresh per-case state + deliberately makes the learning layers inert. +- `Finding` has individual marker fields, so a material effect is often visible, but it does not + provide a complete ordered causal path, opportunity denominator or record of a demotion that a + protection rule prevented. +- Post-aggregation artifacts cannot always reconstruct the counterfactual. FP-ledger, for example, + overwrites severity with INFO without persisting the pre-suppression value. + +This is policy sedimentation, not generic “too many tests” overengineering. The remedy is to make +every outcome-changing pass accountable, measure it with the correct denominator, prune it when the +evidence warrants that, and only then extract the surviving policy from the large modules. + +## Program boundary + +This spec is Slice 1 of a three-slice program: + +1. **Policy Trace & Replay** — behavior-neutral instrumentation and exact internal ablation. +2. **Policy Measurement & Pruning** — opportunity-conditioned measurement, followed by deletion or + consolidation of harmful or redundant policy. Behavior may change in Slice 2. +3. **Survivor Consolidation** — extract only the retained passes and remove obsolete config, + schemas, markers, tests and documentation. + +Two already-open reliability fixes precede this program and are not re-planned here: + +- the rig stale-report ownership fix in + `docs/superpowers/specs/2026-08-07-rig-stale-report-design.md`; +- project-scoped behavior for the currently user-scoped Stop hook, so unrelated directories cannot + be blocked by a missing project policy baseline. + +Slice 1 may refactor the mechanical path by which a policy transition is recorded and ablated, but +it must not change any production finding, marker, count, order or verdict. + +## Goals + +1. Give every outcome-changing pass a stable identity, class, execution position, opportunity + definition and protection relationship. +2. Explain every material severity change, drop, re-anchor or prevented demotion with a compact, + machine-readable ordered trace. +3. Distinguish `not-run`, `no-opportunity`, `would-apply`, `protected` and `applied` rather than + collapsing all non-activations to zero. +4. Run paired ablations through the same production policy path over byte-identical raw reviewer + responses. +5. Make missing, corrupt or incomplete traces invalidate authoritative Bench/Rig measurements, + without ever changing a production gate verdict. +6. Produce the inputs Slice 2 needs to delete policy safely. + +## Non-goals + +- No policy deletion, threshold change or new demotion in Slice 1. +- No new reviewer, provider, learning store, dashboard or public policy feature. +- No public config switches for fact-check, redaction or any other currently always-on pass. +- No general plugin/rule engine and no data-driven dynamic policy loading. +- No semantic unification of FP-ledger, reputation, region memory, Agent Lessons, Brain and Lore. + Their storage mechanics may be candidates for later consolidation, but they have different + subjects, trust authorities, lifetimes and consumers. +- No complete extraction of the policy pipeline from Orchestrator/Aggregator before pruning. +- No claim that the current 30-case benchmark alone can validate rare or stateful policy. + +## Terminology + +- **considered** — the finding reached the pass while the pass was configured to run. +- **opportunity** — required inputs and starting state were present, and the pass could have changed + this finding if its positive predicate matched. +- **would apply** — the positive predicate matched before protection and before an internal + ablation suppressed the mutation. +- **protected** — the positive predicate matched, but a named guard intentionally prevented the + mutation. +- **applied** — the pass changed severity, dropped the finding, suppressed it, capped it or + re-anchored it. +- **not run** — the production configuration or stage precondition kept the whole pass inactive. +- **policy effect** — one material `applied` or `protected` event attached to a finding. +- **policy trace** — the full per-run artifact containing per-finding evaluations and aggregate + counters, including findings later dropped by policy. + +An opportunity is pass-specific and must be stated in the catalog. It is never inferred from an +activation after the fact. A zero activation count without the opportunity count is not evidence. + +## Policy catalog + +`src/core/policy/catalog.ts` becomes the single catalog of stable IDs and metadata. It does not load +or execute dynamic code. Each entry has: + +- `id`; +- `order` (strictly increasing in production execution order); +- `class` (`evidence`, `value-judgment`, `scope`, `history`); +- possible `actions`; +- closed reason codes; +- opportunity definition; +- `depends_on` and `overlaps_with` relationships; +- whether the pass is internally ablatable; +- the Slice-2 metric appropriate to that pass. + +### Initial outcome-changing inventory + +| Order | Pass ID | Class | Current implementation | Possible effect | +|---:|---|---|---|---| +| 10 | `evidence.fact-location` | evidence | `src/core/fact-check.ts` | INFO demote or re-anchor | +| 20 | `evidence.self-refutation` | evidence | `src/core/self-refutation.ts` | INFO demote | +| 30 | `judgment.hypothetical` | value-judgment | `src/core/hypothetical-demote.ts` | CRITICAL → WARN | +| 40 | `evidence.grounding-token` | evidence | `src/core/grounding.ts` | CRITICAL → WARN | +| 50 | `judgment.grounding-llm` | value-judgment | `src/core/grounding.ts` | CRITICAL → WARN | +| 60 | `evidence.redaction-placeholder` | evidence | `src/core/aggregator.ts` | INFO demote | +| 70 | `judgment.critic` | value-judgment | `src/core/aggregator.ts` | one-step demote or INFO drop | +| 80 | `scope.diff` | scope | `src/core/aggregator.ts` | INFO demote | +| 90 | `scope.delta` | scope | `src/core/aggregator.ts` | INFO demote | +| 100 | `scope.session` | scope | `src/core/aggregator.ts` | INFO demote | +| 110 | `history.fp-signature` | history | `src/core/aggregator.ts` | INFO suppression | +| 120 | `history.cycle-rejected` | history | `src/core/aggregator.ts` | INFO suppression | +| 130 | `history.fp-cluster` | history | `src/core/aggregator.ts` | INFO suppression | +| 140 | `judgment.confidence` | value-judgment | `src/core/aggregator.ts` | INFO demote or CRITICAL → WARN clamp | +| 150 | `judgment.reputation` | value-judgment | `src/core/aggregator.ts` | one-step/INFO demote or CRITICAL → WARN clamp | +| 160 | `history.region-rejected` | history | `src/core/aggregator.ts` | INFO suppression | +| 170 | `judgment.test-security` | value-judgment | `src/core/aggregator.ts` | INFO demote | +| 180 | `judgment.docs-cap` | value-judgment | `src/core/aggregator.ts` | CRITICAL → WARN | + +The catalog also records two non-ablatable explanatory stages: + +- `aggregation.cluster` — raw-finding lineage, representative selection, maximum severity and + consensus; +- `verdict.compute` — final counts and the rule that produced PASS/SOFT-PASS/FAIL. + +They are core aggregation semantics rather than experimental suppressors. + +### Protection rules + +Protection rules do not become standalone demoters. A matched pass records `action: protected` plus +a closed `protected_by` reason. Initial protection reasons include: + +- `claimed-fixed-pin`; +- `security-correctness-floor`; +- `corroborated-majority`; +- `corroborated-unanimous`; +- `high-precision-reviewer`; +- `out-of-diff-blocking-hatch`; +- `critical-floor`; +- `single-reviewer-critical-floor`. + +Example: if the critic returns `likely_fp` but majority consensus protects the finding, the trace +says that `judgment.critic` would have applied and was protected by `corroborated-majority`. It does +not invent a separate “corroboration pass”. + +Lore is excluded from this inventory because it does not demote. Lore's added INFO/decision load +remains a separate metric. + +## Trace data model + +A new `src/schemas/policy-trace.ts` owns the persisted trace contract. + +### Full evaluation versus material effect + +The full trace stores one terminal evaluation result for every finding that reaches a configured +pass: + +```ts +interface PolicyEvaluation { + pass_id: PolicyPassId; + order: number; + result: "no-opportunity" | "no-match" | "would-apply" | "protected" | "applied"; + before: Severity; + after: Severity | null; // null only when an applied pass drops the finding + reason_code: string; // validated against the pass catalog + protected_by?: string; + source_signatures: string[]; +} +``` + +This makes the opportunity denominator auditable per case/finding rather than leaving only an +aggregate count. A configured pass that did not run has no per-finding evaluations and carries +`status: not-run` in its summary instead. + +Visible findings do not carry this full matrix. They receive only the material subset: + +Conceptual shape: + +```ts +interface PolicyEffect { + pass_id: PolicyPassId; + order: number; + action: "demoted" | "capped" | "dropped" | "protected" | "suppressed" | "reanchored"; + before: Severity; + after: Severity | null; // null only for drop + reason_code: string; // validated against the pass catalog + protected_by?: string; + source_signatures: string[]; +} +``` + +Constraints: + +- no free-form model text; +- no message, details, diff hunk, source line or source file content; +- `source_signatures` are sorted and deduplicated; +- `order` must match the catalog entry; +- `after` must match the action contract; +- duplicate idempotent effects collapse deterministically; +- effects remain in ascending catalog order. + +`FindingSchema` gains optional `policy_effects`. Existing marker fields remain in Slice 1 because +they are consumed throughout the existing renderer, loop and rig. Slice 2/3 decides which marker +fields can be deleted after policy pruning. + +### Run-level counters + +Each configured pass emits one compact row: + +```ts +interface PolicyPassSummary { + pass_id: PolicyPassId; + status: "ran" | "not-run" | "error"; + considered: number; + opportunities: number; + would_apply: number; + applied: number; + protected: number; + blocking_removed: number; + blocking_preserved: number; + dropped: number; +} +``` + +All numeric fields are required when `status: ran`. Missing data is not defaulted to zero. The +summary schema rejects impossible relationships such as `applied > would_apply` or +`protected > would_apply`. + +### Full policy trace artifact + +Conceptual shape: + +```ts +interface PolicyTrace { + schema: "reviewgate.policy-trace.v1"; + catalog_version: "reviewgate.policy-catalog.v1"; + run_id: string; + iter: number; + ablated: PolicyPassId[]; + raw_response_sha256: string[]; + passes: PolicyPassSummary[]; + evaluations: Array; + final: { + verdict: "PASS" | "SOFT-PASS" | "FAIL" | "ERROR"; + counts: { critical: number; warn: number; info: number }; + finding_signatures: string[]; + }; +} +``` + +The full artifact includes evaluations for findings later dropped by a pass and `would-apply` +observations from an ablated pass. A visible final finding contains only its own applied/protected +material effects. + +## Recorder and transition boundary + +`src/core/policy/trace.ts` provides an in-memory recorder and a shared transition helper. The +recorder is deterministic and performs no filesystem I/O while policy is running. + +Every one of the 18 passes routes material mutation through this helper. The helper receives: + +- pass ID; +- original finding or cluster lineage; +- opportunity/match facts; +- optional protection reason; +- proposed mutation; +- the internal ablation set. + +Behavior: + +1. Count `considered` and `opportunities` from explicit booleans supplied by the pass. +2. When the positive predicate does not match, return the original finding. +3. When a guard protects the finding, record `would_apply + protected`, return the original. +4. When the pass is internally ablated, record `would_apply` with no applied mutation and return the + original. +5. Otherwise apply the existing mutation exactly, record the material effect, and return the + result (or `null` for a drop). + +Pure severity-calculation helpers such as `demoteOneStep` may remain, but a production pass must not +assign a demoted/capped/suppressed severity or drop a finding outside the transition boundary. + +### Clustering and lineage + +Pre-cluster effects travel with their raw finding. When aggregation clusters findings: + +- the representative retains its own effects; +- member effects are copied to the final cluster with their source signatures; +- identical idempotent effects are deduplicated; +- `aggregation.cluster` records every contributing signature and the representative; +- `demoted_from_critical` and `anchor_repaired` keep their existing OR-propagation behavior. + +The trace is explanatory only. Final severity and verdict remain canonical in the existing Finding +and PendingReport data. + +## Internal ablation contract + +Orchestrator/Aggregator receive an internal `policyAblations: ReadonlySet`. The normal +gate never supplies it. It is not part of `reviewgate.config.ts`, the config schema, environment +variables or a public gate CLI flag. + +Only Bench/Rig replay code may supply the set. Tests enforce that normal gate/setup/config commands +cannot construct it. + +For an ablated pass: + +- configuration and stage preconditions remain unchanged; +- the pass evaluates opportunities and its positive predicate; +- `would_apply` remains observable; +- the mutation and material marker do not apply; +- every later pass runs normally on the counterfactual finding; +- reviewer inputs, provider calls, state reads and raw responses remain unchanged. + +This is a production-path ablation, not a second hand-built model of pass precedence. + +### Pairing and interactions + +An ablation result is attributable only when baseline and counterfactual carry identical ordered raw +response hashes. Any mismatch invalidates the pair. + +Slice 1 enables one-pass ablation and records `depends_on`/`overlaps_with`; it does not claim +leave-one-out captures all interactions. Slice 2 must additionally evaluate co-activating groups, +especially: + +- critic × confidence × reputation; +- diff scope × delta scope × session scope; +- cycle rejection × region rejection × FP signature/cluster; +- fact location × token grounding × LLM grounding × redaction × self-refutation. + +## Persistence and compatibility + +### Pending report + +`PendingReportSchema` gains an optional compact `policy_summary`. Visible findings gain optional +`policy_effects`. The existing `reviewgate.pending.v1` literal remains valid because the additions +are optional and old reports remain parseable. + +### Audit artifact + +One trace file is written atomically per reviewed iteration inside the existing audit day +partition: + +```text +.reviewgate/audit/YYYY/MM/DD/policy/-i-.json +``` + +`run-sha12` is derived from `sha256(run_id)`; untrusted run IDs never become path components. The +canonical JSON bytes determine the content SHA-256 and filename. Canonicalization uses the same +implementation as the audit hash chain, extracted as a shared helper rather than reimplemented. +`run.complete` gains optional +`policy_trace_ref` and `policy_trace_sha256`; `RunSummary` gains an optional `policy_trace_status` +with the closed values `complete | not-run | error | overflow`. Reference and hash are required +only for `complete` and forbidden for every other status. +The audit hash chain therefore binds the reference and content hash without embedding the full +trace in every JSONL event. + +The existing day-partition retention deletes policy artifacts with the corresponding audit day. +The verifier validates referenced artifacts when the optional fields are present; legacy chains +without them remain valid. + +### Rendering + +For newly traced findings, only already-existing badge variants are derived from `policy_effects`; +Slice 1 adds no new badge text. Legacy marker fields remain the fallback for old artifacts. The +rendered `pending.md` must remain byte-identical for the same findings. Slice 1 adds no second +block of verbose policy prose to the report. + +The machine-readable path is available in `pending.json`; the compact badges remain the normal +agent/human explanation. A new top-level CLI subcommand is explicitly out of scope. + +### Cache and structured reviewer output + +- Trace fields do not participate in production review-cache keys. +- The strict `REVIEW_OUTPUT_SCHEMA` is unchanged. Policy effects are server-authored and are never + accepted from a reviewer model. +- Bench already disables review caching. Rig replay must bind ablations and the catalog version in + its own replay identity so two policy profiles cannot share a derived result accidentally. + +## Failure behavior + +### Production gate + +Trace telemetry must never decide whether code passes: + +- recorder errors preserve the existing finding and continue the gate; +- artifact write errors leave ref/hash absent and set trace status `error`; +- an artifact over the size limit sets `overflow` and is not silently truncated; +- pending/report writing continues with the canonical existing finding data; +- no trace failure changes a verdict, dirty flag, iteration or decision requirement. + +The initial artifact limit is 1 MiB per iteration. The plan must verify the limit against a +worst-case synthetic run before fixing it permanently; changing the value changes storage only, +not policy. + +### Authoritative Bench/Rig + +An authoritative measurement is invalid when: + +- a configured pass row is missing; +- required counters are absent or inconsistent; +- trace status is not `complete`; +- referenced content is missing, over limit or fails SHA-256 validation; +- baseline/counterfactual raw response hashes differ; +- the catalog version differs between paired runs. + +The command exits 4 and states the exact cause. It must never interpret missing counters as zero. + +## Work packages + +1. **Catalog and schemas** — pass metadata, effect/summary/artifact schemas and semantic validation. +2. **Recorder and transition helper** — in-memory totals, material effects, protection and ablation. +3. **Pre-aggregation instrumentation** — fact location, self-refutation, hypothetical and both + grounding layers. +4. **Aggregator instrumentation** — redaction, critic, three scopes, FP/cycle/cluster, confidence, + reputation, region, test security and docs cap. +5. **Lineage and rendering** — cluster effect propagation plus byte-identical badge fallback. +6. **Persistence and audit binding** — atomic content-addressed artifact, run-complete ref/hash and + verifier support. +7. **Internal ablation plumbing** — inaccessible from normal gate/config paths. +8. **Bench/Rig validation** — trace completeness, hash pairing, invalid-result handling and summary + output. +9. **Documentation and Slice-2 handoff** — exact pass inventory, catalog semantics and measurement + limitations. + +## Test design + +### Required contract cases for each pass + +Every one of the 18 passes has explicit numbers for: + +1. no opportunity: `0 opportunities / 0 would_apply / 0 applied`; +2. one opportunity without predicate match: `1 / 0 / 0`; +3. activation: `1 / 1 / 1`; +4. internal ablation: `1 / 1 / 0` with unchanged finding behavior; +5. protection, when the pass has guards: `1 / 1 / 0 applied / 1 protected`; +6. blocking findings with and without the mechanism. + +Example for `judgment.confidence`: + +- uncorroborated low-confidence WARN, active: raw blocking `1` → final blocking `0`; +- same raw finding, ablated: raw blocking `1` → final blocking `1`; +- same raw finding from a proven high-precision reviewer: `would_apply 1`, `protected 1`, final + blocking `1`. + +The implementation plan must enumerate the corresponding numbers for all 18 passes before code is +written; a test whose active and ablated blocking result are both equal cannot guard that pass's +effect unless it is explicitly a protection/no-opportunity test. + +### Behavior-neutral equivalence + +Given identical raw reviews and state: + +- trace recorder enabled versus disabled produces identical findings, order, legacy markers, + counts and verdict after optional trace fields are removed; +- provider call counts and ordered raw response hashes are identical; +- report Markdown is byte-identical; +- no trace error changes the gate decision. + +### Completeness and consistency + +Tests prove: + +- all 18 catalog entries emit exactly one run-summary row when configured; +- effect order is monotonic by catalog order; +- every material marker produced by a catalogued pass maps to a material effect; +- every material effect is valid for its catalogued pass/action/reason; +- cluster members do not lose effects; +- dropped findings remain in the full trace but not the final pending findings; +- missing data is invalid rather than coerced to zero. + +### Failure and security cases + +- atomic write failure → production verdict unchanged, status `error`; +- 1 MiB overflow → no partial artifact; production status `overflow`, authoritative result invalid; +- artifact byte tamper → verifier failure and Bench/Rig exit 4; +- path traversal in run ID cannot escape the fixed audit directory; +- reviewer-controlled messages/rule IDs cannot enter reason codes or artifact paths; +- legacy pending/audit artifacts parse and render as before; +- normal gate/config/setup cannot supply internal ablations. + +### Mutation requirements + +Each new contract is seen red in a copy. At minimum, mutations must prove tests catch: + +- an unregistered pass; +- a pass that mutates without recording an effect; +- a missing opportunity increment; +- swapped effect order; +- an ablated pass that still mutates severity; +- a lost member effect during clustering; +- a missing raw response hash comparison; +- a tampered artifact hash; +- missing trace interpreted as zero activations; +- trace persistence failure leaking into verdict behavior. + +## Verification and acceptance + +Slice 1 is complete only when all of the following hold: + +1. All 18 outcome-changing passes are catalogued and instrumented. +2. Both non-ablatable explanatory stages are present. +3. Trace-on versus trace-off is behavior-identical after optional telemetry fields are stripped. +4. Existing `pending.md` output is byte-identical for unchanged fixtures. +5. No public config/CLI path exposes internal ablation. +6. An offline replay produces a complete trace for each of the four pass classes without live + provider calls. +7. Baseline/ablation pairs prove identical ordered raw response hashes. +8. Bench/Rig reject missing, corrupt, overflowed and cross-catalog traces as non-authoritative. +9. Every new guard test has been mutation-proven red in a copy. +10. `bunx tsc --noEmit`, `bun run lint` and the full `bun test` suite pass. +11. The compiled binary paths affected by persistence/CLI validation pass their focused smoke tests. +12. The repository's independent post-implementation review pipeline passes. + +## Slice-2 handoff + +Slice 1 does not rank or delete passes. It hands Slice 2: + +- the versioned policy catalog; +- complete opportunity/activation/protection counts; +- paired raw response hashes and ablation outputs; +- a list of co-activating pass groups; +- explicit measurement limits for stateful and rare passes; +- evidence that production behavior did not move while the measurement layer was installed. + +Slice 2 then evaluates stateless passes with paired 30-case × 3-repeat response replays, stateful +passes with seeded multi-turn state/Rig/Cassettes, and all passes against real dogfood dispositions. +Deletion requires sufficient opportunities plus either measured harm or no unique contribution +beyond a retained pass. Zero opportunities alone never justifies deletion. From 6d5080e0e67cf1cdba9550ded404cd3f0cda1acb Mon Sep 17 00:00:00 2001 From: Codevena Date: Mon, 10 Aug 2026 00:13:27 +0200 Subject: [PATCH 15/55] docs: plan policy accountability trace implementation --- .../2026-08-09-policy-accountability-trace.md | 914 ++++++++++++++++++ ...8-09-policy-accountability-trace-design.md | 88 +- 2 files changed, 991 insertions(+), 11 deletions(-) create mode 100644 docs/superpowers/plans/2026-08-09-policy-accountability-trace.md diff --git a/docs/superpowers/plans/2026-08-09-policy-accountability-trace.md b/docs/superpowers/plans/2026-08-09-policy-accountability-trace.md new file mode 100644 index 0000000..7c44aa1 --- /dev/null +++ b/docs/superpowers/plans/2026-08-09-policy-accountability-trace.md @@ -0,0 +1,914 @@ +# Policy Accountability Trace & Replay Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Make all 18 outcome-changing review-policy passes observable and exactly ablatable without changing production findings, ordering, legacy markers, Markdown, counts, or verdicts. + +**Architecture:** A static policy catalog and Zod-owned trace schema sit beside a fail-open in-memory recorder. Existing pass predicates remain where they are, but every material transition crosses one shared helper; Orchestrator owns the recorder lifecycle, AuditLogger persists the finished content-addressed trace, and Bench/Rig are the only callers allowed to supply internal ablations. Bench replays captured provider responses through the same production policy path; Rig validates captured traces and replays only from branch-isolated snapshots and cassette data. + +**Tech Stack:** Bun, TypeScript with `exactOptionalPropertyTypes`, Zod, Biome, `bun:test`, JSONL audit chains, JSON state beneath `.reviewgate/`. + +## Global Constraints + +- Execute this plan only after the rig stale-report ownership fix and the project-scoped Stop-hook fix are merged and verified. +- Slice 1 changes telemetry and internal replay only; it must not delete policy or change thresholds, findings, markers, order, counts, verdicts, cache behavior, or `pending.md` bytes. +- Keep `REVIEW_OUTPUT_SCHEMA` unchanged; every policy field is server-authored. +- Do not add a gate/config/setup flag, environment variable, or top-level CLI subcommand for policy ablation. +- Only Bench/Rig construction paths may pass `policyAblations`; normal Gate, Setup, Config, and one-shot plan-review paths cannot source it from user configuration. +- Production telemetry is fail-open: trace errors never change the canonical finding or verdict. +- Authoritative Bench/Rig measurement is fail-closed: missing, invalid, corrupt, overflowed, cross-catalog, or response-mismatched traces exit `4`. +- Persist a complete trace only when canonical UTF-8 JSON is at most `1_048_576` bytes; never truncate, sample, or emit a summary-only trace artifact. +- Preserve legacy persisted artifacts through optional additive fields and the existing `reviewgate.pending.v1`, `reviewgate.audit.v1`, `reviewgate.bench.result.v1`, and `reviewgate.rig.result.v1` literals. +- Use Bun built-ins where they fit, `writeFileAtomic` for standalone artifacts, and the existing audit day partition for production policy traces. +- Every task follows red-green-refactor, mutation-proves its new guard red in a disposable copy, runs focused tests, and commits only its owned files. +- Before declaring Slice 1 complete, run `bunx tsc --noEmit`, `bun run lint`, full `bun test`, affected compiled-binary smoke tests, and the repository's independent review pipeline. +- Never stage the foreign `.reviewgate/lore/approvals.jsonl` change and never push without Markus' explicit permission. + +--- + +## Preconditions and File Boundaries + +### Preconditions to verify before Task 1 + +```bash +git log -1 --oneline -- docs/superpowers/specs/2026-08-07-rig-stale-report-design.md +node /Users/markus/.claude/scripts/verify-map.js +test ! -e .reviewgate/gate.lock +git status --short --branch +``` + +Expected: the stale-report implementation commit is present, the Trailhead reports `MAP OK`, no live gate lock exists, and only known user/foreign state is dirty. + +### File structure locked by this plan + +| File | Responsibility | +|---|---| +| `src/core/policy/catalog.ts` | Static IDs, order, class, actions, opportunity text, reason/protection codes, dependencies, overlaps, Slice-2 metric | +| `src/schemas/policy-trace.ts` | Zod contracts and semantic validation for effects, evaluations, summaries, stages, full trace, compact pending summary | +| `src/core/policy/trace.ts` | Fail-open recorder, transition helper, cluster lineage, stage recording, finalization | +| `src/core/policy/response-hashes.ts` | Ordered SHA-256 capture for raw reviewer/judge/critic response text | +| `src/core/policy/replay.ts` | Internal execution options, trace validation, baseline/counterfactual pairing, branch-state checks | +| `src/schemas/policy-replay.ts` | Strict Rig-only per-iteration replay envelope; no config or ablation controls | +| `src/core/policy/replay-capture.ts` | Contained external capture, lossless/redacted status, envelope hashing | +| `src/audit/canonical.ts` | One canonical JSON implementation shared by audit hashing and policy artifacts | +| `src/audit/policy-trace-store.ts` | Size check, safe content-addressed path, atomic write, reference/hash verification | +| Existing five pre-aggregation helpers | Keep current predicates and mutations; route their outcomes through the transition helper | +| `src/core/aggregator.ts` | Keep current pass order; instrument orders 60–180 plus cluster/verdict stages | +| `src/core/orchestrator.ts` | Create/finalize runtime, collect raw hashes, persist/report trace, expose in-memory trace to Bench/Rig | +| Bench modules | Run baseline and ablations under identical config/provider responses; validate traces authoritatively | +| Rig modules | Bind catalog/profile identity, validate captured traces, clone state into isolated replay branches | + +Do not create 18 tiny pass files in Slice 1. Extraction waits until Slice 2 has measured and deleted policy. + +## Closed Catalog Contract + +Common evaluation reasons are `ineligible-starting-state`, `predicate-miss`, `configured-off`, and `stage-precondition-miss`. Pass-specific applied reasons and allowed protections are fixed below; unlisted strings fail schema validation. + +| Pass | Opportunity | Applied reason | Allowed protection codes | +|---|---|---|---| +| `evidence.fact-location` | cited repo file is safely readable and the finding has a positive line | `location-out-of-range`, `evidence-line-reanchored` | none | +| `evidence.self-refutation` | blocking, non-deterministic finding | `terminal-self-refutation` | `security-correctness-floor`, `deterministic-ground-truth` | +| `judgment.hypothetical` | CRITICAL, non-deterministic finding | `hypothetical-critical` | `security-correctness-floor`, `deterministic-ground-truth` | +| `evidence.grounding-token` | CRITICAL with at least one extractable token | `cited-token-absent` | `security-correctness-floor` | +| `judgment.grounding-llm` | CRITICAL with a judge verdict for its signature | `judge-ungrounded` | `security-correctness-floor` | +| `evidence.redaction-placeholder` | blocking finding whose subject contains a redaction placeholder | `placeholder-code-hallucination` | `security-correctness-floor`, `secret-evidence-backstop` | +| `judgment.critic` | critic emitted a verdict for representative/member signature | `critic-likely-fp` | `claimed-fixed-pin`, `self-refutation-visibility`, `security-correctness-floor`, `corroborated-majority`, `corroborated-unanimous`, `high-precision-reviewer` | +| `scope.diff` | blocking finding has a usable line while changed ranges exist | `outside-changed-file`, `outside-changed-lines`, `preexisting-harness-config` | `out-of-diff-blocking-hatch` | +| `scope.delta` | blocking finding while a delta scope exists | `outside-delta-scope` | `claimed-fixed-pin`, `security-correctness-floor`, `critical-floor`, `out-of-diff-blocking-hatch` | +| `scope.session` | blocking finding while foreign-file facts exist | `foreign-to-session` | `out-of-diff-blocking-hatch` | +| `history.fp-signature` | blocking finding while an active signature snapshot exists | `active-fp-signature` | none | +| `history.cycle-rejected` | blocking finding while rejected signatures exist | `cycle-signature-rejected` | `critical-floor`, `security-correctness-floor` | +| `history.fp-cluster` | blocking finding while active cluster keys exist | `active-fp-cluster` | none | +| `judgment.confidence` | blocking, uncorroborated finding while floor is positive | `below-confidence-floor` | `claimed-fixed-pin`, `security-correctness-floor`, `corroborated-majority`, `corroborated-unanimous`, `high-precision-reviewer` | +| `judgment.reputation` | blocking, uncorroborated finding while unreliable reviewers exist | `unreliable-reviewer` | `claimed-fixed-pin`, `security-floor`, `correctness-demote-disabled`, `corroborated-majority`, `corroborated-unanimous`, `critical-floor` | +| `history.region-rejected` | blocking finding with a usable line while rejected regions exist | `rejected-region-overlap` | `claimed-fixed-pin`, `insufficient-distinct-rejections`, `category-change`, `severity-increase`, `critical-floor`, `security-correctness-floor` | +| `judgment.test-security` | blocking finding in a classified test/fixture file | `test-only-security` | `mixed-category-cluster` | +| `judgment.docs-cap` | CRITICAL finding in a classified docs file | `docs-critical-cap` | `security-correctness-floor` | + +## Required 18-Pass Numeric Contract Matrix + +Tuple order is `considered/opportunities/would_apply/applied/protected/blocking_removed/blocking_preserved/dropped`. Every row below also gets a configured-but-inactive assertion: `status:not-run`, the pass-specific closed `reason_code`, and no numeric fields. + +| Pass | No opportunity | Predicate miss | Active | Ablated | Protection case | Blocking result | +|---|---|---|---|---|---|---| +| `evidence.fact-location` | unreadable WARN: `1/0/0/0/0/0/0/0` | valid line WARN: `1/1/0/0/0/0/0/0` | out-of-range WARN→INFO: `1/1/1/1/0/1/0/0` | same stays WARN: `1/1/1/0/0/0/1/0` | no guard | active `1→0`, ablated `1→1`; re-anchor variant is `1/1/1/1/0/0/1/0` | +| `evidence.self-refutation` | INFO: `1/0/0/0/0/0/0/0` | ordinary WARN: `1/1/0/0/0/0/0/0` | retracting WARN→INFO: `1/1/1/1/0/1/0/0` | same stays WARN: `1/1/1/0/0/0/1/0` | retracting correctness WARN: `1/1/1/0/1/0/1/0` | active `1→0`, ablated/protected `1→1` | +| `judgment.hypothetical` | WARN: `1/0/0/0/0/0/0/0` | present-defect CRITICAL: `1/1/0/0/0/0/0/0` | hypothetical CRITICAL→WARN: `1/1/1/1/0/0/1/0` | stays CRITICAL: `1/1/1/0/0/0/1/0` | hypothetical security CRITICAL: `1/1/1/0/1/0/1/0` | all remain blocking; severity must differ active vs ablated | +| `evidence.grounding-token` | tokenless WARN: `1/0/0/0/0/0/0/0` | present token CRITICAL: `1/1/0/0/0/0/0/0` | absent token CRITICAL→WARN: `1/1/1/1/0/0/1/0` | stays CRITICAL: `1/1/1/0/0/0/1/0` | absent-token security CRITICAL: `1/1/1/0/1/0/1/0` | all remain blocking; severity must differ | +| `judgment.grounding-llm` | CRITICAL without judge row: `1/0/0/0/0/0/0/0` | `grounded:true`: `1/1/0/0/0/0/0/0` | `grounded:false` CRITICAL→WARN: `1/1/1/1/0/0/1/0` | stays CRITICAL: `1/1/1/0/0/0/1/0` | ungrounded correctness CRITICAL: `1/1/1/0/1/0/1/0` | all remain blocking; severity must differ | +| `evidence.redaction-placeholder` | INFO placeholder: `1/0/0/0/0/0/0/0` | bland placeholder WARN: `1/1/0/0/0/0/0/0` | undefined-placeholder WARN→INFO: `1/1/1/1/0/1/0/0` | stays WARN: `1/1/1/0/0/0/1/0` | secret-word/security placeholder: `1/1/1/0/1/0/1/0` | active `1→0`, ablated/protected `1→1` | +| `judgment.critic` | signature omitted: `1/0/0/0/0/0/0/0` | critic `keep`: `1/1/0/0/0/0/0/0` | likely-FP WARN→INFO: `1/1/1/1/0/1/0/0` | stays WARN: `1/1/1/0/0/0/1/0` | majority WARN: `1/1/1/0/1/0/1/0` | active `1→0`, ablated/protected `1→1`; INFO-drop variant has `dropped:1` | +| `scope.diff` | line `0`: `1/0/0/0/0/0/0/0` | inside hunk WARN: `1/1/0/0/0/0/0/0` | outside hunk WARN→INFO: `1/1/1/1/0/1/0/0` | stays WARN: `1/1/1/0/0/0/1/0` | escaped category WARN: `1/1/1/0/1/0/1/0` | active `1→0`, ablated/protected `1→1` | +| `scope.delta` | INFO: `1/0/0/0/0/0/0/0` | file inside delta: `1/1/0/0/0/0/0/0` | outside-delta WARN→INFO: `1/1/1/1/0/1/0/0` | stays WARN: `1/1/1/0/0/0/1/0` | correctness WARN outside: `1/1/1/0/1/0/1/0` | active `1→0`, ablated/protected `1→1` | +| `scope.session` | INFO: `1/0/0/0/0/0/0/0` | owned WARN: `1/1/0/0/0/0/0/0` | foreign WARN→INFO: `1/1/1/1/0/1/0/0` | stays WARN: `1/1/1/0/0/0/1/0` | escaped category foreign WARN: `1/1/1/0/1/0/1/0` | active `1→0`, ablated/protected `1→1` | +| `history.fp-signature` | INFO: `1/0/0/0/0/0/0/0` | unknown WARN signature: `1/1/0/0/0/0/0/0` | active WARN signature→INFO: `1/1/1/1/0/1/0/0` | stays WARN: `1/1/1/0/0/0/1/0` | no guard | active `1→0`, ablated `1→1` | +| `history.cycle-rejected` | INFO: `1/0/0/0/0/0/0/0` | unknown WARN signature: `1/1/0/0/0/0/0/0` | rejected quality WARN→INFO: `1/1/1/1/0/1/0/0` | stays WARN: `1/1/1/0/0/0/1/0` | rejected correctness WARN: `1/1/1/0/1/0/1/0` | active `1→0`, ablated/protected `1→1` | +| `history.fp-cluster` | INFO: `1/0/0/0/0/0/0/0` | unknown cluster key: `1/1/0/0/0/0/0/0` | active cluster WARN→INFO: `1/1/1/1/0/1/0/0` | stays WARN: `1/1/1/0/0/0/1/0` | no guard | active `1→0`, ablated `1→1` | +| `judgment.confidence` | majority WARN: `1/0/0/0/0/0/0/0` | confidence at floor: `1/1/0/0/0/0/0/0` | low-confidence WARN→INFO: `1/1/1/1/0/1/0/0` | stays WARN: `1/1/1/0/0/0/1/0` | high-precision WARN: `1/1/1/0/1/0/1/0` | active `1→0`, ablated/protected `1→1`; CRITICAL clamp preserves blocking | +| `judgment.reputation` | majority WARN: `1/0/0/0/0/0/0/0` | reliable reviewer WARN: `1/1/0/0/0/0/0/0` | unreliable quality WARN→INFO: `1/1/1/1/0/1/0/0` | stays WARN: `1/1/1/0/0/0/1/0` | unreliable security WARN: `1/1/1/0/1/0/1/0` | active `1→0`, ablated/protected `1→1`; CRITICAL quality/correctness clamp preserves blocking | +| `history.region-rejected` | line `0`: `1/0/0/0/0/0/0/0` | no overlapping region: `1/1/0/0/0/0/0/0` | eligible overlap WARN→INFO: `1/1/1/1/0/1/0/0` | stays WARN: `1/1/1/0/0/0/1/0` | one-prior-reject overlap: `1/1/1/0/1/0/1/0` | active `1→0`, ablated/protected `1→1` | +| `judgment.test-security` | INFO test finding: `1/0/0/0/0/0/0/0` | quality WARN in test: `1/1/0/0/0/0/0/0` | security WARN in test→INFO: `1/1/1/1/0/1/0/0` | stays WARN: `1/1/1/0/0/0/1/0` | mixed security/correctness cluster: `1/1/1/0/1/0/1/0` | active `1→0`, ablated/protected `1→1` | +| `judgment.docs-cap` | WARN docs finding: `1/0/0/0/0/0/0/0` | quality CRITICAL in source: `1/1/0/0/0/0/0/0` | quality CRITICAL docs→WARN: `1/1/1/1/0/0/1/0` | stays CRITICAL: `1/1/1/0/0/0/1/0` | correctness CRITICAL docs: `1/1/1/0/1/0/1/0` | all remain blocking; severity must differ | + +--- + +### Task 1: Static Catalog and Persisted Schemas + +**Files:** +- Create: `src/core/policy/catalog.ts` +- Create: `src/schemas/policy-trace.ts` +- Modify: `src/schemas/finding.ts` +- Modify: `src/schemas/pending-report.ts` +- Modify: `src/schemas/audit-event.ts` +- Create: `tests/unit/policy-catalog.test.ts` +- Create: `tests/unit/policy-trace-schema.test.ts` +- Modify: `tests/unit/finding-schema.test.ts` +- Modify: `tests/unit/pending-report.test.ts` +- Modify: `tests/unit/run-summary-schema.test.ts` + +**Interfaces:** +- Produces: `PolicyPassId`, `PolicyStageId`, `PolicyCatalogId`, `PolicyReasonCode`, `PolicyProtectionCode`, `PolicyEffectAction`, `POLICY_CATALOG_VERSION`, `POLICY_PASS_IDS`, `POLICY_PASSES`, `POLICY_STAGES`, `PolicyEffectSchema`, `PolicyEvaluationSchema`, `PolicyPassSummarySchema`, `PolicyStageEvaluationSchema`, `PolicyTraceFinalSchema`, `PolicyTraceSchema`, `PolicySummarySchema`. +- Consumers: all later tasks; no catalog entry executes dynamic code. + +- [ ] **Step 1: Write the failing catalog and schema tests** + +```ts +expect(POLICY_PASSES.map((p) => [p.order, p.id])).toEqual([ + [10, "evidence.fact-location"], + [20, "evidence.self-refutation"], + [30, "judgment.hypothetical"], + [40, "evidence.grounding-token"], + [50, "judgment.grounding-llm"], + [60, "evidence.redaction-placeholder"], + [70, "judgment.critic"], + [80, "scope.diff"], + [90, "scope.delta"], + [100, "scope.session"], + [110, "history.fp-signature"], + [120, "history.cycle-rejected"], + [130, "history.fp-cluster"], + [140, "judgment.confidence"], + [150, "judgment.reputation"], + [160, "history.region-rejected"], + [170, "judgment.test-security"], + [180, "judgment.docs-cap"], +]); +expect(POLICY_STAGES.map((p) => p.id)).toEqual(["aggregation.cluster", "verdict.compute"]); +expect(PolicyPassSummarySchema.safeParse({ + pass_id: "judgment.confidence", + status: "ran", + considered: 1, + opportunities: 1, + would_apply: 1, + applied: 2, + protected: 0, + blocking_removed: 1, + blocking_preserved: 0, + dropped: 0, +}).success).toBe(false); +expect(PolicyPassSummarySchema.safeParse({ + pass_id: "judgment.confidence", + status: "not-run", + reason_code: "configured-off", + considered: 0, +}).success).toBe(false); +``` + +- [ ] **Step 2: Run the new tests and verify missing-module failures** + +Run: `bun test tests/unit/policy-catalog.test.ts tests/unit/policy-trace-schema.test.ts` + +Expected: FAIL because catalog/schema modules do not exist. + +- [ ] **Step 3: Implement the catalog and strict semantic schemas** + +Use discriminated unions for ran versus inactive summaries and a `superRefine` for counter relationships and complete/ref/hash invariants: + +```ts +export const POLICY_CATALOG_VERSION = "reviewgate.policy-catalog.v1" as const; +export const POLICY_PASS_IDS = [ + "evidence.fact-location", + "evidence.self-refutation", + "judgment.hypothetical", + "evidence.grounding-token", + "judgment.grounding-llm", + "evidence.redaction-placeholder", + "judgment.critic", + "scope.diff", + "scope.delta", + "scope.session", + "history.fp-signature", + "history.cycle-rejected", + "history.fp-cluster", + "judgment.confidence", + "judgment.reputation", + "history.region-rejected", + "judgment.test-security", + "judgment.docs-cap", +] as const; +export const PolicyPassIdSchema = z.enum(POLICY_PASS_IDS); +export const PolicyPassSummarySchema = z.discriminatedUnion("status", [ + RanPolicyPassSummarySchema, + InactivePolicyPassSummarySchema, +]); +export const PolicyTraceSchema = z.object({ + schema: z.literal("reviewgate.policy-trace.v1"), + catalog_version: z.literal(POLICY_CATALOG_VERSION), + run_id: z.string(), + iter: z.number().int().nonnegative(), + ablated: z.array(PolicyPassIdSchema), + raw_response_sha256: z.array(z.string().regex(/^[0-9a-f]{64}$/)), + passes: z.array(PolicyPassSummarySchema), + evaluations: z.array(PolicyEvaluationSchema), + stages: z.array(PolicyStageEvaluationSchema), + final: PolicyTraceFinalSchema, +}).strict(); +``` + +Add optional `policy_effects` to `FindingSchema`, optional `policy_summary` to `PendingReportSchema`, and optional `policy_trace_status`, `policy_trace_ref`, `policy_trace_sha256` to `RunSummarySchema`. Keep every outer schema literal unchanged. + +- [ ] **Step 4: Add compatibility and malicious-artifact tests** + +Prove old fixtures parse, unknown pass/reason/protection/action fails, result counts cannot violate the catalog, `source_signatures` are sorted/deduplicated, and reviewer-controlled prose cannot occupy `reason_code`. + +- [ ] **Step 5: Run focused schema tests** + +Run: `bun test tests/unit/policy-catalog.test.ts tests/unit/policy-trace-schema.test.ts tests/unit/finding-schema.test.ts tests/unit/pending-report.test.ts tests/unit/run-summary-schema.test.ts` + +Expected: PASS. + +- [ ] **Step 6: Mutation-prove schema guards and commit** + +In a disposable copy, remove `judgment.docs-cap`, permit `applied > would_apply`, and allow a free-form reason. Each mutation must make a named test fail. Restore, then commit: + +```bash +git add src/core/policy/catalog.ts src/schemas/policy-trace.ts src/schemas/finding.ts src/schemas/pending-report.ts src/schemas/audit-event.ts tests/unit/policy-catalog.test.ts tests/unit/policy-trace-schema.test.ts tests/unit/finding-schema.test.ts tests/unit/pending-report.test.ts tests/unit/run-summary-schema.test.ts +git commit -m "feat(policy): define trace catalog and schemas" +``` + +### Task 2: Fail-Open Recorder and Transition Boundary + +**Files:** +- Create: `src/core/policy/trace.ts` +- Create: `src/core/policy/response-hashes.ts` +- Create: `tests/unit/policy-trace-recorder.test.ts` +- Create: `tests/unit/policy-response-hashes.test.ts` + +**Interfaces:** +- Consumes: catalog and schema types from Task 1. +- Produces: `PolicyRuntime`, `PolicyTraceRecorder`, `transitionFinding`, `mergePolicyEffects`, `OrderedResponseHashes`. + +- [ ] **Step 1: Write recorder tests for every terminal result** + +```ts +const warnFinding: Finding = { + id: "F-001", + signature: "sig-confidence", + severity: "WARN", + category: "quality", + rule_id: "naming", + file: "src/x.ts", + line_start: 1, + line_end: 1, + message: "name is unclear", + details: "rename the value", + reviewer: { provider: "codex", model: "m", persona: "quality" }, + confidence: 0.2, + consensus: "singleton", +}; +const runtime = PolicyTraceRecorder.start({ runId: "run-1", iter: 1, ablated: [] }); +const after = transitionFinding({ + runtime, + passId: "judgment.confidence", + finding: warnFinding, + opportunity: true, + matched: true, + reasonCode: "below-confidence-floor", + action: "demoted", + proposed: () => ({ ...warnFinding, severity: "INFO", low_confidence: true }), +}); +expect(after?.severity).toBe("INFO"); +expect(runtime.summary("judgment.confidence")).toMatchObject({ + considered: 1, + opportunities: 1, + would_apply: 1, + applied: 1, + protected: 0, + blocking_removed: 1, + blocking_preserved: 0, + dropped: 0, +}); +``` + +Also assert `no-opportunity`, `no-match`, `protected`, `would-apply`, applied drop, applied re-anchor, deduplicated effects, ascending order, and final-signature linking. + +- [ ] **Step 2: Verify the recorder tests fail** + +Run: `bun test tests/unit/policy-trace-recorder.test.ts tests/unit/policy-response-hashes.test.ts` + +Expected: FAIL because recorder classes do not exist. + +- [ ] **Step 3: Implement a mutation-first transition helper** + +The helper calculates the existing proposed result outside telemetry error handling. A recorder exception marks telemetry failed but returns the same proposed finding production would have returned; only an explicit internal ablation returns the original matched finding. + +```ts +export interface TransitionInput { + runtime?: PolicyRuntime; + passId: PolicyPassId; + finding: Finding; + opportunity: boolean; + matched: boolean; + reasonCode: PolicyReasonCode; + action: PolicyEffectAction; + protectedBy?: PolicyProtectionCode; + proposed: () => Finding | null; +} + +export function transitionFinding(input: TransitionInput): Finding | null { + if (!input.runtime) return input.matched && !input.protectedBy ? input.proposed() : input.finding; + return input.runtime.transition(input); +} +``` + +`PolicyRuntime.transition` must catch only recorder/effect attachment failures, keep the pre-existing mutation result, record `telemetryError`, and never catch a predicate or proposal error that existing code would already surface. + +- [ ] **Step 4: Implement deterministic response hashing** + +`OrderedResponseHashes.record(kind, ordinal, rawText)` stores only `sha256(Buffer.from(rawText, "utf8"))` in deterministic logical-call order. Empty successful output hashes as SHA-256 of the empty byte string; thrown calls add no response. + +- [ ] **Step 5: Run focused tests and mutation checks** + +Mutate the helper to apply an ablated transition, omit an opportunity increment, swap effect order, and return the original on recorder failure. Each mutation must fail a specific test. + +- [ ] **Step 6: Commit** + +```bash +git add src/core/policy/trace.ts src/core/policy/response-hashes.ts tests/unit/policy-trace-recorder.test.ts tests/unit/policy-response-hashes.test.ts +git commit -m "feat(policy): add fail-open transition recorder" +``` + +### Task 3: Instrument the Five Pre-Aggregation Passes + +**Files:** +- Modify: `src/core/fact-check.ts` +- Modify: `src/core/self-refutation.ts` +- Modify: `src/core/hypothetical-demote.ts` +- Modify: `src/core/grounding.ts` +- Modify: `src/core/critic.ts` +- Modify: `tests/unit/fact-check.test.ts` +- Modify: `tests/unit/fact-check-reanchor.test.ts` +- Modify: `tests/unit/self-refutation.test.ts` +- Modify: `tests/unit/hypothetical-demote.test.ts` +- Modify: `tests/unit/grounding.test.ts` +- Modify: `tests/unit/grounding-judge.test.ts` +- Modify: `tests/unit/critic-runner.test.ts` +- Create: `tests/unit/policy-preaggregation-contracts.test.ts` + +**Interfaces:** +- Consumes: optional final `runtime?: PolicyRuntime` parameter on each pure pass. +- Produces: contract rows 10–50 and raw judge/critic response hashes. + +- [ ] **Step 1: Add failing active/ablated/protected tests for orders 10–50** + +For each of the first five matrix rows, call the existing exported function twice with identical inputs: a normal runtime and a runtime ablating only that pass. Strip `policy_effects` and assert the normal result equals the existing legacy fixture while the ablated result retains its starting severity. + +```ts +const criticalQualityFinding: Finding = { + id: "F-001", + signature: "sig-grounding", + severity: "CRITICAL", + category: "quality", + rule_id: "missing-token", + file: "src/x.ts", + line_start: 1, + line_end: 1, + message: "`theme.missingToken` is referenced", + details: "the token is absent", + reviewer: { provider: "codex", model: "m", persona: "quality" }, + confidence: 0.9, + consensus: "singleton", +}; +const activeRuntime = PolicyTraceRecorder.start({ runId: "active", iter: 1, ablated: [] }); +const ablatedRuntime = PolicyTraceRecorder.start({ + runId: "ablated", + iter: 1, + ablated: ["evidence.grounding-token"], +}); +const active = groundFindings([criticalQualityFinding], "export const present = true", activeRuntime); +const ablated = groundFindings([criticalQualityFinding], "export const present = true", ablatedRuntime); +expect(active[0]).toMatchObject({ + severity: "WARN", + grounding_demoted: true, + demoted_from_critical: true, +}); +expect(ablated[0]).toMatchObject({ severity: "CRITICAL" }); +expect(activeRuntime.summary("evidence.grounding-token")).toMatchObject({ + considered: 1, opportunities: 1, would_apply: 1, applied: 1, protected: 0, + blocking_removed: 0, blocking_preserved: 1, dropped: 0, +}); +``` + +- [ ] **Step 2: Verify red failures** + +Run: `bun test tests/unit/policy-preaggregation-contracts.test.ts` + +Expected: FAIL because the helpers cannot accept a policy runtime. + +- [ ] **Step 3: Route existing predicates through `transitionFinding`** + +Keep filesystem containment, regexes, prompts, parse logic, notes, marker fields, and call order unchanged. Extend signatures only at the final optional position: + +```ts +validateFindingFacts(findings, repoRoot, deletedPaths, runtime?) +demoteSelfRefuting(findings, enabled, runtime?) +demoteHypotheticalCriticals(findings, enabled, runtime?) +groundFindings(findings, corpus, runtime?) +applyGroundingJudgeVerdicts(findings, map, runtime?) +``` + +Return raw response SHA-256 from `judgeGrounding` and `runCritic` as optional additive result fields; do not persist response text in the policy trace. + +- [ ] **Step 4: Run all affected legacy and contract tests** + +Run: `bun test tests/unit/fact-check.test.ts tests/unit/fact-check-reanchor.test.ts tests/unit/self-refutation.test.ts tests/unit/hypothetical-demote.test.ts tests/unit/grounding.test.ts tests/unit/grounding-judge.test.ts tests/unit/critic-runner.test.ts tests/unit/policy-preaggregation-contracts.test.ts` + +Expected: PASS with legacy assertions unchanged after telemetry stripping. + +- [ ] **Step 5: Mutation-prove and commit** + +Mutate one security/correctness protection, one re-anchor action, and one raw-response hash return. Restore after each named test fails. + +```bash +git add src/core/fact-check.ts src/core/self-refutation.ts src/core/hypothetical-demote.ts src/core/grounding.ts src/core/critic.ts tests/unit/fact-check.test.ts tests/unit/fact-check-reanchor.test.ts tests/unit/self-refutation.test.ts tests/unit/hypothetical-demote.test.ts tests/unit/grounding.test.ts tests/unit/grounding-judge.test.ts tests/unit/critic-runner.test.ts tests/unit/policy-preaggregation-contracts.test.ts +git commit -m "feat(policy): trace pre-aggregation decisions" +``` + +### Task 4: Instrument Redaction, Clustering, Critic, and Scope Passes + +**Files:** +- Modify: `src/core/aggregator.ts` +- Modify: `tests/unit/aggregator-redaction-demote.test.ts` +- Modify: `tests/unit/aggregator-members.test.ts` +- Modify: `tests/unit/aggregator-critic.test.ts` +- Modify: `tests/unit/aggregator-scope.test.ts` +- Modify: `tests/unit/aggregator-claude-scope.test.ts` +- Modify: `tests/unit/aggregator-foreign-session.test.ts` +- Create: `tests/unit/policy-aggregator-first-half.test.ts` + +**Interfaces:** +- Modify `AggregateInput` with optional `policyRuntime?: PolicyRuntime`. +- Produce orders 60–100 plus `aggregation.cluster` stage rows and final lineage links. + +- [ ] **Step 1: Add failing matrix tests for orders 60–100 and cluster lineage** + +Use one finding per numeric case so each expected tuple is exact. Add a two-reviewer cluster where one member was redaction-demoted before clustering; the final representative must carry the member effect with `source_signatures`, and both input signatures must link to the same `final_signature`. + +- [ ] **Step 2: Verify red failures** + +Run: `bun test tests/unit/policy-aggregator-first-half.test.ts` + +Expected: FAIL because `AggregateInput` has no runtime and no effects/stages are emitted. + +- [ ] **Step 3: Instrument without extracting policy into new pass modules** + +Preserve the current sequence: + +```text +redaction → normalize/sort → cluster → claimed-fixed pin → critic → diff scope → delta scope → session scope +``` + +Each existing branch supplies explicit `opportunity`, `matched`, applied reason, protection, action, and lazy `proposed` mutation to `transitionFinding`. Do not derive opportunity from whether a marker appeared afterward. + +- [ ] **Step 4: Merge effects and record cluster stages deterministically** + +The internal cluster accumulator keeps a separate `effects: PolicyEffect[]`; `mergePolicyEffects` sorts/deduplicates them before assigning the final representative. Record one `aggregation.cluster` stage row per output cluster, including singletons, then call `runtime.linkFinal(inputSignatures, representative.signature)`. + +- [ ] **Step 5: Run affected aggregator suites** + +Run: `bun test tests/unit/aggregator-redaction-demote.test.ts tests/unit/aggregator-members.test.ts tests/unit/aggregator-critic.test.ts tests/unit/aggregator-scope.test.ts tests/unit/aggregator-claude-scope.test.ts tests/unit/aggregator-foreign-session.test.ts tests/unit/policy-aggregator-first-half.test.ts` + +Expected: PASS. + +- [ ] **Step 6: Mutation-prove and commit** + +Mutate critic drop attribution, scope protection, and member-effect propagation. Confirm each targeted test goes red, restore, and commit. + +```bash +git add src/core/aggregator.ts tests/unit/aggregator-redaction-demote.test.ts tests/unit/aggregator-members.test.ts tests/unit/aggregator-critic.test.ts tests/unit/aggregator-scope.test.ts tests/unit/aggregator-claude-scope.test.ts tests/unit/aggregator-foreign-session.test.ts tests/unit/policy-aggregator-first-half.test.ts +git commit -m "feat(policy): trace clustering and scope decisions" +``` + +### Task 5: Instrument History, Confidence, Reputation, Category Caps, and Verdict + +**Files:** +- Modify: `src/core/aggregator.ts` +- Modify: `tests/unit/aggregator-fp.test.ts` +- Modify: `tests/unit/aggregator-cycle-rejected.test.ts` +- Modify: `tests/unit/aggregator-fp-cluster.test.ts` +- Modify: `tests/unit/aggregator-confidence.test.ts` +- Modify: `tests/unit/aggregator-reputation.test.ts` +- Modify: `tests/unit/aggregator-region-rejected.test.ts` +- Modify: `tests/unit/aggregator-test-severity.test.ts` +- Modify: `tests/unit/aggregator-docs-cap.test.ts` +- Modify: `tests/unit/aggregator-protect-high-precision.test.ts` +- Create: `tests/unit/policy-aggregator-second-half.test.ts` + +**Interfaces:** +- Consumes: Task 4 runtime threading. +- Produces: orders 110–180 and exactly one `verdict.compute` stage row. + +- [ ] **Step 1: Add failing matrix tests for orders 110–180** + +Encode every tuple from the matrix, including the protected cases and these blocking-preserving variants: low-confidence CRITICAL clamp, reputation CRITICAL quality clamp, reputation CRITICAL correctness corroboration clamp, and docs CRITICAL cap. + +- [ ] **Step 2: Verify red failures** + +Run: `bun test tests/unit/policy-aggregator-second-half.test.ts` + +Expected: FAIL because no second-half trace rows exist. + +- [ ] **Step 3: Instrument in the exact current order** + +```text +FP signature → cycle rejection → FP cluster → confidence → reputation → region rejection → test security → docs cap → lone-critical tag → verdict +``` + +Protection is recorded on the attempted pass itself. Render-only `lone_critical_uncorroborated` remains outside the outcome-changing catalog. + +- [ ] **Step 4: Record the verdict stage** + +Emit one `verdict.compute` row after counts are final with sorted blocking signatures and one closed reason: `hard-critical`, `corroborated-warn`, `claimed-fixed-recurrence`, `blocking-present`, or `no-blocking-findings`. + +- [ ] **Step 5: Run affected suites** + +Run: `bun test tests/unit/aggregator-fp.test.ts tests/unit/aggregator-cycle-rejected.test.ts tests/unit/aggregator-fp-cluster.test.ts tests/unit/aggregator-confidence.test.ts tests/unit/aggregator-reputation.test.ts tests/unit/aggregator-region-rejected.test.ts tests/unit/aggregator-test-severity.test.ts tests/unit/aggregator-docs-cap.test.ts tests/unit/aggregator-protect-high-precision.test.ts tests/unit/policy-aggregator-second-half.test.ts` + +Expected: PASS. + +- [ ] **Step 6: Mutation-prove and commit** + +Mutate FP-cluster attribution, the G0 clamp, a high-precision protection, and verdict reason selection; verify named failures and restore. + +```bash +git add src/core/aggregator.ts tests/unit/aggregator-fp.test.ts tests/unit/aggregator-cycle-rejected.test.ts tests/unit/aggregator-fp-cluster.test.ts tests/unit/aggregator-confidence.test.ts tests/unit/aggregator-reputation.test.ts tests/unit/aggregator-region-rejected.test.ts tests/unit/aggregator-test-severity.test.ts tests/unit/aggregator-docs-cap.test.ts tests/unit/aggregator-protect-high-precision.test.ts tests/unit/policy-aggregator-second-half.test.ts +git commit -m "feat(policy): trace history and judgment decisions" +``` + +### Task 6: Orchestrator Lifecycle and Behavior-Neutral Equivalence + +**Files:** +- Modify: `src/core/orchestrator.ts` +- Modify: `src/core/run-summary.ts` +- Create: `src/core/policy/replay.ts` +- Create: `tests/unit/orchestrator-policy-trace.test.ts` +- Create: `tests/integration/policy-trace-equivalence.test.ts` +- Modify: `tests/integration/run-summary-orchestrator.test.ts` + +**Interfaces:** +- Add internal `PolicyExecutionOptions` with `trace: "off" | "memory" | "persist"`, `policyAblations: ReadonlySet`, `authoritative: boolean`, and optional isolated state metadata. +- Add optional `policyTrace` and `policySummary` to `IterationResult`; these are server-owned and absent on legacy/non-policy paths. +- Add optional policy runtime parameters to `buildRunSummary` without changing legacy defaults. + +- [ ] **Step 1: Write a failing trace-on/trace-off integration test** + +Run the same deterministic adapter twice from byte-identical temporary repos. One uses `trace:"memory"`, one uses `trace:"off"`. Assert equal provider calls, raw response hashes, verdict, counts, order, legacy markers, and JSON after recursively stripping only `policy_effects`, `policy_summary`, and policy trace fields. Assert both rendered Markdown files are byte-identical. + +- [ ] **Step 2: Verify the equivalence test fails** + +Run: `bun test tests/integration/policy-trace-equivalence.test.ts` + +Expected: FAIL because Orchestrator cannot construct/finalize a policy runtime. + +- [ ] **Step 3: Create and thread one runtime on the full-panel path** + +Create the recorder immediately before `validateFindingFacts`, pass it through all prepasses and `aggregate`, append reviewer hashes in configured slot order plus grounding/critic response hashes in logical call order, finalize after final findings/counts are known, and expose the validated trace in `IterationResult`. + +Mode selection is closed and deterministic: omitted options plus an `AuditLogger` means `persist` +(the production Gate path); omitted options without an audit logger means `off` (legacy direct unit +construction); Bench/Rig explicitly request `memory` or `persist`. A memory-only run exposes the +full trace in `IterationResult` but omits `policy_summary` from its one-shot pending artifact until +Bench/Rig writes a real trace ref/hash of its own. + +Every early skip/cache/check/error return uses `policy_trace_status:"not-run"` and emits no artifact reference. No trace fields enter cache keys or cached values. + +- [ ] **Step 4: Enforce internal-only ablation plumbing** + +`src/cli/commands/gate.ts`, `src/cli/commands/config.ts`, `src/cli/commands/setup.ts`, config schemas, and environment parsing must contain no mapping into `policyAblations`. Add a source-level guard test over those files and a runtime test showing ordinary gate construction uses an empty set. + +- [ ] **Step 5: Run orchestrator and equivalence suites** + +Run: `bun test tests/unit/orchestrator-policy-trace.test.ts tests/integration/policy-trace-equivalence.test.ts tests/integration/run-summary-orchestrator.test.ts tests/unit/orchestrator.test.ts tests/unit/orchestrator-raw-reviews.test.ts` + +Expected: PASS. + +- [ ] **Step 6: Mutation-prove and commit** + +Mutate raw-response ordering, add a trace field to the cache key, and let a recorder error change one severity. Each named equivalence assertion must fail. + +```bash +git add src/core/orchestrator.ts src/core/run-summary.ts src/core/policy/replay.ts tests/unit/orchestrator-policy-trace.test.ts tests/integration/policy-trace-equivalence.test.ts tests/integration/run-summary-orchestrator.test.ts +git commit -m "feat(policy): wire trace lifecycle through orchestration" +``` + +### Task 7: Content-Addressed Audit Persistence and Verification + +**Files:** +- Create: `src/audit/canonical.ts` +- Create: `src/audit/policy-trace-store.ts` +- Modify: `src/audit/logger.ts` +- Modify: `src/audit/verifier.ts` +- Modify: `src/core/orchestrator.ts` +- Modify: `src/core/report-writer.ts` +- Modify: `src/core/loop-driver.ts` +- Create: `tests/unit/policy-trace-store.test.ts` +- Modify: `tests/unit/audit-logger.test.ts` +- Modify: `tests/unit/audit-logger-retention.test.ts` +- Modify: `tests/unit/audit-verify-corruption.test.ts` +- Modify: `tests/unit/report-writer.test.ts` + +**Interfaces:** +- Produce `canonicalJson(value): string`, `writePolicyTrace(input): PolicyTraceWriteResult`, and `verifyPolicyTraceReference(input): PolicyTraceVerification`. +- `AuditLogger.writePolicyTrace` returns `{status, ref?, sha256?}` and never throws into verdict code. + +- [ ] **Step 1: Write failing persistence, overflow, traversal, and tamper tests** + +```ts +const stored = writePolicyTrace({ auditDir, trace, maxBytes: 1_048_576, now }); +expect(stored).toMatchObject({ status: "complete", sha256: expect.stringMatching(/^[0-9a-f]{64}$/) }); +expect(stored.ref).toMatch(/^2026\/08\/10\/policy\/[0-9a-f]{12}-i1-[0-9a-f]{12}\.json$/); +expect(verifyPolicyTraceReference({ auditDir, ref: stored.ref!, sha256: stored.sha256! }).ok).toBe(true); +``` + +Also assert a `1_048_577`-byte canonical artifact produces `overflow`, creates no policy file/temp file, and leaves ref/hash absent; a run ID of `../../escape` cannot affect the path; byte tampering fails verification. + +- [ ] **Step 2: Verify red failures** + +Run: `bun test tests/unit/policy-trace-store.test.ts tests/unit/audit-verify-corruption.test.ts` + +Expected: FAIL because the store and reference verifier do not exist. + +- [ ] **Step 3: Extract canonical JSON and implement atomic storage** + +Move the identical sorted-key canonicalizer out of logger/verifier into `src/audit/canonical.ts`. Derive `run-sha12` from `sha256(run_id)`, derive `content-sha12` from canonical bytes, use the logger's UTC day partition, and write with `writeFileAtomic(destinationPath, canonicalBytes, { mode: 0o600 })` only after the complete size check. + +- [ ] **Step 4: Bind trace status/ref/hash into pending and run.complete** + +Persist before `writeReport`, attach the same compact `PolicySummary` to pending JSON, pass fields into `buildRunSummary`, and let LoopDriver's existing best-effort `run.complete` append bind them into the hash chain. Existing badge copy must be derived only for already-existing badge variants; legacy marker fallback remains and Markdown fixtures stay byte-identical. + +- [ ] **Step 5: Extend chain verification and retention tests** + +`verifyChain` validates any complete policy reference relative to the audit root, rejects missing/hash-mismatched/escaping files, and continues accepting legacy events. Day-partition deletion must remove its `policy/` child with no special case. + +- [ ] **Step 6: Verify the 1 MiB limit with a worst-case synthetic trace** + +Construct maximum-length allowed signatures and one evaluation for every configured pass across enough findings to cross the boundary. Assert the largest under-limit fixture is complete and the next evaluation causes overflow; record the exact evaluated byte sizes in the test names/output. + +- [ ] **Step 7: Run focused suites and commit** + +Run: `bun test tests/unit/policy-trace-store.test.ts tests/unit/audit-logger.test.ts tests/unit/audit-logger-retention.test.ts tests/unit/audit-verify-corruption.test.ts tests/unit/report-writer.test.ts tests/unit/pending-report.test.ts` + +```bash +git add src/audit/canonical.ts src/audit/policy-trace-store.ts src/audit/logger.ts src/audit/verifier.ts src/core/orchestrator.ts src/core/report-writer.ts src/core/loop-driver.ts tests/unit/policy-trace-store.test.ts tests/unit/audit-logger.test.ts tests/unit/audit-logger-retention.test.ts tests/unit/audit-verify-corruption.test.ts tests/unit/report-writer.test.ts +git commit -m "feat(policy): persist and verify audit traces" +``` + +### Task 8: Exact Bench Ablation on Captured Responses + +**Files:** +- Modify: `src/bench/runner.ts` +- Modify: `src/cli/commands/bench.ts` +- Modify: `src/schemas/bench-result.ts` +- Modify: `src/bench/report.ts` +- Modify: `tests/unit/bench-runner.test.ts` +- Modify: `tests/unit/bench-matrix.test.ts` +- Modify: `tests/unit/bench-result-schema.test.ts` +- Modify: `tests/unit/bench-report.test.ts` +- Modify: `tests/unit/bench-preregistration.test.ts` + +**Interfaces:** +- `RunBenchCaseInput` gains internal `policyExecution?: PolicyExecutionOptions`. +- Matrix pass names resolve to catalog IDs; legacy aliases `critic`, `confidence-floor`, `reputation`, and `scope-to-diff` remain accepted and normalize to their catalog IDs. +- Produce `validateAuthoritativeTracePair(baseline, counterfactual)` returning exact invalidity reasons. + +- [ ] **Step 1: Replace the confidence toggle test with a true internal ablation test** + +The baseline and variant must use byte-identical effective config, including a still-enabled confidence floor. Only `policyAblations = new Set(["judgment.confidence"])` differs. Assert the same metric delta as the current test and identical ordered raw response hashes. + +- [ ] **Step 2: Add authoritative invalidity tests** + +Table-test missing pass row, missing trace, `error`, `overflow`, content hash mismatch, catalog mismatch, and raw response mismatch. Every case must return exit `4`, include the precise cause, and never turn missing counters into zero. + +- [ ] **Step 3: Verify red failures** + +Run: `bun test tests/unit/bench-matrix.test.ts tests/unit/bench-result-schema.test.ts` + +Expected: FAIL because matrix still changes public suppressor config instead of ablating the production transition. + +- [ ] **Step 4: Capture and replay every provider response once** + +Extend the current capture wrappers to record both `review` and `complete` results. Baseline is the only live call path. Every variant replays exact stored raw text/result objects, recomputes request identity, and fails before scoring on a request or response mismatch. Do not run the critic live per variant. + +- [ ] **Step 5: Persist trace provenance with matrix artifacts** + +Each variant carries catalog version, normalized ablated pass ID, trace status/ref/hash, and ordered response hashes. Keep the existing immutable output checks and result SHA-256 references. + +- [ ] **Step 6: Run Bench suites and mutation checks** + +Run: `bun test tests/unit/bench-runner.test.ts tests/unit/bench-matrix.test.ts tests/unit/bench-result-schema.test.ts tests/unit/bench-report.test.ts tests/unit/bench-preregistration.test.ts` + +Mutate the response comparison, configured-pass completeness check, and catalog comparison; each authoritative test must fail. + +- [ ] **Step 7: Commit** + +```bash +git add src/bench/runner.ts src/cli/commands/bench.ts src/schemas/bench-result.ts src/bench/report.ts tests/unit/bench-runner.test.ts tests/unit/bench-matrix.test.ts tests/unit/bench-result-schema.test.ts tests/unit/bench-report.test.ts tests/unit/bench-preregistration.test.ts +git commit -m "feat(bench): run exact policy ablations" +``` + +### Task 9: Rig Trace Validation and Branch-Isolated Replay + +**Files:** +- Create: `src/schemas/policy-replay.ts` +- Create: `src/core/policy/replay-capture.ts` +- Modify: `src/schemas/rig-manifest.ts` +- Modify: `src/schemas/rig-result.ts` +- Modify: `src/cli/commands/gate.ts` +- Modify: `src/rig/driver.ts` +- Modify: `src/rig/harvest.ts` +- Modify: `src/rig/replay.ts` +- Modify: `src/rig/ablate.ts` +- Modify: `src/cli/commands/rig.ts` +- Create: `src/rig/policy-replay-state.ts` +- Modify: `tests/unit/rig-driver.test.ts` +- Modify: `tests/unit/rig-harvest.test.ts` +- Modify: `tests/unit/rig-replay.test.ts` +- Modify: `tests/unit/rig-ablate.test.ts` +- Create: `tests/unit/policy-replay-capture.test.ts` + +**Interfaces:** +- Add optional manifest `policyReplay` metadata: catalog version, source commit, initial state snapshot ref/hash, and cassette hash. +- Add per-turn policy trace refs/statuses to RigResult without changing old artifact parsing. +- Produce strict `PolicyReplayEnvelopeSchema` with run/iteration identity, exact reviewed diff, policy-input findings, grounding corpus, aggregate inputs, state digest, ordered response hashes, and `lossless:boolean`. +- Produce `createReplayBranches(input)` returning separate baseline/counterfactual temporary checkouts with equal starting-state digests. + +- [ ] **Step 1: Write failing branch-isolation tests** + +Create a production-like repo with FP-ledger and reputation files. Build a baseline/counterfactual pair, mutate each branch independently, and assert the source repo bytes never change, initial digests match, later branch digests may diverge, and neither branch path aliases the source `.reviewgate/` directory. + +- [ ] **Step 2: Add Rig invalidity tests** + +Harvest/replay must exit `4` for missing/corrupt/overflow/cross-catalog traces and state-digest mismatch. Legacy runs remain harvestable but are explicitly non-authoritative for policy ablation rather than counted as zero opportunities. + +- [ ] **Step 3: Verify red failures** + +Run: `bun test tests/unit/policy-replay-capture.test.ts tests/unit/rig-replay.test.ts tests/unit/rig-harvest.test.ts` + +Expected: FAIL because Rig currently checks only post-hoc determinism and four heuristic layers. + +- [ ] **Step 4: Capture one exact envelope per gate iteration outside the measured repo** + +`runRigRun` creates `/policy-replay/`, realpath-confirms it is beneath the Rig output directory, and exports only `REVIEWGATE_RIG_REPLAY_DIR` to the driven agent process. Gate may use that variable solely to construct a capture sink; it never reads pass IDs or ablation controls from the environment. Orchestrator writes `-i.json` mode `0600` containing the exact diff, post-review/pre-policy findings, grounding corpus, aggregate input sets/maps as sorted arrays, state digest, and response hashes. + +Run every string leaf through the cassette's entropy redactor before persistence and compare pre/post canonical bytes. When redaction changes any policy-relevant byte, set `lossless:false`; Rig preserves the artifact for diagnosis but authoritative replay exits `4` rather than pretending it is exact. The envelope carries no credentials, raw environment, prompt text, config code, or free-form filesystem path. + +- [ ] **Step 5: Record immutable replay identity without writing measured state** + +At Rig start, copy the initial `.reviewgate` state into the result directory, hash it, record the source commit/catalog/cassette hash, and never write into the measured repo from replay code. Per turn, preserve the trace-bearing audit snapshot already copied by the driver. + +- [ ] **Step 6: Implement isolated replay branches** + +Use two temporary checkouts hydrated from the same source commit and each envelope's exact iteration diff, never the final turn diff as a substitute. Copy the same initial/previous-turn state snapshot into each branch, verify its digest, route all learning writes into that branch, and delete branches after use. Feed the envelope's exact policy inputs through the production pass functions; cassette responses are matched by ordered raw hash and no provider method may execute live. Baseline and counterfactual retain their own writes across a multi-turn sequence so downstream causal state divergence is measured rather than erased. + +- [ ] **Step 7: Replace heuristic pass labels with catalog IDs** + +Keep lore reporting separate because it is additive, not a demoter. Preserve the old four-layer output only as explicitly non-authoritative legacy analysis; exact policy rows come from validated traces and internal replay. + +- [ ] **Step 8: Run Rig suites and mutation checks** + +Run: `bun test tests/unit/policy-replay-capture.test.ts tests/unit/rig-driver.test.ts tests/unit/rig-harvest.test.ts tests/unit/rig-replay.test.ts tests/unit/rig-ablate.test.ts tests/unit/rig-preregistration.test.ts` + +Mutate capture-path containment, lossless enforcement, exact iteration-diff selection, source-state containment, starting digest equality, and missing-trace handling; each test must fail. + +- [ ] **Step 9: Commit** + +```bash +git add src/schemas/policy-replay.ts src/core/policy/replay-capture.ts src/schemas/rig-manifest.ts src/schemas/rig-result.ts src/cli/commands/gate.ts src/rig/driver.ts src/rig/harvest.ts src/rig/replay.ts src/rig/ablate.ts src/cli/commands/rig.ts src/rig/policy-replay-state.ts tests/unit/policy-replay-capture.test.ts tests/unit/rig-driver.test.ts tests/unit/rig-harvest.test.ts tests/unit/rig-replay.test.ts tests/unit/rig-ablate.test.ts +git commit -m "feat(rig): validate and replay policy traces" +``` + +### Task 10: Cross-Pass Contract Harness and Completeness Mutations + +**Files:** +- Create: `tests/fixtures/policy-pass-contracts.ts` +- Create: `tests/unit/policy-pass-contract-matrix.test.ts` +- Create: `tests/integration/policy-trace-offline-replay.test.ts` +- Create: `docs/dev/2026-08-10-policy-trace-mutation-evidence.md` + +**Interfaces:** +- Produce `POLICY_PASS_CONTRACTS`, one fixture builder per catalogued pass, consumed only by tests. +- Consume the production transition path and Bench/Rig replay APIs; no shadow policy implementation is permitted in fixtures. + +- [ ] **Step 1: Encode all 18 rows before touching any remaining production behavior** + +```ts +for (const contract of POLICY_PASS_CONTRACTS) { + it(`${contract.passId}: numeric contract`, async () => { + const result = await contract.run(); + expect(result.noOpportunity).toEqual(contract.expected.noOpportunity); + expect(result.noMatch).toEqual(contract.expected.noMatch); + expect(result.active).toEqual(contract.expected.active); + expect(result.ablated).toEqual(contract.expected.ablated); + if (contract.expected.protected) expect(result.protected).toEqual(contract.expected.protected); + expect(result.activeBlocking).toBe(contract.expected.activeBlocking); + expect(result.ablatedBlocking).toBe(contract.expected.ablatedBlocking); + }); +} +``` + +The expected tuples and blocking results are copied literally from this plan's matrix; fixture builders call exported production passes/aggregate rather than reproducing predicates. + +- [ ] **Step 2: Add four-class offline replay acceptance** + +Use one deterministic evidence pass, value-judgment pass, scope pass, and stateful history pass. Assert complete traces, identical raw response hashes, correct active/ablated verdicts, no live provider calls in counterfactual runs, and no production-state writes. + +- [ ] **Step 3: Run the complete contract tests** + +Run: `bun test tests/unit/policy-pass-contract-matrix.test.ts tests/integration/policy-trace-offline-replay.test.ts` + +Expected: PASS for all 18 pass IDs and both explanatory stages. + +- [ ] **Step 4: Execute the required mutation dossier** + +In disposable copies, perform and restore each mutation: remove a catalog entry; mutate without recording; omit opportunity increment; swap effect order; let ablated severity change; drop a member effect; skip raw hash comparison; accept tampered hash; coerce missing trace to zero; let persistence failure alter verdict. Record command, failing test name, and restored clean result in `docs/dev/2026-08-10-policy-trace-mutation-evidence.md`. + +- [ ] **Step 5: Commit** + +```bash +git add tests/fixtures/policy-pass-contracts.ts tests/unit/policy-pass-contract-matrix.test.ts tests/integration/policy-trace-offline-replay.test.ts docs/dev/2026-08-10-policy-trace-mutation-evidence.md +git commit -m "test(policy): prove all pass contracts and mutations" +``` + +### Task 11: Documentation, Full Verification, and Independent Review + +**Files:** +- Modify: `docs/architecture.md` +- Modify: `TEST_PLAN.md` +- Modify: `NEXT_SESSION.md` +- Modify: `docs/superpowers/specs/2026-08-09-policy-accountability-trace-design.md` +- Modify: `AGENTS.md` only if entry-point paths changed materially + +**Interfaces:** +- Document catalog ownership, trace location/status, authoritative-versus-production failure semantics, internal ablation boundary, and Slice-2 measurement limits. + +- [ ] **Step 1: Update durable repository documentation** + +Mark Slice 1 implemented with its final commit/test evidence. State explicitly that zero opportunities is not evidence of uselessness, Lore is excluded, stateful passes require seeded sequences, and no pass has yet been deleted. + +- [ ] **Step 2: Run focused source-runtime tests once more** + +```bash +bun test tests/unit/policy-catalog.test.ts tests/unit/policy-trace-schema.test.ts tests/unit/policy-trace-recorder.test.ts tests/unit/policy-pass-contract-matrix.test.ts tests/integration/policy-trace-equivalence.test.ts tests/integration/policy-trace-offline-replay.test.ts tests/unit/bench-matrix.test.ts tests/unit/rig-replay.test.ts tests/unit/audit-verify-corruption.test.ts +``` + +Expected: all pass, zero fail. + +- [ ] **Step 3: Run mandatory repository verification** + +```bash +bunx tsc --noEmit +bun run lint +bun test +``` + +Expected: Typecheck and lint clean; full suite has zero failures. + +- [ ] **Step 4: Build and smoke the compiled CLI paths** + +```bash +bun run build +./dist/reviewgate bench matrix --help +./dist/reviewgate rig replay --help +./dist/reviewgate audit --help +``` + +Expected: build succeeds and each affected command exits normally with usage text. Do not run a live provider benchmark in this verification step. + +- [ ] **Step 5: Run the repository's independent review pipeline** + +Review the complete implementation diff against the approved spec and this plan. Resolve every CRITICAL/WARN through the normal Reviewgate decision protocol; require a final PASS before completion. + +- [ ] **Step 6: Update Brain and commit the handoff** + +Persist final commit, exact suite counts, mutation evidence, remaining Slice-2 work, and any measured limit change in `/Users/markus/Documents/Brain/02 Projekte/Aktiv/ReviewGate.md` plus the current daily note. + +```bash +git add docs/architecture.md TEST_PLAN.md NEXT_SESSION.md docs/superpowers/specs/2026-08-09-policy-accountability-trace-design.md AGENTS.md +git commit -m "docs: hand off policy trace slice one" +``` + +Do not stage `AGENTS.md` if it did not require a real entry-point change. Do not push. + +## Execution Handoff + +This plan is deliberately sequential: Tasks 1–7 establish the trusted trace, Task 8 makes Bench authoritative, Task 9 makes Rig state-safe, and Tasks 10–11 prove completeness. Execute one task per fresh review gate and stop if either prerequisite reliability fix is still absent. diff --git a/docs/superpowers/specs/2026-08-09-policy-accountability-trace-design.md b/docs/superpowers/specs/2026-08-09-policy-accountability-trace-design.md index 0b138ba..f4b9d15 100644 --- a/docs/superpowers/specs/2026-08-09-policy-accountability-trace-design.md +++ b/docs/superpowers/specs/2026-08-09-policy-accountability-trace-design.md @@ -1,6 +1,6 @@ # Policy Accountability & Pruning — Slice 1: Policy Trace & Replay -_Written 2026-08-09. Status: design approved in dialogue; written spec awaiting Markus review._ +_Written 2026-08-09. Status: approved by Markus; implementation planning in progress._ ## Context @@ -184,9 +184,16 @@ interface PolicyEvaluation { reason_code: string; // validated against the pass catalog protected_by?: string; source_signatures: string[]; + final_signature?: string; } ``` +`source_signatures` identify the raw or clustered finding lineage at the moment the pass evaluates +it. `final_signature` identifies the visible final finding (or cluster representative) that carries +that lineage after all later passes; every evaluation that converges into the same surviving +cluster uses the same value. It is omitted when that lineage is dropped and has no visible final +finding. + This makes the opportunity denominator auditable per case/finding rather than leaving only an aggregate count. A configured pass that did not run has no per-finding evaluations and carries `status: not-run` in its summary instead. @@ -206,6 +213,15 @@ interface PolicyEffect { protected_by?: string; source_signatures: string[]; } + +interface PolicyStageEvaluation { + stage_id: "aggregation.cluster" | "verdict.compute"; + order: number; + reason_code: string; + input_signatures: string[]; + output_signature?: string; + verdict?: "PASS" | "SOFT-PASS" | "FAIL"; +} ``` Constraints: @@ -227,9 +243,9 @@ fields can be deleted after policy pruning. Each configured pass emits one compact row: ```ts -interface PolicyPassSummary { +interface RanPolicyPassSummary { pass_id: PolicyPassId; - status: "ran" | "not-run" | "error"; + status: "ran"; considered: number; opportunities: number; would_apply: number; @@ -239,11 +255,22 @@ interface PolicyPassSummary { blocking_preserved: number; dropped: number; } + +interface InactivePolicyPassSummary { + pass_id: PolicyPassId; + status: "not-run" | "error"; + reason_code: string; +} + +type PolicyPassSummary = RanPolicyPassSummary | InactivePolicyPassSummary; ``` -All numeric fields are required when `status: ran`. Missing data is not defaulted to zero. The -summary schema rejects impossible relationships such as `applied > would_apply` or -`protected > would_apply`. +All numeric fields are required when `status: ran` and forbidden otherwise. `blocking_removed` +counts applied transitions from CRITICAL/WARN to INFO/drop. `blocking_preserved` counts matched +events that were blocking before the pass and remain CRITICAL/WARN afterward, whether because of a +protection or an applied blocking-preserving transition such as re-anchoring or CRITICAL→WARN. +Missing data is not defaulted to zero. The summary schema rejects impossible relationships such as +`applied > would_apply`, `protected > would_apply`, or `dropped > applied`. ### Full policy trace artifact @@ -258,7 +285,8 @@ interface PolicyTrace { ablated: PolicyPassId[]; raw_response_sha256: string[]; passes: PolicyPassSummary[]; - evaluations: Array; + evaluations: PolicyEvaluation[]; + stages: PolicyStageEvaluation[]; final: { verdict: "PASS" | "SOFT-PASS" | "FAIL" | "ERROR"; counts: { critical: number; warn: number; info: number }; @@ -269,7 +297,9 @@ interface PolicyTrace { The full artifact includes evaluations for findings later dropped by a pass and `would-apply` observations from an ablated pass. A visible final finding contains only its own applied/protected -material effects. +material effects. `aggregation.cluster` emits one stage row per resulting cluster (including a +singleton row), and `verdict.compute` emits exactly one row with the final blocking signatures and +the selected verdict reason. Neither stage is ablatable or counted as a demoter. ## Recorder and transition boundary @@ -327,7 +357,21 @@ For an ablated pass: - `would_apply` remains observable; - the mutation and material marker do not apply; - every later pass runs normally on the counterfactual finding; -- reviewer inputs, provider calls, state reads and raw responses remain unchanged. +- reviewer inputs, provider calls and raw responses remain unchanged. + +Authoritative replay state is branch-isolated. Baseline and counterfactual start from byte-identical +immutable snapshots with the same recorded state digest, then each receives its own scratch copy. +All learning-store and pass-owned writes go only to that branch's scratch copy; an ablated pass does +not disable otherwise-normal reads or writes. This preserves downstream causal effects in +multi-turn sequences without contaminating the paired branch. Scratch state is never reused across +pairs and must never resolve to the production checkout's `.reviewgate/` tree. Bench/Rig reject a +pair before execution when the starting digests differ or either scratch target aliases production +state. Tests prove authoritative runs leave every production learning store byte-identical. + +Within a single compared iteration, both branches therefore perform the same state-read code paths +against the same starting snapshot. In a multi-turn sequence, later read values may intentionally +diverge only because earlier branch-local outcomes produced different branch-local writes; that +divergence is part of the measured policy effect and is recorded in the sequence result. This is a production-path ablation, not a second hand-built model of pass precedence. @@ -353,6 +397,23 @@ especially: `policy_effects`. The existing `reviewgate.pending.v1` literal remains valid because the additions are optional and old reports remain parseable. +Conceptual shape: + +```ts +interface PolicySummary { + catalog_version: "reviewgate.policy-catalog.v1"; + status: "complete" | "not-run" | "error" | "overflow"; + passes: PolicyPassSummary[]; + policy_trace_ref?: string; + policy_trace_sha256?: string; +} +``` + +`passes` contains exactly one ordered row for each of the 18 catalogued passes; inactive passes use +`status: not-run`. +Reference and hash are both required for `complete` and forbidden for every other status. The +summary contains no per-finding evaluations and remains optional as a unit for legacy reports. + ### Audit artifact One trace file is written atomically per reviewed iteration inside the existing audit day @@ -400,9 +461,14 @@ agent/human explanation. A new top-level CLI subcommand is explicitly out of sco Trace telemetry must never decide whether code passes: -- recorder errors preserve the existing finding and continue the gate; +- recorder/effect-attachment errors preserve the pre-instrumentation policy result (including an + existing mutation or drop) and continue the gate; - artifact write errors leave ref/hash absent and set trace status `error`; -- an artifact over the size limit sets `overflow` and is not silently truncated; +- the complete trace is canonicalized in memory before any artifact path is created; when its bytes + exceed the limit, no temporary or destination artifact is written, ref/hash remain absent, the + compact pending summary remains available with status `overflow`, and the full buffer is + discarded after ordinary report data is produced; +- the recorder never stops mid-run and no summary-only, sampled or truncated trace file is emitted; - pending/report writing continues with the canonical existing finding data; - no trace failure changes a verdict, dirty flag, iteration or decision requirement. From 06257123ef9ae724617b27447a7cfc0fbcd0ca5b Mon Sep 17 00:00:00 2001 From: Codevena Date: Mon, 10 Aug 2026 00:38:36 +0200 Subject: [PATCH 16/55] test(rig): fixture reports carry their own turn's run_id --- tests/unit/rig-harvest.test.ts | 27 ++++++++++++++++++++++++--- 1 file changed, 24 insertions(+), 3 deletions(-) diff --git a/tests/unit/rig-harvest.test.ts b/tests/unit/rig-harvest.test.ts index 88d8f90..5ae1fb9 100644 --- a/tests/unit/rig-harvest.test.ts +++ b/tests/unit/rig-harvest.test.ts @@ -70,6 +70,13 @@ interface FxTurn { * keys on the whole file's hash and a report rewritten for an unrelated reason is a new file. */ reportIters?: number[]; + /** + * parallel to `reports`: the `run_id` that version carries. Defaults to this turn's own + * (`session-`), i.e. a report the turn's own gate produced. Set it to an EARLIER + * turn's id to model the archiver catching a leftover `pending.json`, or to an id no audit + * event carries to model a report that cannot be attributed to any turn. + */ + reportRunIds?: (string | undefined)[]; agentExitCode?: number; } @@ -138,10 +145,15 @@ function turnAuditJsonl(turn: FxTurn, turnIndex: number): string { return lines.length === 0 ? "" : `${lines.join("\n")}\n`; } -function pendingReport(findings: FxFinding[], iter: number, critic?: FxCritic): string { +function pendingReport( + findings: FxFinding[], + iter: number, + critic: FxCritic | undefined, + runId: string, +): string { return JSON.stringify({ schema: "reviewgate.pending.v1", - run_id: "session-x", + run_id: runId, iter, max_iter: 5, verdict: findings.length === 0 ? "PASS" : "FAIL", @@ -253,7 +265,12 @@ function buildFixture(turns: FxTurn[], opts: { scriptId?: string } = {}): Fixtur for (const [n, findings] of reports.entries()) { writeFileSync( join(reportDir, `${n + 1}-pending.json`), - pendingReport(findings, turn.reportIters?.[n] ?? n + 1, turn.critics?.[n]), + pendingReport( + findings, + turn.reportIters?.[n] ?? n + 1, + turn.critics?.[n], + turn.reportRunIds?.[n] ?? `session-${index}`, + ), ); } } @@ -361,11 +378,13 @@ describe("rig harvest", () => { const { manifestPath, scriptPath } = buildFixture([ { // Critic RAN and kept everything: demoted 0, so suppression.critic is 0 too. + iterations: [{}], reports: [[{ signature: "sig-a", severity: "WARN", message: "a finding" }]], critics: [{ provider: "openrouter", status: "ran", verdicts: 4, demoted: 0 }], }, { // pilot-01's shape: findings, but no critic key at all. + iterations: [{}], reports: [[{ signature: "sig-b", severity: "WARN", message: "another finding" }]], }, ]); @@ -393,6 +412,7 @@ describe("rig harvest", () => { [{ signature: "sig-b", severity: "WARN", message: "two" }], ], reportIters: [1, 1], + iterations: [{}], critics: [same, same], }, ]); @@ -413,6 +433,7 @@ describe("rig harvest", () => { [{ signature: "sig-b", severity: "WARN", message: "two" }], ], reportIters: [1, 2], + iterations: [{}, {}], critics: [same, same], }, ]); From 0d77be1147667b7a18a866b4c84e542bce22896e Mon Sep 17 00:00:00 2001 From: Codevena Date: Mon, 10 Aug 2026 00:42:35 +0200 Subject: [PATCH 17/55] test(rig): keep no-panel fixture report attributed --- tests/unit/rig-harvest.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/rig-harvest.test.ts b/tests/unit/rig-harvest.test.ts index 5ae1fb9..72c8d5a 100644 --- a/tests/unit/rig-harvest.test.ts +++ b/tests/unit/rig-harvest.test.ts @@ -908,7 +908,7 @@ describe("rig harvest", () => { join(reportDir, "9-pending.json"), JSON.stringify({ schema: "reviewgate.pending.v1", - run_id: "session-x", + run_id: "session-1", iter: 1, max_iter: 5, verdict: "PASS", From 825662e2f936685e0c2dad31fbc414ec4c98e92b Mon Sep 17 00:00:00 2001 From: Codevena Date: Mon, 10 Aug 2026 00:53:57 +0200 Subject: [PATCH 18/55] fix(rig): a report belongs to the turn whose audit delta owns its run_id --- src/rig/harvest.ts | 39 +++++++++++++ tests/unit/rig-harvest.test.ts | 100 +++++++++++++++++++++++++++++++++ 2 files changed, 139 insertions(+) diff --git a/src/rig/harvest.ts b/src/rig/harvest.ts index b466091..73820bd 100644 --- a/src/rig/harvest.ts +++ b/src/rig/harvest.ts @@ -142,6 +142,8 @@ function collectTurnFindings( snapshotDir: string, turnIndex: number, warnings: string[], + ownedRunIds: Set, + knownRunIds: Set, ): { findings: Finding[]; panel: PanelSlot[]; reportsRead: number; criticRuns: CriticInfo[] } { const reportsDir = join(snapshotDir, "reports"); const bySignature = new Map(); @@ -163,6 +165,8 @@ function collectTurnFindings( .sort((a, b) => (Number.parseInt(a, 10) || 0) - (Number.parseInt(b, 10) || 0)); let reportsRead = 0; + let inheritedCount = 0; + const orphanNames: string[] = []; for (const name of names) { // JSON.parse inside the try, not just the schema check: the archiver writes atomically, // but a snapshot copied while a file was being renamed away can still land truncated, and @@ -184,6 +188,20 @@ function collectTurnFindings( ); continue; } + // OWNERSHIP. The archiver captures whatever `pending.json` is on disk, which on 31 of 36 + // recorded pilot turns was the PREVIOUS turn's leftover — counting it here would count one + // finding once in the turn that produced it and again in the turn that merely saw it, the + // very double-count the per-turn signature dedup above exists to prevent. A gate run lives + // inside one Stop hook and therefore one turn, so `run_id` alone identifies the owner + // (verified 1:1 across all 34 recorded gate runs). Keyed on run_id and NOT on (run_id, iter): + // a gate that writes a report and then dies before appending `run.complete` would otherwise + // have its real report discarded as unattributable. + const runId = parsed.data.run_id; + if (!ownedRunIds.has(runId)) { + if (knownRunIds.has(runId)) inheritedCount++; + else orphanNames.push(name); + continue; + } reportsRead++; if (parsed.data.critic) criticRuns.set(`${parsed.data.run_id}:${parsed.data.iter}`, parsed.data.critic); @@ -202,6 +220,20 @@ function collectTurnFindings( }); } } + // One line per TURN, not per report: naming each of eleven inherited files would bury the + // signal. Nothing is lost — each is counted in the turn whose gate produced it. + if (inheritedCount > 0) { + warnings.push( + `turn ${turnIndex}: ${inheritedCount} archived report(s) carry a run_id produced by an EARLIER turn — the gate did not write them during this turn. They are EXCLUDED here and counted where they were produced, so one finding is not counted twice across turns.`, + ); + } + // One line per REPORT, and loud: unlike an inherited report, an orphan is not counted anywhere, + // so this is real data loss rather than a correction. + for (const orphan of orphanNames) { + warnings.push( + `turn ${turnIndex}: archived report ${orphan} carries a run_id that appears in NO turn's audit events and was EXCLUDED — it cannot be attributed to any turn (pruned audit day-partition, or a snapshot from a different run). This turn's findings may be UNDERSTATED.`, + ); + } return { findings: [...bySignature.values()], panel: [...panel.values()], @@ -410,10 +442,17 @@ function harvestTurn( ); } + // `window.runs` is cumulative for this snapshot, so it carries every earlier turn's runs too — + // which is exactly what distinguishes an INHERITED report (owned by an earlier turn) from an + // ORPHAN (owned by none). + const ownedRunIds = new Set(runDelta.added.map((r) => r.run_id)); + const knownRunIds = new Set(window.runs.map((r) => r.run_id)); const { findings, panel, reportsRead, criticRuns } = collectTurnFindings( snapshotDir, index, warnings, + ownedRunIds, + knownRunIds, ); const blocking = findings.filter(isBlocking); const iterations = runDelta.added.length; diff --git a/tests/unit/rig-harvest.test.ts b/tests/unit/rig-harvest.test.ts index 72c8d5a..e8b8755 100644 --- a/tests/unit/rig-harvest.test.ts +++ b/tests/unit/rig-harvest.test.ts @@ -938,4 +938,104 @@ describe("rig harvest", () => { { provider: "openrouter", model: "anthropic/claude-sonnet-4.5", persona: "security" }, ]); }); + + test("an inherited report is not counted again in the turn that merely saw it", () => { + const fx = buildFixture([ + { seeded: null, iterations: [{ warn: 1 }], reports: [[{ signature: "s1" }]] }, + { + seeded: null, + iterations: [{ warn: 1 }], + // The archiver's first poll caught turn 1's leftover, then turn 2's own report. + reports: [[{ signature: "s1" }], [{ signature: "s2" }]], + reportRunIds: ["session-1", undefined], + reportIters: [1, 1], + }, + ]); + const result = harvest(fx.manifestPath, fx.scriptPath); + + expect(result.turns[1]?.findingsTotal).toBe(1); + expect(result.turns[1]?.findings[0]?.signature).toBe("s2"); + expect(result.warnings.some((w) => w.includes("turn 2") && /EARLIER turn/.test(w))).toBe(true); + }); + + test("a turn the gate never reviewed reports nothing, not its predecessor's findings", () => { + const fx = buildFixture([ + { + seeded: null, + iterations: [{ warn: 3 }], + reports: [[{ signature: "a" }, { signature: "b" }, { signature: "c" }]], + }, + // The agent died; the gate never ran. Only turn 1's leftover was on disk to archive. + { + seeded: null, + iterations: [], + reports: [[{ signature: "a" }, { signature: "b" }, { signature: "c" }]], + reportRunIds: ["session-1"], + }, + ]); + const result = harvest(fx.manifestPath, fx.scriptPath); + + expect(result.turns[1]?.iterations).toBe(0); + expect(result.turns[1]?.findingsTotal).toBe(0); + }); + + test("a report owned by NO turn is dropped and warned about, not charged to this turn", () => { + const fx = buildFixture([ + { + seeded: null, + iterations: [{ warn: 1 }], + reports: [[{ signature: "own" }], [{ signature: "ghost" }]], + reportRunIds: [undefined, "session-99"], + reportIters: [1, 1], + }, + ]); + const result = harvest(fx.manifestPath, fx.scriptPath); + + expect(result.turns[0]?.findingsTotal).toBe(1); + expect(result.warnings.some((w) => /NO turn's audit events/.test(w))).toBe(true); + }); + + test("criticRuns is not attributed to a turn that only INHERITED the report", () => { + const critic = { provider: "ollama", status: "ran" as const, verdicts: 2, demoted: 1 }; + const fx = buildFixture([ + { + seeded: null, + iterations: [{ warn: 1 }], + reports: [[{ signature: "s1" }]], + critics: [critic], + }, + { + seeded: null, + iterations: [{ warn: 0 }], + reports: [[{ signature: "s1" }], []], + critics: [critic, undefined], + reportRunIds: ["session-1", undefined], + reportIters: [1, 1], + }, + ]); + const result = harvest(fx.manifestPath, fx.scriptPath); + + expect(result.turns[0]?.criticRuns?.length).toBe(1); + expect(result.turns[1]?.criticRuns ?? []).toEqual([]); + }); + + test("reportsRead counts only OWNED reports, so the unmeasured-turn warning still fires", () => { + const fx = buildFixture([ + { seeded: null, iterations: [{ warn: 1 }], reports: [[{ signature: "s1" }]] }, + // The gate DID run, but the archiver caught only turn 1's leftover — none of turn 2's own. + { + seeded: null, + iterations: [{ warn: 1 }], + reports: [[{ signature: "s1" }]], + reportRunIds: ["session-1"], + }, + ]); + const result = harvest(fx.manifestPath, fx.scriptPath); + + expect(result.turns[1]?.iterations).toBe(1); + expect(result.turns[1]?.findingsTotal).toBe(0); + expect( + result.warnings.some((w) => w.includes("turn 2") && /NO pending\.json was archived/.test(w)), + ).toBe(true); + }); }); From cce7ed237dafcf3e7f9b6bf2904b51379295ad44 Mon Sep 17 00:00:00 2001 From: Codevena Date: Mon, 10 Aug 2026 01:03:13 +0200 Subject: [PATCH 19/55] fix(rig): archive only reports that appear DURING the turn --- src/rig/driver.ts | 15 +++++++++++ tests/unit/rig-driver.test.ts | 47 +++++++++++++++++++++++++++++++++++ 2 files changed, 62 insertions(+) diff --git a/src/rig/driver.ts b/src/rig/driver.ts index 63bf718..1afd47a 100644 --- a/src/rig/driver.ts +++ b/src/rig/driver.ts @@ -219,6 +219,21 @@ function startReportArchiver(repoRoot: string, destDir: string): () => void { // this archiver exists to keep (gate finding F-001). const seq = new Map(); const seen = new Set(); + // Seed with the state on disk BEFORE the agent runs. The docstring promises every version that + // APPEARS while the turn runs; without this seed the first poll (250ms in, long before this + // turn's gate has written anything) captures the PREVIOUS turn's leftover pending.json as this + // turn's report #1. Nothing is lost: the previous turn's own final sweep already archived those + // exact bytes. Hashed, not merely name-checked — a report REWRITTEN during this turn must still + // be archived. + for (const name of ["pending.json", "pending.md"]) { + const src = join(reviewgateDir(repoRoot), name); + if (!existsSync(src)) continue; + try { + seen.add(`${name}:${createHash("sha256").update(readFileSync(src, "utf8")).digest("hex")}`); + } catch { + /* unreadable this instant → it is simply not seeded, and a later tick captures it */ + } + } const capture = () => { for (const name of ["pending.json", "pending.md"]) { const src = join(reviewgateDir(repoRoot), name); diff --git a/tests/unit/rig-driver.test.ts b/tests/unit/rig-driver.test.ts index 8d496e7..1f20976 100644 --- a/tests/unit/rig-driver.test.ts +++ b/tests/unit/rig-driver.test.ts @@ -312,6 +312,53 @@ describe("rig driver", () => { expect(mds.some((c) => c.includes("iteration 2"))).toBe(true); }, 20_000); + test("does NOT archive a pending.json left behind by the PREVIOUS turn", async () => { + // The archiver promises every version that APPEARS while the turn runs. A file already on + // disk when the turn starts did not appear during it — on 31 of 36 recorded pilot turns this + // leftover was archived as that turn's report #1. + const { root, scriptPath } = sandbox(1); + const pending = join(root, ".reviewgate", "pending.json"); + writeFileSync(pending, '{"verdict":"FAIL","findings":[{"rule_id":"stale-from-last-turn"}]}'); + const manifest = await runDriver({ + scriptPath, + outDir: join(root, "out"), + repoRoot: root, + agentCmd: appendingAgent(root), // touches agent.log only; pending.json is never rewritten + maxTurns: 1, + }); + const reportsDir = join(manifest.turns[0]?.snapshotDir ?? "", "reports"); + const archived = existsSync(reportsDir) + ? readdirSync(reportsDir).filter((f) => f.endsWith("pending.json")) + : []; + expect(archived).toEqual([]); + }, 20_000); + + test("still archives a pending.json that CHANGES during the turn, even if one existed before", async () => { + // The over-suppression guard. A fix that skipped on filename, or on "a file was already + // there", rather than on CONTENT HASH would swallow this turn's real report. + const { root, scriptPath } = sandbox(1); + const pending = join(root, ".reviewgate", "pending.json"); + writeFileSync(pending, '{"verdict":"FAIL","findings":[{"rule_id":"stale-from-last-turn"}]}'); + const manifest = await runDriver({ + scriptPath, + outDir: join(root, "out"), + repoRoot: root, + agentCmd: gateLikeWriter([ + // Keep the stale file in place for one poll before replacing it. Without this delay the + // fake agent can win the race and rewrite pending.json before the archiver's first tick. + { file: join(root, ".reviewgate", "unrelated.txt"), body: "delay one poll" }, + { file: pending, body: '{"verdict":"FAIL","findings":[{"rule_id":"fresh-this-turn"}]}' }, + ]), + maxTurns: 1, + }); + const reportsDir = join(manifest.turns[0]?.snapshotDir ?? "", "reports"); + const archived = readdirSync(reportsDir) + .filter((f) => f.endsWith("pending.json")) + .map((f) => readFileSync(join(reportsDir, f), "utf8")); + expect(archived.some((c) => c.includes("fresh-this-turn"))).toBe(true); + expect(archived.some((c) => c.includes("stale-from-last-turn"))).toBe(false); + }, 20_000); + test("records the agent's exit code instead of swallowing a failed turn", async () => { const { root, scriptPath } = sandbox(1); const manifest = await runDriver({ From eb602b38fb82ce5ae7b980f4205c0782c4af25a5 Mon Sep 17 00:00:00 2001 From: Codevena Date: Mon, 10 Aug 2026 01:17:38 +0200 Subject: [PATCH 20/55] docs: correct the pilot metrics invalidated by the rig stale-report defect --- docs/dev/2026-08-05-pilot-01-result.md | 4 ++ docs/dev/2026-08-05-pilot-02-result.md | 4 ++ docs/dev/2026-08-06-pilot-03-result.md | 4 ++ .../2026-08-07-rig-stale-report-correction.md | 61 +++++++++++++++++++ ...26-07-29-longitudinal-effectiveness-rig.md | 4 ++ ...26-08-05-fp-ledger-evidence-unit-design.md | 4 ++ 6 files changed, 81 insertions(+) create mode 100644 docs/dev/2026-08-07-rig-stale-report-correction.md diff --git a/docs/dev/2026-08-05-pilot-01-result.md b/docs/dev/2026-08-05-pilot-01-result.md index e596c73..c16a96b 100644 --- a/docs/dev/2026-08-05-pilot-01-result.md +++ b/docs/dev/2026-08-05-pilot-01-result.md @@ -4,6 +4,10 @@ _2026-08-05. Task 6 of `docs/superpowers/plans/2026-07-29-longitudinal-effective _Preregistered in `rig/preregistrations/pilot-01.json` **before** the run. Nothing below was registered in light of a result._ +> **Correction (2026-08-10):** The stale-report defect invalidated the M2 value and sample size +> reported here. The preserved before/after record is in +> `docs/dev/2026-08-07-rig-stale-report-correction.md`. + ## Headline 12 turns, 5 seeded defects, 61 minutes of wall clock, **every turn reviewed by the gate**. diff --git a/docs/dev/2026-08-05-pilot-02-result.md b/docs/dev/2026-08-05-pilot-02-result.md index 9749492..68bc555 100644 --- a/docs/dev/2026-08-05-pilot-02-result.md +++ b/docs/dev/2026-08-05-pilot-02-result.md @@ -4,6 +4,10 @@ _2026-08-05. Second longitudinal run. Preregistered in `rig/preregistrations/pil (commit `ac2f5d5`, frozen and pushed **before** the run started). Baseline: `docs/dev/2026-08-05-pilot-01-result.md`._ +> **Correction (2026-08-10):** The stale-report defect reduced the valid M2 samples for both +> pilots quoted here and changed pilot-01's M2 value. The preserved before/after record is in +> `docs/dev/2026-08-07-rig-stale-report-correction.md`. + ## Headline 12 turns, 39.8 minutes, **every turn reviewed**, zero failed turns. The critic ran on every diff --git a/docs/dev/2026-08-06-pilot-03-result.md b/docs/dev/2026-08-06-pilot-03-result.md index 09e7acb..3cdc865 100644 --- a/docs/dev/2026-08-06-pilot-03-result.md +++ b/docs/dev/2026-08-06-pilot-03-result.md @@ -5,6 +5,10 @@ _2026-08-06. Third longitudinal run. Preregistered in `rig/preregistrations/pilo started). Baselines: `docs/dev/2026-08-05-pilot-02-result.md`, and the design under test, `docs/superpowers/specs/2026-08-05-true-positive-hole-design.md`._ +> **Correction (2026-08-10):** The stale-report defect reduced the valid M2 samples for both +> pilots quoted here; their rounded slope values remain unchanged. The preserved before/after +> record is in `docs/dev/2026-08-07-rig-stale-report-correction.md`. + ## Headline 12 turns, 53 minutes. **All three registered primary outcomes were met. The run's most diff --git a/docs/dev/2026-08-07-rig-stale-report-correction.md b/docs/dev/2026-08-07-rig-stale-report-correction.md new file mode 100644 index 0000000..4a9639f --- /dev/null +++ b/docs/dev/2026-08-07-rig-stale-report-correction.md @@ -0,0 +1,61 @@ +# Correction — pilot metrics after the rig stale-report defect + +_Published 2026-08-10. This preserves the pre-fix record and reports an offline re-harvest of +pilot-01, pilot-02 and pilot-03._ + +## Defect and scope + +The rig archived a previous turn's final pending report again at the start of the next turn, and +the harvester treated that inherited report as if the later turn had produced it. Across the three +pilots, 31 of 36 turns inherited a report; 13 of 36 turns consequently counted findings they did +not produce, and 9 of those 13 turns produced no findings of their own. The corrected ownership +rule counts a report only in the turn whose audit delta owns its `run_id`. + +## Metrics before correction + +This table was captured before the fix. It is retained verbatim so the correction cannot be +back-fitted to the new output. + +| | pilot-01 | pilot-02 | pilot-03 | +|---|---|---|---| +| recall | 0.60 (3/5) | 0.33 (1/3) | 1.00 (2/2) | +| escape rate | 0.20 (1/5) | 0.67 (2/3) | 0.00 (0/2) | +| M2 slope | 0.0239/turn (n=10) | 0.0000/turn (n=9) | 0.0014/turn (n=9) | +| iterations median | 1 over 12 reviewed | 1 over 12 reviewed | 1 over 10 reviewed | +| cost | $0.0236 | $0.0125 | $0.0136 | + +## Metrics after correction + +These values come from a fresh offline re-harvest with the corrected source harvester. All three +commands exited successfully. The ownership filter emitted 11 `EARLIER turn` warnings for +pilot-01, 11 for pilot-02 and 9 for pilot-03, confirming that it engaged on every inherited +report identified in the defect analysis. + +| | pilot-01 | pilot-02 | pilot-03 | +|---|---|---|---| +| recall | 0.60 (3/5) | 0.33 (1/3) | 1.00 (2/2) | +| escape rate | 0.20 (1/5) | 0.67 (2/3) | 0.00 (0/2) | +| M2 slope | 0.0371/turn (n=7) | 0.0000/turn (n=5) | 0.0014/turn (n=7) | +| iterations median | 1 over 12 reviewed | 1 over 12 reviewed | 1 over 10 reviewed | +| cost | $0.0236 | $0.0125 | $0.0136 | + +## What moved and what did not + +- **Recall did not move** for any pilot: 0.60, 0.33 and 1.00 remain the reported rates and the + numerators and denominators are unchanged. +- **Escape rate did not move** for any pilot: 0.20, 0.67 and 0.00 remain unchanged, including the + numerators and denominators. +- **M2 moved only for pilot-01 at the reported precision:** its slope rose from 0.0239/turn to + 0.0371/turn, while its valid sample fell from n=10 to n=7. Pilot-02 remains 0.0000/turn, with + n reduced from 9 to 5. Pilot-03 remains 0.0014/turn at four decimals, with n reduced from 9 to + 7. +- **Median iterations did not move** for any pilot: each remains 1, over 12, 12 and 10 reviewed + turns respectively. +- **Cost did not move** for any pilot: the rounded totals remain $0.0236, $0.0125 and $0.0136. + +## Reproducibility boundary + +The raw pilot evidence under `rig/results/` is gitignored, so this re-harvest is reproducible only +on the machine that retains those artifacts. The correction rests on the report-ownership fix in +harvester commit `825662e`. The fresh measurements were run from descendant `cce7ed2`, which also +contains the associated guards. diff --git a/docs/superpowers/plans/2026-07-29-longitudinal-effectiveness-rig.md b/docs/superpowers/plans/2026-07-29-longitudinal-effectiveness-rig.md index 42a82bc..ca0f55c 100644 --- a/docs/superpowers/plans/2026-07-29-longitudinal-effectiveness-rig.md +++ b/docs/superpowers/plans/2026-07-29-longitudinal-effectiveness-rig.md @@ -984,6 +984,10 @@ NOT the mechanism for any counterfactual, and it must never be given an `--ablat ### Task 6: Preregistration, the pilot run, and the honest write-up ✅ DONE 2026-08-05 +> **Correction (2026-08-10):** The stale-report defect superseded pilot-01's M2 value and sample +> size below. See `docs/dev/2026-08-07-rig-stale-report-correction.md` for the preserved +> before/after record. + Ran on the fourth attempt — 12/12 turns, **zero unreviewed**, 61 min wall clock, $0.0166 billed. Result `rig/results/pilot-01/result.json` + 40-entry cassette; write-up `docs/dev/2026-08-05-pilot-01-result.md`. The three earlier attempts (0 audit events, cassette diff --git a/docs/superpowers/specs/2026-08-05-fp-ledger-evidence-unit-design.md b/docs/superpowers/specs/2026-08-05-fp-ledger-evidence-unit-design.md index a1286d7..f611a77 100644 --- a/docs/superpowers/specs/2026-08-05-fp-ledger-evidence-unit-design.md +++ b/docs/superpowers/specs/2026-08-05-fp-ledger-evidence-unit-design.md @@ -3,6 +3,10 @@ _2026-08-05. Follows `docs/dev/2026-08-05-pilot-01-result.md` (pilot-01 baseline)._ _Task (a) from `NEXT_SESSION.md`, re-scoped after the measurement below contradicted its premise._ +> **Correction (2026-08-10):** The stale-report defect superseded the pilot-01 M2 value and +> sample size cited as context below. See +> `docs/dev/2026-08-07-rig-stale-report-correction.md` for the preserved before/after record. + ## Why this exists Pilot-01 reported M6 suppression `critic 0 · reputation 0 · fp-ledger 0` and an M2 FP-burden From 9bc72c1b09250f9c6c8ba80e37f5c6e737080b72 Mon Sep 17 00:00:00 2001 From: Codevena Date: Mon, 10 Aug 2026 01:45:51 +0200 Subject: [PATCH 21/55] docs: refresh trailhead after rig reliability fix --- CLAUDE.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 089e435..e04554e 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -1,6 +1,6 @@ --- -verified_commit: cef7022 -verified_at: 2026-08-08 +verified_commit: eb602b3 +verified_at: 2026-08-10 --- # reviewgate — Trailhead From 250fadd2a7ffcc01fd6ebc81eb386b92c821e582 Mon Sep 17 00:00:00 2001 From: Codevena Date: Mon, 10 Aug 2026 03:22:33 +0200 Subject: [PATCH 22/55] feat(policy): define trace catalog and schemas --- src/core/policy/catalog.ts | 1161 ++++++++++++++++++++++++ src/schemas/audit-event.ts | 42 +- src/schemas/finding.ts | 4 + src/schemas/pending-report.ts | 3 + src/schemas/policy-trace.ts | 818 +++++++++++++++++ tests/unit/finding-schema.test.ts | 48 + tests/unit/pending-report.test.ts | 100 ++ tests/unit/policy-catalog.test.ts | 249 +++++ tests/unit/policy-trace-schema.test.ts | 1000 ++++++++++++++++++++ tests/unit/run-summary-schema.test.ts | 40 + 10 files changed, 3464 insertions(+), 1 deletion(-) create mode 100644 src/core/policy/catalog.ts create mode 100644 src/schemas/policy-trace.ts create mode 100644 tests/unit/policy-catalog.test.ts create mode 100644 tests/unit/policy-trace-schema.test.ts diff --git a/src/core/policy/catalog.ts b/src/core/policy/catalog.ts new file mode 100644 index 0000000..3dc8ae0 --- /dev/null +++ b/src/core/policy/catalog.ts @@ -0,0 +1,1161 @@ +export const POLICY_CATALOG_VERSION = "reviewgate.policy-catalog.v1" as const; + +export const POLICY_PASS_IDS = [ + "evidence.fact-location", + "evidence.self-refutation", + "judgment.hypothetical", + "evidence.grounding-token", + "judgment.grounding-llm", + "evidence.redaction-placeholder", + "judgment.critic", + "scope.diff", + "scope.delta", + "scope.session", + "history.fp-signature", + "history.cycle-rejected", + "history.fp-cluster", + "judgment.confidence", + "judgment.reputation", + "history.region-rejected", + "judgment.test-security", + "judgment.docs-cap", +] as const; + +export const POLICY_STAGE_IDS = ["aggregation.cluster", "verdict.compute"] as const; + +export const POLICY_EFFECT_ACTIONS = [ + "demoted", + "capped", + "dropped", + "protected", + "suppressed", + "reanchored", +] as const; + +export const POLICY_REASON_CODES = [ + "ineligible-starting-state", + "predicate-miss", + "configured-off", + "stage-precondition-miss", + "location-out-of-range", + "evidence-line-reanchored", + "terminal-self-refutation", + "hypothetical-critical", + "cited-token-absent", + "judge-ungrounded", + "placeholder-code-hallucination", + "critic-likely-fp", + "outside-changed-file", + "outside-changed-lines", + "preexisting-harness-config", + "outside-delta-scope", + "foreign-to-session", + "active-fp-signature", + "cycle-signature-rejected", + "active-fp-cluster", + "below-confidence-floor", + "unreliable-reviewer", + "rejected-region-overlap", + "test-only-security", + "docs-critical-cap", + "singleton", + "clustered", + "hard-critical", + "corroborated-warn", + "claimed-fixed-recurrence", + "blocking-present", + "no-blocking-findings", +] as const; + +export const POLICY_PROTECTION_CODES = [ + "claimed-fixed-pin", + "security-correctness-floor", + "deterministic-ground-truth", + "secret-evidence-backstop", + "self-refutation-visibility", + "corroborated-majority", + "corroborated-unanimous", + "high-precision-reviewer", + "out-of-diff-blocking-hatch", + "critical-floor", + "security-floor", + "correctness-demote-disabled", + "insufficient-distinct-rejections", + "category-change", + "severity-increase", + "mixed-category-cluster", +] as const; + +export type PolicyPassId = (typeof POLICY_PASS_IDS)[number]; +export type PolicyStageId = (typeof POLICY_STAGE_IDS)[number]; +export type PolicyCatalogId = PolicyPassId | PolicyStageId; +export type PolicyEffectAction = (typeof POLICY_EFFECT_ACTIONS)[number]; +export type PolicyReasonCode = (typeof POLICY_REASON_CODES)[number]; +export type PolicyProtectionCode = (typeof POLICY_PROTECTION_CODES)[number]; +export type PolicyPassClass = "evidence" | "value-judgment" | "scope" | "history"; +export type PolicySeverity = "CRITICAL" | "WARN" | "INFO"; + +export interface PolicyMaterialTransition { + readonly reason_code: PolicyReasonCode; + readonly action: Exclude; + readonly before: PolicySeverity; + readonly after: PolicySeverity | null; +} + +export interface PolicyProtectionRule { + readonly reason_code: PolicyReasonCode; + readonly protected_by: PolicyProtectionCode; + readonly before: PolicySeverity; +} + +export interface PolicyPassCatalogEntry { + readonly id: PolicyPassId; + readonly order: number; + readonly class: PolicyPassClass; + readonly actions: readonly PolicyEffectAction[]; + readonly reason_codes: readonly PolicyReasonCode[]; + readonly protection_codes: readonly PolicyProtectionCode[]; + readonly material_transitions: readonly PolicyMaterialTransition[]; + readonly protection_rules: readonly PolicyProtectionRule[]; + readonly opportunity: string; + readonly depends_on: readonly PolicyCatalogId[]; + readonly overlaps_with: readonly PolicyPassId[]; + readonly ablatable: true; + readonly slice_2_metric: string; +} + +export interface PolicyStageCatalogEntry { + readonly id: PolicyStageId; + readonly order: number; + readonly reason_codes: readonly PolicyReasonCode[]; + readonly depends_on: readonly PolicyCatalogId[]; + readonly ablatable: false; +} + +const COMMON_REASONS = [ + "ineligible-starting-state", + "predicate-miss", + "configured-off", + "stage-precondition-miss", +] as const satisfies readonly PolicyReasonCode[]; + +export const POLICY_PASSES = [ + { + id: "evidence.fact-location", + order: 10, + class: "evidence", + actions: ["demoted", "reanchored"], + reason_codes: [...COMMON_REASONS, "location-out-of-range", "evidence-line-reanchored"], + protection_codes: [], + material_transitions: [ + { + reason_code: "location-out-of-range", + action: "demoted", + before: "CRITICAL", + after: "INFO", + }, + { reason_code: "location-out-of-range", action: "demoted", before: "WARN", after: "INFO" }, + { + reason_code: "evidence-line-reanchored", + action: "reanchored", + before: "CRITICAL", + after: "CRITICAL", + }, + { + reason_code: "evidence-line-reanchored", + action: "reanchored", + before: "WARN", + after: "WARN", + }, + { + reason_code: "evidence-line-reanchored", + action: "reanchored", + before: "INFO", + after: "INFO", + }, + ], + protection_rules: [], + opportunity: "cited repo file is safely readable and the finding has a positive line", + depends_on: [], + overlaps_with: [ + "evidence.self-refutation", + "judgment.hypothetical", + "evidence.grounding-token", + "judgment.grounding-llm", + "evidence.redaction-placeholder", + ], + ablatable: true, + slice_2_metric: "re-anchor yield and blocking delta per opportunity", + }, + { + id: "evidence.self-refutation", + order: 20, + class: "evidence", + actions: ["demoted", "protected"], + reason_codes: [...COMMON_REASONS, "terminal-self-refutation"], + protection_codes: ["security-correctness-floor", "deterministic-ground-truth"], + material_transitions: [ + { + reason_code: "terminal-self-refutation", + action: "demoted", + before: "CRITICAL", + after: "INFO", + }, + { + reason_code: "terminal-self-refutation", + action: "demoted", + before: "WARN", + after: "INFO", + }, + ], + protection_rules: [ + { + reason_code: "terminal-self-refutation", + protected_by: "security-correctness-floor", + before: "CRITICAL", + }, + { + reason_code: "terminal-self-refutation", + protected_by: "security-correctness-floor", + before: "WARN", + }, + { + reason_code: "terminal-self-refutation", + protected_by: "deterministic-ground-truth", + before: "CRITICAL", + }, + { + reason_code: "terminal-self-refutation", + protected_by: "deterministic-ground-truth", + before: "WARN", + }, + ], + opportunity: "blocking, non-deterministic finding", + depends_on: ["evidence.fact-location"], + overlaps_with: [ + "evidence.fact-location", + "judgment.hypothetical", + "evidence.grounding-token", + "judgment.grounding-llm", + "evidence.redaction-placeholder", + ], + ablatable: true, + slice_2_metric: "blocking delta and disposition accuracy per opportunity", + }, + { + id: "judgment.hypothetical", + order: 30, + class: "value-judgment", + actions: ["demoted", "protected"], + reason_codes: [...COMMON_REASONS, "hypothetical-critical"], + protection_codes: ["security-correctness-floor", "deterministic-ground-truth"], + material_transitions: [ + { + reason_code: "hypothetical-critical", + action: "demoted", + before: "CRITICAL", + after: "WARN", + }, + ], + protection_rules: [ + { + reason_code: "hypothetical-critical", + protected_by: "security-correctness-floor", + before: "CRITICAL", + }, + { + reason_code: "hypothetical-critical", + protected_by: "deterministic-ground-truth", + before: "CRITICAL", + }, + ], + opportunity: "CRITICAL, non-deterministic finding", + depends_on: ["evidence.self-refutation"], + overlaps_with: [ + "evidence.fact-location", + "evidence.self-refutation", + "evidence.grounding-token", + "judgment.grounding-llm", + "evidence.redaction-placeholder", + ], + ablatable: true, + slice_2_metric: "severity delta and disposition accuracy per opportunity", + }, + { + id: "evidence.grounding-token", + order: 40, + class: "evidence", + actions: ["demoted", "protected"], + reason_codes: [...COMMON_REASONS, "cited-token-absent"], + protection_codes: ["security-correctness-floor"], + material_transitions: [ + { + reason_code: "cited-token-absent", + action: "demoted", + before: "CRITICAL", + after: "WARN", + }, + ], + protection_rules: [ + { + reason_code: "cited-token-absent", + protected_by: "security-correctness-floor", + before: "CRITICAL", + }, + ], + opportunity: "CRITICAL finding with at least one extractable token", + depends_on: ["judgment.hypothetical"], + overlaps_with: [ + "evidence.fact-location", + "evidence.self-refutation", + "judgment.hypothetical", + "judgment.grounding-llm", + "evidence.redaction-placeholder", + ], + ablatable: true, + slice_2_metric: "severity delta and disposition accuracy per opportunity", + }, + { + id: "judgment.grounding-llm", + order: 50, + class: "value-judgment", + actions: ["demoted", "protected"], + reason_codes: [...COMMON_REASONS, "judge-ungrounded"], + protection_codes: ["security-correctness-floor"], + material_transitions: [ + { + reason_code: "judge-ungrounded", + action: "demoted", + before: "CRITICAL", + after: "WARN", + }, + ], + protection_rules: [ + { + reason_code: "judge-ungrounded", + protected_by: "security-correctness-floor", + before: "CRITICAL", + }, + ], + opportunity: "CRITICAL finding with a judge verdict for its signature", + depends_on: ["evidence.grounding-token"], + overlaps_with: [ + "evidence.fact-location", + "evidence.self-refutation", + "judgment.hypothetical", + "evidence.grounding-token", + "evidence.redaction-placeholder", + ], + ablatable: true, + slice_2_metric: "severity delta and disposition accuracy per opportunity", + }, + { + id: "evidence.redaction-placeholder", + order: 60, + class: "evidence", + actions: ["demoted", "protected"], + reason_codes: [...COMMON_REASONS, "placeholder-code-hallucination"], + protection_codes: ["security-correctness-floor", "secret-evidence-backstop"], + material_transitions: [ + { + reason_code: "placeholder-code-hallucination", + action: "demoted", + before: "CRITICAL", + after: "INFO", + }, + { + reason_code: "placeholder-code-hallucination", + action: "demoted", + before: "WARN", + after: "INFO", + }, + ], + protection_rules: [ + { + reason_code: "placeholder-code-hallucination", + protected_by: "security-correctness-floor", + before: "CRITICAL", + }, + { + reason_code: "placeholder-code-hallucination", + protected_by: "security-correctness-floor", + before: "WARN", + }, + { + reason_code: "placeholder-code-hallucination", + protected_by: "secret-evidence-backstop", + before: "CRITICAL", + }, + { + reason_code: "placeholder-code-hallucination", + protected_by: "secret-evidence-backstop", + before: "WARN", + }, + ], + opportunity: "blocking finding whose subject contains a redaction placeholder", + depends_on: ["judgment.grounding-llm"], + overlaps_with: [ + "evidence.fact-location", + "evidence.self-refutation", + "judgment.hypothetical", + "evidence.grounding-token", + "judgment.grounding-llm", + ], + ablatable: true, + slice_2_metric: "blocking delta and disposition accuracy per opportunity", + }, + { + id: "judgment.critic", + order: 70, + class: "value-judgment", + actions: ["demoted", "dropped", "protected"], + reason_codes: [...COMMON_REASONS, "critic-likely-fp"], + protection_codes: [ + "claimed-fixed-pin", + "self-refutation-visibility", + "security-correctness-floor", + "corroborated-majority", + "corroborated-unanimous", + "high-precision-reviewer", + ], + material_transitions: [ + { + reason_code: "critic-likely-fp", + action: "demoted", + before: "CRITICAL", + after: "WARN", + }, + { + reason_code: "critic-likely-fp", + action: "demoted", + before: "WARN", + after: "INFO", + }, + { + reason_code: "critic-likely-fp", + action: "dropped", + before: "INFO", + after: null, + }, + ], + protection_rules: [ + { + reason_code: "critic-likely-fp", + protected_by: "claimed-fixed-pin", + before: "CRITICAL", + }, + { + reason_code: "critic-likely-fp", + protected_by: "claimed-fixed-pin", + before: "WARN", + }, + { + reason_code: "critic-likely-fp", + protected_by: "claimed-fixed-pin", + before: "INFO", + }, + { + reason_code: "critic-likely-fp", + protected_by: "self-refutation-visibility", + before: "INFO", + }, + { + reason_code: "critic-likely-fp", + protected_by: "security-correctness-floor", + before: "CRITICAL", + }, + { + reason_code: "critic-likely-fp", + protected_by: "corroborated-majority", + before: "CRITICAL", + }, + { + reason_code: "critic-likely-fp", + protected_by: "corroborated-majority", + before: "WARN", + }, + { + reason_code: "critic-likely-fp", + protected_by: "corroborated-majority", + before: "INFO", + }, + { + reason_code: "critic-likely-fp", + protected_by: "corroborated-unanimous", + before: "CRITICAL", + }, + { + reason_code: "critic-likely-fp", + protected_by: "corroborated-unanimous", + before: "WARN", + }, + { + reason_code: "critic-likely-fp", + protected_by: "corroborated-unanimous", + before: "INFO", + }, + { + reason_code: "critic-likely-fp", + protected_by: "high-precision-reviewer", + before: "CRITICAL", + }, + { + reason_code: "critic-likely-fp", + protected_by: "high-precision-reviewer", + before: "WARN", + }, + ], + opportunity: "critic emitted a verdict for representative or member signature", + depends_on: ["aggregation.cluster"], + overlaps_with: ["judgment.confidence", "judgment.reputation"], + ablatable: true, + slice_2_metric: "precision and recall delta per opportunity", + }, + { + id: "scope.diff", + order: 80, + class: "scope", + actions: ["demoted", "protected"], + reason_codes: [ + ...COMMON_REASONS, + "outside-changed-file", + "outside-changed-lines", + "preexisting-harness-config", + ], + protection_codes: ["out-of-diff-blocking-hatch"], + material_transitions: [ + { + reason_code: "outside-changed-file", + action: "demoted", + before: "CRITICAL", + after: "INFO", + }, + { + reason_code: "outside-changed-file", + action: "demoted", + before: "WARN", + after: "INFO", + }, + { + reason_code: "outside-changed-lines", + action: "demoted", + before: "CRITICAL", + after: "INFO", + }, + { + reason_code: "outside-changed-lines", + action: "demoted", + before: "WARN", + after: "INFO", + }, + { + reason_code: "preexisting-harness-config", + action: "demoted", + before: "CRITICAL", + after: "INFO", + }, + { + reason_code: "preexisting-harness-config", + action: "demoted", + before: "WARN", + after: "INFO", + }, + ], + protection_rules: [ + { + reason_code: "outside-changed-file", + protected_by: "out-of-diff-blocking-hatch", + before: "CRITICAL", + }, + { + reason_code: "outside-changed-file", + protected_by: "out-of-diff-blocking-hatch", + before: "WARN", + }, + { + reason_code: "outside-changed-lines", + protected_by: "out-of-diff-blocking-hatch", + before: "CRITICAL", + }, + { + reason_code: "outside-changed-lines", + protected_by: "out-of-diff-blocking-hatch", + before: "WARN", + }, + ], + opportunity: "blocking finding has a usable line while changed ranges exist", + depends_on: ["aggregation.cluster"], + overlaps_with: ["scope.delta", "scope.session"], + ablatable: true, + slice_2_metric: "blocking delta and disposition accuracy per opportunity", + }, + { + id: "scope.delta", + order: 90, + class: "scope", + actions: ["demoted", "protected"], + reason_codes: [...COMMON_REASONS, "outside-delta-scope"], + protection_codes: [ + "claimed-fixed-pin", + "security-correctness-floor", + "critical-floor", + "out-of-diff-blocking-hatch", + ], + material_transitions: [ + { + reason_code: "outside-delta-scope", + action: "demoted", + before: "CRITICAL", + after: "INFO", + }, + { + reason_code: "outside-delta-scope", + action: "demoted", + before: "WARN", + after: "INFO", + }, + ], + protection_rules: [ + { + reason_code: "outside-delta-scope", + protected_by: "claimed-fixed-pin", + before: "CRITICAL", + }, + { + reason_code: "outside-delta-scope", + protected_by: "claimed-fixed-pin", + before: "WARN", + }, + { + reason_code: "outside-delta-scope", + protected_by: "security-correctness-floor", + before: "CRITICAL", + }, + { + reason_code: "outside-delta-scope", + protected_by: "security-correctness-floor", + before: "WARN", + }, + { + reason_code: "outside-delta-scope", + protected_by: "critical-floor", + before: "CRITICAL", + }, + { + reason_code: "outside-delta-scope", + protected_by: "critical-floor", + before: "WARN", + }, + { + reason_code: "outside-delta-scope", + protected_by: "out-of-diff-blocking-hatch", + before: "CRITICAL", + }, + { + reason_code: "outside-delta-scope", + protected_by: "out-of-diff-blocking-hatch", + before: "WARN", + }, + ], + opportunity: "blocking finding while a delta scope exists", + depends_on: ["scope.diff"], + overlaps_with: ["scope.diff", "scope.session"], + ablatable: true, + slice_2_metric: "blocking delta and disposition accuracy per opportunity", + }, + { + id: "scope.session", + order: 100, + class: "scope", + actions: ["demoted", "protected"], + reason_codes: [...COMMON_REASONS, "foreign-to-session"], + protection_codes: ["out-of-diff-blocking-hatch"], + material_transitions: [ + { + reason_code: "foreign-to-session", + action: "demoted", + before: "CRITICAL", + after: "INFO", + }, + { + reason_code: "foreign-to-session", + action: "demoted", + before: "WARN", + after: "INFO", + }, + ], + protection_rules: [ + { + reason_code: "foreign-to-session", + protected_by: "out-of-diff-blocking-hatch", + before: "CRITICAL", + }, + { + reason_code: "foreign-to-session", + protected_by: "out-of-diff-blocking-hatch", + before: "WARN", + }, + ], + opportunity: "blocking finding while foreign-file facts exist", + depends_on: ["scope.delta"], + overlaps_with: ["scope.diff", "scope.delta"], + ablatable: true, + slice_2_metric: "blocking delta and disposition accuracy per opportunity", + }, + { + id: "history.fp-signature", + order: 110, + class: "history", + actions: ["suppressed"], + reason_codes: [...COMMON_REASONS, "active-fp-signature"], + protection_codes: [], + material_transitions: [ + { + reason_code: "active-fp-signature", + action: "suppressed", + before: "CRITICAL", + after: "INFO", + }, + { + reason_code: "active-fp-signature", + action: "suppressed", + before: "WARN", + after: "INFO", + }, + ], + protection_rules: [], + opportunity: "blocking finding while an active signature snapshot exists", + depends_on: ["scope.session"], + overlaps_with: ["history.cycle-rejected", "history.fp-cluster", "history.region-rejected"], + ablatable: true, + slice_2_metric: "state-conditioned precision and recall delta per opportunity", + }, + { + id: "history.cycle-rejected", + order: 120, + class: "history", + actions: ["suppressed", "protected"], + reason_codes: [...COMMON_REASONS, "cycle-signature-rejected"], + protection_codes: ["critical-floor", "security-correctness-floor"], + material_transitions: [ + { + reason_code: "cycle-signature-rejected", + action: "suppressed", + before: "WARN", + after: "INFO", + }, + ], + protection_rules: [ + { + reason_code: "cycle-signature-rejected", + protected_by: "critical-floor", + before: "CRITICAL", + }, + { + reason_code: "cycle-signature-rejected", + protected_by: "security-correctness-floor", + before: "CRITICAL", + }, + { + reason_code: "cycle-signature-rejected", + protected_by: "security-correctness-floor", + before: "WARN", + }, + ], + opportunity: "blocking finding while rejected signatures exist", + depends_on: ["history.fp-signature"], + overlaps_with: ["history.fp-signature", "history.fp-cluster", "history.region-rejected"], + ablatable: true, + slice_2_metric: "state-conditioned precision and recall delta per opportunity", + }, + { + id: "history.fp-cluster", + order: 130, + class: "history", + actions: ["suppressed"], + reason_codes: [...COMMON_REASONS, "active-fp-cluster"], + protection_codes: [], + material_transitions: [ + { + reason_code: "active-fp-cluster", + action: "suppressed", + before: "CRITICAL", + after: "INFO", + }, + { + reason_code: "active-fp-cluster", + action: "suppressed", + before: "WARN", + after: "INFO", + }, + ], + protection_rules: [], + opportunity: "blocking finding while active cluster keys exist", + depends_on: ["history.cycle-rejected"], + overlaps_with: ["history.fp-signature", "history.cycle-rejected", "history.region-rejected"], + ablatable: true, + slice_2_metric: "state-conditioned precision and recall delta per opportunity", + }, + { + id: "judgment.confidence", + order: 140, + class: "value-judgment", + actions: ["demoted", "capped", "protected"], + reason_codes: [...COMMON_REASONS, "below-confidence-floor"], + protection_codes: [ + "claimed-fixed-pin", + "security-correctness-floor", + "corroborated-majority", + "corroborated-unanimous", + "high-precision-reviewer", + ], + material_transitions: [ + { + reason_code: "below-confidence-floor", + action: "capped", + before: "CRITICAL", + after: "WARN", + }, + { + reason_code: "below-confidence-floor", + action: "demoted", + before: "WARN", + after: "INFO", + }, + ], + protection_rules: [ + { + reason_code: "below-confidence-floor", + protected_by: "claimed-fixed-pin", + before: "CRITICAL", + }, + { + reason_code: "below-confidence-floor", + protected_by: "claimed-fixed-pin", + before: "WARN", + }, + { + reason_code: "below-confidence-floor", + protected_by: "security-correctness-floor", + before: "CRITICAL", + }, + { + reason_code: "below-confidence-floor", + protected_by: "corroborated-majority", + before: "CRITICAL", + }, + { + reason_code: "below-confidence-floor", + protected_by: "corroborated-majority", + before: "WARN", + }, + { + reason_code: "below-confidence-floor", + protected_by: "corroborated-unanimous", + before: "CRITICAL", + }, + { + reason_code: "below-confidence-floor", + protected_by: "corroborated-unanimous", + before: "WARN", + }, + { + reason_code: "below-confidence-floor", + protected_by: "high-precision-reviewer", + before: "CRITICAL", + }, + { + reason_code: "below-confidence-floor", + protected_by: "high-precision-reviewer", + before: "WARN", + }, + ], + opportunity: "blocking, uncorroborated finding while the confidence floor is positive", + depends_on: ["history.fp-cluster"], + overlaps_with: ["judgment.critic", "judgment.reputation"], + ablatable: true, + slice_2_metric: "precision and recall delta per opportunity", + }, + { + id: "judgment.reputation", + order: 150, + class: "value-judgment", + actions: ["demoted", "capped", "protected"], + reason_codes: [...COMMON_REASONS, "unreliable-reviewer"], + protection_codes: [ + "claimed-fixed-pin", + "security-floor", + "correctness-demote-disabled", + "corroborated-majority", + "corroborated-unanimous", + "critical-floor", + ], + material_transitions: [ + { + reason_code: "unreliable-reviewer", + action: "demoted", + before: "CRITICAL", + after: "WARN", + }, + { + reason_code: "unreliable-reviewer", + action: "capped", + before: "CRITICAL", + after: "WARN", + }, + { + reason_code: "unreliable-reviewer", + action: "demoted", + before: "WARN", + after: "INFO", + }, + ], + protection_rules: [ + { + reason_code: "unreliable-reviewer", + protected_by: "claimed-fixed-pin", + before: "CRITICAL", + }, + { + reason_code: "unreliable-reviewer", + protected_by: "claimed-fixed-pin", + before: "WARN", + }, + { + reason_code: "unreliable-reviewer", + protected_by: "security-floor", + before: "CRITICAL", + }, + { + reason_code: "unreliable-reviewer", + protected_by: "security-floor", + before: "WARN", + }, + { + reason_code: "unreliable-reviewer", + protected_by: "correctness-demote-disabled", + before: "CRITICAL", + }, + { + reason_code: "unreliable-reviewer", + protected_by: "correctness-demote-disabled", + before: "WARN", + }, + { + reason_code: "unreliable-reviewer", + protected_by: "corroborated-majority", + before: "CRITICAL", + }, + { + reason_code: "unreliable-reviewer", + protected_by: "corroborated-majority", + before: "WARN", + }, + { + reason_code: "unreliable-reviewer", + protected_by: "corroborated-unanimous", + before: "CRITICAL", + }, + { + reason_code: "unreliable-reviewer", + protected_by: "corroborated-unanimous", + before: "WARN", + }, + { + reason_code: "unreliable-reviewer", + protected_by: "critical-floor", + before: "CRITICAL", + }, + { + reason_code: "unreliable-reviewer", + protected_by: "critical-floor", + before: "WARN", + }, + ], + opportunity: "blocking, uncorroborated finding while unreliable reviewers exist", + depends_on: ["judgment.confidence"], + overlaps_with: ["judgment.critic", "judgment.confidence"], + ablatable: true, + slice_2_metric: "state-conditioned precision and recall delta per opportunity", + }, + { + id: "history.region-rejected", + order: 160, + class: "history", + actions: ["suppressed", "protected"], + reason_codes: [...COMMON_REASONS, "rejected-region-overlap"], + protection_codes: [ + "claimed-fixed-pin", + "insufficient-distinct-rejections", + "category-change", + "severity-increase", + "critical-floor", + "security-correctness-floor", + ], + material_transitions: [ + { + reason_code: "rejected-region-overlap", + action: "suppressed", + before: "WARN", + after: "INFO", + }, + ], + protection_rules: [ + { + reason_code: "rejected-region-overlap", + protected_by: "claimed-fixed-pin", + before: "CRITICAL", + }, + { + reason_code: "rejected-region-overlap", + protected_by: "claimed-fixed-pin", + before: "WARN", + }, + { + reason_code: "rejected-region-overlap", + protected_by: "insufficient-distinct-rejections", + before: "CRITICAL", + }, + { + reason_code: "rejected-region-overlap", + protected_by: "insufficient-distinct-rejections", + before: "WARN", + }, + { + reason_code: "rejected-region-overlap", + protected_by: "category-change", + before: "CRITICAL", + }, + { + reason_code: "rejected-region-overlap", + protected_by: "category-change", + before: "WARN", + }, + { + reason_code: "rejected-region-overlap", + protected_by: "severity-increase", + before: "CRITICAL", + }, + { + reason_code: "rejected-region-overlap", + protected_by: "severity-increase", + before: "WARN", + }, + { + reason_code: "rejected-region-overlap", + protected_by: "critical-floor", + before: "CRITICAL", + }, + { + reason_code: "rejected-region-overlap", + protected_by: "critical-floor", + before: "WARN", + }, + { + reason_code: "rejected-region-overlap", + protected_by: "security-correctness-floor", + before: "CRITICAL", + }, + { + reason_code: "rejected-region-overlap", + protected_by: "security-correctness-floor", + before: "WARN", + }, + ], + opportunity: "blocking finding with a usable line while rejected regions exist", + depends_on: ["judgment.reputation"], + overlaps_with: ["history.fp-signature", "history.cycle-rejected", "history.fp-cluster"], + ablatable: true, + slice_2_metric: "state-conditioned precision and recall delta per opportunity", + }, + { + id: "judgment.test-security", + order: 170, + class: "value-judgment", + actions: ["demoted", "protected"], + reason_codes: [...COMMON_REASONS, "test-only-security"], + protection_codes: ["mixed-category-cluster"], + material_transitions: [ + { + reason_code: "test-only-security", + action: "demoted", + before: "CRITICAL", + after: "INFO", + }, + { + reason_code: "test-only-security", + action: "demoted", + before: "WARN", + after: "INFO", + }, + ], + protection_rules: [ + { + reason_code: "test-only-security", + protected_by: "mixed-category-cluster", + before: "CRITICAL", + }, + { + reason_code: "test-only-security", + protected_by: "mixed-category-cluster", + before: "WARN", + }, + ], + opportunity: "blocking finding in a classified test or fixture file", + depends_on: ["history.region-rejected"], + overlaps_with: ["judgment.docs-cap"], + ablatable: true, + slice_2_metric: "blocking delta and disposition accuracy per opportunity", + }, + { + id: "judgment.docs-cap", + order: 180, + class: "value-judgment", + actions: ["capped", "protected"], + reason_codes: [...COMMON_REASONS, "docs-critical-cap"], + protection_codes: ["security-correctness-floor"], + material_transitions: [ + { + reason_code: "docs-critical-cap", + action: "capped", + before: "CRITICAL", + after: "WARN", + }, + ], + protection_rules: [ + { + reason_code: "docs-critical-cap", + protected_by: "security-correctness-floor", + before: "CRITICAL", + }, + ], + opportunity: "CRITICAL finding in a classified docs file", + depends_on: ["judgment.test-security"], + overlaps_with: ["judgment.test-security"], + ablatable: true, + slice_2_metric: "severity delta and disposition accuracy per opportunity", + }, +] as const satisfies readonly PolicyPassCatalogEntry[]; + +export const POLICY_STAGES = [ + { + id: "aggregation.cluster", + order: 65, + reason_codes: ["singleton", "clustered"], + depends_on: ["evidence.redaction-placeholder"], + ablatable: false, + }, + { + id: "verdict.compute", + order: 190, + reason_codes: [ + "hard-critical", + "corroborated-warn", + "claimed-fixed-recurrence", + "blocking-present", + "no-blocking-findings", + ], + depends_on: ["judgment.docs-cap"], + ablatable: false, + }, +] as const satisfies readonly PolicyStageCatalogEntry[]; diff --git a/src/schemas/audit-event.ts b/src/schemas/audit-event.ts index 9efdc04..570b5d2 100644 --- a/src/schemas/audit-event.ts +++ b/src/schemas/audit-event.ts @@ -1,6 +1,7 @@ // src/schemas/audit-event.ts import { z } from "zod"; import { Severity } from "./finding.ts"; +import { PolicySha256Schema, PolicyTraceStatusSchema } from "./policy-trace.ts"; export const EventType = z.enum([ "session.start", @@ -90,7 +91,7 @@ export const ProviderStatSchema = z.object({ duration_ms: z.number().int().nonnegative(), }); -export const RunSummarySchema = z.object({ +const RunSummaryObjectSchema = z.object({ verdict: z.enum(["PASS", "SOFT-PASS", "FAIL", "ERROR"]), // "content-cache" (T5/R3, field report 2026-07-03): PASS served because every diff // file is byte-identical to the pass_ledger (content that already passed a clean @@ -122,6 +123,45 @@ export const RunSummarySchema = z.object({ // demoted CRITICAL→WARN this run. Pure observability (feeds the decision whether to // extend the clamp with audit-precision evidence later); never read by gating logic. corroboration_clamped: z.number().int().nonnegative().optional(), + // Policy traces are additive to the existing reviewgate.audit.v1 event. A complete + // status is content-addressed; every non-complete status deliberately has no artifact. + policy_trace_status: PolicyTraceStatusSchema.optional(), + policy_trace_ref: z.string().min(1).optional(), + policy_trace_sha256: PolicySha256Schema.optional(), +}); +export const RunSummarySchema = RunSummaryObjectSchema.superRefine((summary, ctx) => { + if (summary.policy_trace_status === "complete") { + if (summary.policy_trace_ref === undefined) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + path: ["policy_trace_ref"], + message: "complete requires policy_trace_ref", + }); + } + if (summary.policy_trace_sha256 === undefined) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + path: ["policy_trace_sha256"], + message: "complete requires policy_trace_sha256", + }); + } + return; + } + + if (summary.policy_trace_ref !== undefined) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + path: ["policy_trace_ref"], + message: "policy_trace_ref is only valid for complete", + }); + } + if (summary.policy_trace_sha256 !== undefined) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + path: ["policy_trace_sha256"], + message: "policy_trace_sha256 is only valid for complete", + }); + } }); export type RunSummary = z.infer; export type ProviderStat = z.infer; diff --git a/src/schemas/finding.ts b/src/schemas/finding.ts index 44d1fb8..73e5204 100644 --- a/src/schemas/finding.ts +++ b/src/schemas/finding.ts @@ -1,4 +1,5 @@ import { z } from "zod"; +import { PolicyEffectsSchema } from "./policy-trace.ts"; export const Severity = z.enum(["CRITICAL", "WARN", "INFO"]); export type Severity = z.infer; @@ -76,6 +77,9 @@ export const FindingSchema = z.object({ consensus: Consensus, critic_verdict: z.enum(["keep", "likely_fp"]).optional(), critic_reason: z.string().optional(), + // Policy Accountability Slice 1: compact, server-authored material effects only. + // Reviewer output cannot supply this field because REVIEW_OUTPUT_SCHEMA remains closed. + policy_effects: PolicyEffectsSchema.optional(), // M5 Part A: set true when the aggregator demoted this finding to INFO because // its range falls outside the changed hunks (advisory, non-blocking). scope_demoted: z.boolean().optional(), diff --git a/src/schemas/pending-report.ts b/src/schemas/pending-report.ts index 8bfa465..fdddf3f 100644 --- a/src/schemas/pending-report.ts +++ b/src/schemas/pending-report.ts @@ -1,5 +1,6 @@ import { z } from "zod"; import { FindingSchema } from "./finding.ts"; +import { PolicySummarySchema } from "./policy-trace.ts"; export const ReviewerStatus = z.enum(["ok", "error", "abstain", "timeout", "quota-exhausted"]); export type ReviewerStatus = z.infer; @@ -69,6 +70,8 @@ export const PendingReportSchema = z.object({ }), ), findings: z.array(FindingSchema), + // Optional as one unit so legacy reviewgate.pending.v1 artifacts remain valid. + policy_summary: PolicySummarySchema.optional(), // S2 (field report 2026-06-23): true iff the reviewed diff contains at least one file this // session is responsible for (the attributable ∩ diff set is non-empty). The load-bearing // whole-diff guard for the out-of-session honest handoff: the agent may disown the change-set diff --git a/src/schemas/policy-trace.ts b/src/schemas/policy-trace.ts new file mode 100644 index 0000000..0325c4b --- /dev/null +++ b/src/schemas/policy-trace.ts @@ -0,0 +1,818 @@ +import { z } from "zod"; +import { + POLICY_CATALOG_VERSION, + POLICY_EFFECT_ACTIONS, + POLICY_PASSES, + POLICY_PASS_IDS, + POLICY_PROTECTION_CODES, + POLICY_REASON_CODES, + POLICY_STAGES, + POLICY_STAGE_IDS, + type PolicyPassCatalogEntry, + type PolicyPassId, + type PolicySeverity, + type PolicyStageCatalogEntry, +} from "../core/policy/catalog.ts"; + +export const PolicyPassIdSchema = z.enum(POLICY_PASS_IDS); +export const PolicyStageIdSchema = z.enum(POLICY_STAGE_IDS); +export const PolicyReasonCodeSchema = z.enum(POLICY_REASON_CODES); +export const PolicyProtectionCodeSchema = z.enum(POLICY_PROTECTION_CODES); +export const PolicyEffectActionSchema = z.enum(POLICY_EFFECT_ACTIONS); +export const PolicyTraceStatusSchema = z.enum(["complete", "not-run", "error", "overflow"]); +export const PolicySha256Schema = z.string().regex(/^[0-9a-f]{64}$/); + +const PolicySeveritySchema = z.enum(["CRITICAL", "WARN", "INFO"]); +const EvaluationResultSchema = z.enum([ + "no-opportunity", + "no-match", + "would-apply", + "protected", + "applied", +]); +const TraceVerdictSchema = z.enum(["PASS", "SOFT-PASS", "FAIL", "ERROR"]); +const StageVerdictSchema = z.enum(["PASS", "SOFT-PASS", "FAIL"]); + +const INACTIVE_REASON_CODES = new Set(["configured-off", "stage-precondition-miss"]); +const NON_MATERIAL_REASON_CODES = new Set([ + "ineligible-starting-state", + "predicate-miss", + "configured-off", + "stage-precondition-miss", +]); +const VERDICT_BY_REASON = { + "hard-critical": "FAIL", + "corroborated-warn": "FAIL", + "claimed-fixed-recurrence": "FAIL", + "blocking-present": "SOFT-PASS", + "no-blocking-findings": "PASS", +} as const; + +function policyPass(passId: PolicyPassId): PolicyPassCatalogEntry { + const pass: PolicyPassCatalogEntry | undefined = POLICY_PASSES.find( + (candidate) => candidate.id === passId, + ); + if (!pass) throw new Error(`Policy pass is missing from the static catalog: ${passId}`); + return pass; +} + +function policyStage(stageId: z.infer): PolicyStageCatalogEntry { + const stage: PolicyStageCatalogEntry | undefined = POLICY_STAGES.find( + (candidate) => candidate.id === stageId, + ); + if (!stage) throw new Error(`Policy stage is missing from the static catalog: ${stageId}`); + return stage; +} + +function addIssue(ctx: z.RefinementCtx, path: Array, message: string): void { + ctx.addIssue({ code: z.ZodIssueCode.custom, path, message }); +} + +function isSortedUnique(values: readonly string[]): boolean { + for (let index = 1; index < values.length; index += 1) { + const previous = values[index - 1]; + const current = values[index]; + if (previous === undefined || current === undefined || previous >= current) return false; + } + return true; +} + +function validateSortedUnique( + values: readonly string[], + ctx: z.RefinementCtx, + path: Array, +): void { + if (!isSortedUnique(values)) { + addIssue(ctx, path, "signatures must be sorted in ascending byte order and deduplicated"); + } +} + +function isOrderedSubsequence(values: readonly string[], candidates: readonly string[]): boolean { + let valueIndex = 0; + for (const candidate of candidates) { + if (candidate === values[valueIndex]) valueIndex += 1; + } + return valueIndex === values.length; +} + +function passAllowsReason(pass: PolicyPassCatalogEntry, reason: string): boolean { + return pass.reason_codes.some((candidate) => candidate === reason); +} + +function passAllowsAction(pass: PolicyPassCatalogEntry, action: string): boolean { + return pass.actions.some((candidate) => candidate === action); +} + +function passAllowsProtectionRule( + pass: PolicyPassCatalogEntry, + rule: { + reason_code: string; + protected_by: string; + before: PolicySeverity; + }, +): boolean { + return pass.protection_rules.some( + (candidate) => + candidate.reason_code === rule.reason_code && + candidate.protected_by === rule.protected_by && + candidate.before === rule.before, + ); +} + +function passAllowsMaterialTransition( + pass: PolicyPassCatalogEntry, + transition: { + reason_code: string; + action?: string; + before: PolicySeverity; + after?: PolicySeverity | null; + }, +): boolean { + return pass.material_transitions.some( + (candidate) => + candidate.reason_code === transition.reason_code && + (transition.action === undefined || candidate.action === transition.action) && + candidate.before === transition.before && + (transition.after === undefined || candidate.after === transition.after), + ); +} + +const NonEmptySortedSignaturesSchema = z + .array(z.string().min(1)) + .min(1) + .superRefine((values, ctx) => validateSortedUnique(values, ctx, [])); + +const UniqueSignaturesSchema = z.array(z.string().min(1)).superRefine((values, ctx) => { + const seen = new Set(); + for (const [index, value] of values.entries()) { + if (seen.has(value)) addIssue(ctx, [index], "signatures must be deduplicated"); + seen.add(value); + } +}); + +const PolicyEffectObjectSchema = z + .object({ + pass_id: PolicyPassIdSchema, + order: z.number().int().positive(), + action: PolicyEffectActionSchema, + before: PolicySeveritySchema, + after: PolicySeveritySchema.nullable(), + reason_code: PolicyReasonCodeSchema, + protected_by: PolicyProtectionCodeSchema.optional(), + source_signatures: NonEmptySortedSignaturesSchema, + }) + .strict(); + +export const PolicyEffectSchema = PolicyEffectObjectSchema.superRefine((effect, ctx) => { + const pass = policyPass(effect.pass_id); + + if (effect.order !== pass.order) { + addIssue(ctx, ["order"], `order must match ${effect.pass_id} (${pass.order})`); + } + if (!passAllowsAction(pass, effect.action)) { + addIssue(ctx, ["action"], `action is not allowed for ${effect.pass_id}`); + } + if ( + !passAllowsReason(pass, effect.reason_code) || + NON_MATERIAL_REASON_CODES.has(effect.reason_code) + ) { + addIssue(ctx, ["reason_code"], `material reason is not allowed for ${effect.pass_id}`); + } + + if (effect.action === "protected") { + if (effect.protected_by === undefined) { + addIssue(ctx, ["protected_by"], "a protected effect requires protected_by"); + } else if ( + !passAllowsProtectionRule(pass, { + reason_code: effect.reason_code, + protected_by: effect.protected_by, + before: effect.before, + }) + ) { + addIssue( + ctx, + ["protected_by"], + `protection, reason, and severity are not allowed for ${effect.pass_id}`, + ); + } + if (effect.after !== effect.before) { + addIssue(ctx, ["after"], "a protected effect must preserve severity"); + } + } else if (effect.protected_by !== undefined) { + addIssue(ctx, ["protected_by"], "protected_by is only valid for a protected effect"); + } + + if (effect.action === "protected") { + return; + } + if (!passAllowsMaterialTransition(pass, effect)) { + addIssue(ctx, ["after"], `transition is not allowed for ${effect.pass_id}`); + } +}); + +export type PolicyEffect = z.infer; + +export const PolicyEffectsSchema = z.array(PolicyEffectSchema).superRefine((effects, ctx) => { + let priorOrder = -1; + const identities = new Set(); + + for (const [index, effect] of effects.entries()) { + if (effect.order < priorOrder) { + addIssue(ctx, [index, "order"], "policy effects must remain in catalog order"); + } + priorOrder = effect.order; + + const identity = JSON.stringify([ + effect.pass_id, + effect.action, + effect.before, + effect.after, + effect.reason_code, + effect.protected_by ?? null, + effect.source_signatures, + ]); + if (identities.has(identity)) { + addIssue(ctx, [index], "duplicate idempotent policy effect"); + } + identities.add(identity); + } +}); + +const PolicyEvaluationObjectSchema = z + .object({ + pass_id: PolicyPassIdSchema, + order: z.number().int().positive(), + result: EvaluationResultSchema, + before: PolicySeveritySchema, + after: PolicySeveritySchema.nullable(), + reason_code: PolicyReasonCodeSchema, + protected_by: PolicyProtectionCodeSchema.optional(), + source_signatures: NonEmptySortedSignaturesSchema, + final_signature: z.string().min(1).optional(), + }) + .strict(); + +export const PolicyEvaluationSchema = PolicyEvaluationObjectSchema.superRefine( + (evaluation, ctx) => { + const pass = policyPass(evaluation.pass_id); + + if (evaluation.order !== pass.order) { + addIssue(ctx, ["order"], `order must match ${evaluation.pass_id} (${pass.order})`); + } + if (!passAllowsReason(pass, evaluation.reason_code)) { + addIssue(ctx, ["reason_code"], `reason is not allowed for ${evaluation.pass_id}`); + } + + if ( + (evaluation.result === "no-opportunity" && + evaluation.reason_code !== "ineligible-starting-state") || + (evaluation.result === "no-match" && evaluation.reason_code !== "predicate-miss") || + ((evaluation.result === "would-apply" || + evaluation.result === "protected" || + evaluation.result === "applied") && + NON_MATERIAL_REASON_CODES.has(evaluation.reason_code)) + ) { + addIssue(ctx, ["reason_code"], `reason does not match result ${evaluation.result}`); + } + + if (evaluation.result === "protected") { + if (evaluation.protected_by === undefined) { + addIssue(ctx, ["protected_by"], "a protected evaluation requires protected_by"); + } else if ( + !passAllowsProtectionRule(pass, { + reason_code: evaluation.reason_code, + protected_by: evaluation.protected_by, + before: evaluation.before, + }) + ) { + addIssue( + ctx, + ["protected_by"], + `protection, reason, and severity are not allowed for ${evaluation.pass_id}`, + ); + } + } else if (evaluation.protected_by !== undefined) { + addIssue(ctx, ["protected_by"], "protected_by is only valid for a protected evaluation"); + } + + if (evaluation.result !== "applied") { + if (evaluation.after === null || evaluation.after !== evaluation.before) { + addIssue(ctx, ["after"], `${evaluation.result} must preserve severity`); + } + if ( + evaluation.result === "would-apply" && + !passAllowsMaterialTransition(pass, { + reason_code: evaluation.reason_code, + before: evaluation.before, + }) + ) { + addIssue(ctx, ["reason_code"], `no material transition exists for ${evaluation.pass_id}`); + } + } else { + if ( + !passAllowsMaterialTransition(pass, { + reason_code: evaluation.reason_code, + before: evaluation.before, + after: evaluation.after, + }) + ) { + addIssue(ctx, ["after"], `transition is not allowed for ${evaluation.pass_id}`); + } + if (evaluation.after === null && evaluation.final_signature !== undefined) { + addIssue(ctx, ["final_signature"], "a dropped lineage cannot have a final signature"); + } + } + }, +); + +export type PolicyEvaluation = z.infer; + +const RanPolicyPassSummarySchema = z + .object({ + pass_id: PolicyPassIdSchema, + status: z.literal("ran"), + considered: z.number().int().nonnegative(), + opportunities: z.number().int().nonnegative(), + would_apply: z.number().int().nonnegative(), + applied: z.number().int().nonnegative(), + protected: z.number().int().nonnegative(), + blocking_removed: z.number().int().nonnegative(), + blocking_preserved: z.number().int().nonnegative(), + dropped: z.number().int().nonnegative(), + }) + .strict(); + +const InactivePolicyPassSummarySchema = z + .object({ + pass_id: PolicyPassIdSchema, + status: z.enum(["not-run", "error"]), + reason_code: PolicyReasonCodeSchema, + }) + .strict(); + +export const PolicyPassSummarySchema = z + .discriminatedUnion("status", [RanPolicyPassSummarySchema, InactivePolicyPassSummarySchema]) + .superRefine((summary, ctx) => { + const pass = policyPass(summary.pass_id); + + if (summary.status !== "ran") { + if (!INACTIVE_REASON_CODES.has(summary.reason_code)) { + addIssue(ctx, ["reason_code"], "inactive summaries require a closed inactive reason"); + } + return; + } + + const relationships: Array<[boolean, keyof typeof summary, string]> = [ + [summary.opportunities <= summary.considered, "opportunities", "opportunities > considered"], + [summary.would_apply <= summary.opportunities, "would_apply", "would_apply > opportunities"], + [summary.applied <= summary.would_apply, "applied", "applied > would_apply"], + [summary.protected <= summary.would_apply, "protected", "protected > would_apply"], + [ + summary.applied + summary.protected <= summary.would_apply, + "protected", + "applied + protected > would_apply", + ], + [ + summary.blocking_removed <= summary.applied, + "blocking_removed", + "blocking_removed > applied", + ], + [ + summary.blocking_preserved <= summary.would_apply, + "blocking_preserved", + "blocking_preserved > would_apply", + ], + [ + summary.blocking_removed + summary.blocking_preserved <= summary.would_apply, + "blocking_preserved", + "blocking results > would_apply", + ], + [summary.dropped <= summary.applied, "dropped", "dropped > applied"], + [ + summary.blocking_removed + summary.blocking_preserved + summary.dropped <= + summary.would_apply, + "dropped", + "blocking results + dropped > would_apply", + ], + ]; + for (const [valid, path, message] of relationships) { + if (!valid) addIssue(ctx, [path], message); + } + + if (summary.protected > 0 && !passAllowsAction(pass, "protected")) { + addIssue(ctx, ["protected"], `${summary.pass_id} has no protection action`); + } + if (summary.dropped > 0 && !passAllowsAction(pass, "dropped")) { + addIssue(ctx, ["dropped"], `${summary.pass_id} has no drop action`); + } + if ( + summary.blocking_removed > 0 && + !pass.material_transitions.some( + (transition) => + transition.before !== "INFO" && + (transition.after === null || transition.after === "INFO"), + ) + ) { + addIssue(ctx, ["blocking_removed"], `${summary.pass_id} cannot remove blocking status`); + } + + const allTransitionsStartBlocking = pass.material_transitions.every( + (transition) => transition.before !== "INFO", + ); + if ( + allTransitionsStartBlocking && + summary.blocking_removed + summary.blocking_preserved !== summary.would_apply + ) { + addIssue( + ctx, + ["blocking_preserved"], + `${summary.pass_id} must account for every matched blocking outcome`, + ); + } + + const allTransitionsRemoveBlocking = + allTransitionsStartBlocking && + pass.material_transitions.every( + (transition) => transition.after === null || transition.after === "INFO", + ); + if (allTransitionsRemoveBlocking && summary.blocking_removed !== summary.applied) { + addIssue( + ctx, + ["blocking_removed"], + `${summary.pass_id} must count every applied transition as blocking removed`, + ); + } + }); + +export type PolicyPassSummary = z.infer; + +const PolicyStageEvaluationObjectSchema = z + .object({ + stage_id: PolicyStageIdSchema, + order: z.number().int().positive(), + reason_code: PolicyReasonCodeSchema, + input_signatures: UniqueSignaturesSchema, + output_signature: z.string().min(1).optional(), + verdict: StageVerdictSchema.optional(), + }) + .strict(); + +export const PolicyStageEvaluationSchema = PolicyStageEvaluationObjectSchema.superRefine( + (evaluation, ctx) => { + const stage = policyStage(evaluation.stage_id); + if (evaluation.order !== stage.order) { + addIssue(ctx, ["order"], `order must match ${evaluation.stage_id} (${stage.order})`); + } + if (!stage.reason_codes.some((reason) => reason === evaluation.reason_code)) { + addIssue(ctx, ["reason_code"], `reason is not allowed for ${evaluation.stage_id}`); + } + + if (evaluation.stage_id === "aggregation.cluster") { + if (evaluation.input_signatures.length === 0) { + addIssue(ctx, ["input_signatures"], "a cluster stage requires input signatures"); + } + if (evaluation.output_signature === undefined) { + addIssue(ctx, ["output_signature"], "a cluster stage requires an output signature"); + } else if (!evaluation.input_signatures.includes(evaluation.output_signature)) { + addIssue(ctx, ["output_signature"], "the cluster representative must be an input"); + } + if (evaluation.verdict !== undefined) { + addIssue(ctx, ["verdict"], "a cluster stage cannot carry a verdict"); + } + if (evaluation.reason_code === "singleton" && evaluation.input_signatures.length !== 1) { + addIssue(ctx, ["reason_code"], "singleton requires exactly one input signature"); + } + if (evaluation.reason_code === "clustered" && evaluation.input_signatures.length < 2) { + addIssue(ctx, ["reason_code"], "clustered requires at least two input signatures"); + } + return; + } + + if (evaluation.output_signature !== undefined) { + addIssue(ctx, ["output_signature"], "the verdict stage cannot carry an output signature"); + } + if (evaluation.verdict === undefined) { + addIssue(ctx, ["verdict"], "the verdict stage requires a verdict"); + } else { + const expectedVerdict = + VERDICT_BY_REASON[evaluation.reason_code as keyof typeof VERDICT_BY_REASON]; + if (expectedVerdict !== undefined && evaluation.verdict !== expectedVerdict) { + addIssue(ctx, ["verdict"], `${evaluation.reason_code} requires verdict ${expectedVerdict}`); + } + } + if ( + evaluation.reason_code === "no-blocking-findings" && + evaluation.input_signatures.length !== 0 + ) { + addIssue(ctx, ["input_signatures"], "no-blocking-findings requires no blocking signatures"); + } + if ( + evaluation.reason_code !== "no-blocking-findings" && + evaluation.input_signatures.length === 0 + ) { + addIssue(ctx, ["input_signatures"], "a blocking verdict reason requires a signature"); + } + }, +); + +export type PolicyStageEvaluation = z.infer; + +export const PolicyTraceFinalSchema = z + .object({ + verdict: TraceVerdictSchema, + counts: z + .object({ + critical: z.number().int().nonnegative(), + warn: z.number().int().nonnegative(), + info: z.number().int().nonnegative(), + }) + .strict(), + finding_signatures: UniqueSignaturesSchema, + }) + .strict() + .superRefine((final, ctx) => { + const count = final.counts.critical + final.counts.warn + final.counts.info; + if (count !== final.finding_signatures.length) { + addIssue(ctx, ["finding_signatures"], "final counts must match final finding signatures"); + } + const blocking = final.counts.critical + final.counts.warn; + if (final.verdict === "PASS" && blocking !== 0) { + addIssue(ctx, ["verdict"], "PASS requires zero blocking findings"); + } + if ((final.verdict === "SOFT-PASS" || final.verdict === "FAIL") && blocking === 0) { + addIssue(ctx, ["verdict"], `${final.verdict} requires at least one blocking finding`); + } + if (final.verdict === "ERROR" && count !== 0) { + addIssue(ctx, ["verdict"], "ERROR cannot carry canonical findings"); + } + }); + +export type PolicyTraceFinal = z.infer; + +function validateOrderedPassRows( + rows: readonly PolicyPassSummary[], + ctx: z.RefinementCtx, + path: Array, +): void { + if (rows.length !== POLICY_PASS_IDS.length) { + addIssue(ctx, path, `passes must contain exactly ${POLICY_PASS_IDS.length} rows`); + return; + } + for (const [index, expected] of POLICY_PASS_IDS.entries()) { + if (rows[index]?.pass_id !== expected) { + addIssue(ctx, [...path, index, "pass_id"], `expected ordered pass ${expected}`); + } + } +} + +function validateArtifactState( + value: { + status: z.infer; + policy_trace_ref?: string | undefined; + policy_trace_sha256?: string | undefined; + }, + ctx: z.RefinementCtx, +): void { + if (value.status === "complete") { + if (value.policy_trace_ref === undefined) { + addIssue(ctx, ["policy_trace_ref"], "complete requires policy_trace_ref"); + } + if (value.policy_trace_sha256 === undefined) { + addIssue(ctx, ["policy_trace_sha256"], "complete requires policy_trace_sha256"); + } + return; + } + if (value.policy_trace_ref !== undefined) { + addIssue(ctx, ["policy_trace_ref"], `${value.status} forbids policy_trace_ref`); + } + if (value.policy_trace_sha256 !== undefined) { + addIssue(ctx, ["policy_trace_sha256"], `${value.status} forbids policy_trace_sha256`); + } +} + +export const PolicySummarySchema = z + .object({ + catalog_version: z.literal(POLICY_CATALOG_VERSION), + status: PolicyTraceStatusSchema, + passes: z.array(PolicyPassSummarySchema), + policy_trace_ref: z.string().min(1).optional(), + policy_trace_sha256: PolicySha256Schema.optional(), + }) + .strict() + .superRefine((summary, ctx) => { + validateOrderedPassRows(summary.passes, ctx, ["passes"]); + validateArtifactState(summary, ctx); + }); + +export type PolicySummary = z.infer; + +const PolicyTraceObjectSchema = z + .object({ + schema: z.literal("reviewgate.policy-trace.v1"), + catalog_version: z.literal(POLICY_CATALOG_VERSION), + run_id: z.string().min(1), + iter: z.number().int().nonnegative(), + ablated: z.array(PolicyPassIdSchema), + raw_response_sha256: z.array(PolicySha256Schema), + passes: z.array(PolicyPassSummarySchema), + evaluations: z.array(PolicyEvaluationSchema), + stages: z.array(PolicyStageEvaluationSchema), + final: PolicyTraceFinalSchema, + }) + .strict(); + +export const PolicyTraceSchema = PolicyTraceObjectSchema.superRefine((trace, ctx) => { + validateOrderedPassRows(trace.passes, ctx, ["passes"]); + + if (trace.final.verdict === "ERROR") { + addIssue( + ctx, + ["final", "verdict"], + "a complete policy trace requires a representable verdict.compute result", + ); + } + + const ablatedPasses = new Set(trace.ablated); + const ablatedOrders = trace.ablated.map((passId) => policyPass(passId).order); + for (let index = 1; index < trace.ablated.length; index += 1) { + const previous = ablatedOrders[index - 1]; + const current = ablatedOrders[index]; + if (previous === undefined || current === undefined || previous >= current) { + addIssue(ctx, ["ablated", index], "ablated pass IDs must be catalog-ordered and unique"); + } + } + + let priorEvaluationOrder = -1; + const evaluationsByPass = new Map(); + const finalSignatures = new Set(trace.final.finding_signatures); + const droppedEvaluations: Array<{ index: number; source_signatures: string[] }> = []; + for (const [index, evaluation] of trace.evaluations.entries()) { + if (evaluation.order < priorEvaluationOrder) { + addIssue(ctx, ["evaluations", index, "order"], "evaluations must remain in catalog order"); + } + priorEvaluationOrder = evaluation.order; + const rows = evaluationsByPass.get(evaluation.pass_id) ?? []; + rows.push(evaluation); + evaluationsByPass.set(evaluation.pass_id, rows); + if (evaluation.result === "applied" && ablatedPasses.has(evaluation.pass_id)) { + addIssue(ctx, ["evaluations", index, "result"], "an ablated pass cannot apply"); + } + if (evaluation.result === "would-apply" && !ablatedPasses.has(evaluation.pass_id)) { + addIssue( + ctx, + ["evaluations", index, "result"], + "would-apply requires the pass to be ablated", + ); + } + if (evaluation.result === "applied" && evaluation.after === null) { + droppedEvaluations.push({ index, source_signatures: evaluation.source_signatures }); + } + if ( + evaluation.final_signature !== undefined && + !finalSignatures.has(evaluation.final_signature) + ) { + addIssue(ctx, ["evaluations", index, "final_signature"], "unknown final signature"); + } + } + + for (const [index, summary] of trace.passes.entries()) { + const evaluations = evaluationsByPass.get(summary.pass_id) ?? []; + if (summary.status !== "ran") { + if (evaluations.length > 0) { + addIssue(ctx, ["passes", index], "an inactive pass cannot have evaluations"); + } + continue; + } + + const actual = { + considered: evaluations.length, + opportunities: evaluations.filter((row) => row.result !== "no-opportunity").length, + would_apply: evaluations.filter( + (row) => + row.result === "would-apply" || row.result === "protected" || row.result === "applied", + ).length, + applied: evaluations.filter((row) => row.result === "applied").length, + protected: evaluations.filter((row) => row.result === "protected").length, + blocking_removed: evaluations.filter( + (row) => + row.result === "applied" && + row.before !== "INFO" && + (row.after === null || row.after === "INFO"), + ).length, + blocking_preserved: evaluations.filter( + (row) => + (row.result === "would-apply" || + row.result === "protected" || + row.result === "applied") && + row.before !== "INFO" && + row.after !== null && + row.after !== "INFO", + ).length, + dropped: evaluations.filter((row) => row.result === "applied" && row.after === null).length, + }; + + for (const key of Object.keys(actual) as Array) { + if (summary[key] !== actual[key]) { + addIssue(ctx, ["passes", index, key], `${key} disagrees with policy evaluations`); + } + } + } + + let priorStageOrder = -1; + const clusterOutputs: string[] = []; + const clusterOutputSet = new Set(); + let verdictRows = 0; + let verdictStage: PolicyStageEvaluation | undefined; + let verdictStageIndex = -1; + for (const [index, stage] of trace.stages.entries()) { + if (stage.order < priorStageOrder) { + addIssue(ctx, ["stages", index, "order"], "stages must remain in catalog order"); + } + priorStageOrder = stage.order; + if (stage.stage_id === "aggregation.cluster" && stage.output_signature !== undefined) { + if (clusterOutputSet.has(stage.output_signature)) { + addIssue(ctx, ["stages", index, "output_signature"], "duplicate cluster output"); + } + clusterOutputs.push(stage.output_signature); + clusterOutputSet.add(stage.output_signature); + } + if (stage.stage_id === "verdict.compute") { + verdictRows += 1; + if (verdictStage === undefined) { + verdictStage = stage; + verdictStageIndex = index; + } + if (trace.final.verdict !== "ERROR" && stage.verdict !== trace.final.verdict) { + addIssue(ctx, ["stages", index, "verdict"], "stage verdict disagrees with final verdict"); + } + } + } + if (verdictRows !== 1) { + addIssue(ctx, ["stages"], "a complete trace requires exactly one verdict.compute row"); + } + + if (!isOrderedSubsequence(trace.final.finding_signatures, clusterOutputs)) { + addIssue(ctx, ["stages"], "final finding signatures must preserve cluster output order"); + } + const droppedSources = new Set( + droppedEvaluations.flatMap((evaluation) => evaluation.source_signatures), + ); + for (const [index, signature] of clusterOutputs.entries()) { + if (!finalSignatures.has(signature) && !droppedSources.has(signature)) { + addIssue(ctx, ["stages", index], "a non-final cluster output requires a later applied drop"); + } + } + for (const dropped of droppedEvaluations) { + const matchingOutputs = dropped.source_signatures.filter((signature) => + clusterOutputSet.has(signature), + ); + if (matchingOutputs.length !== 1) { + addIssue( + ctx, + ["evaluations", dropped.index, "source_signatures"], + "a dropped lineage requires exactly one aggregation cluster output", + ); + } + } + + if (verdictStage !== undefined) { + const blockingCount = trace.final.counts.critical + trace.final.counts.warn; + if (verdictStage.input_signatures.length !== blockingCount) { + addIssue( + ctx, + ["stages", verdictStageIndex, "input_signatures"], + "verdict inputs must equal the final blocking count", + ); + } + if (!verdictStage.input_signatures.every((signature) => finalSignatures.has(signature))) { + addIssue( + ctx, + ["stages", verdictStageIndex, "input_signatures"], + "verdict inputs must reference final findings", + ); + } + if (!isOrderedSubsequence(verdictStage.input_signatures, trace.final.finding_signatures)) { + addIssue( + ctx, + ["stages", verdictStageIndex, "input_signatures"], + "verdict inputs must preserve final finding order", + ); + } + if (verdictStage.reason_code === "hard-critical" && trace.final.counts.critical === 0) { + addIssue( + ctx, + ["stages", verdictStageIndex, "reason_code"], + "hard-critical requires at least one final CRITICAL finding", + ); + } + if (verdictStage.reason_code === "corroborated-warn" && trace.final.counts.warn === 0) { + addIssue( + ctx, + ["stages", verdictStageIndex, "reason_code"], + "corroborated-warn requires at least one final WARN finding", + ); + } + } +}); + +export type PolicyTrace = z.infer; diff --git a/tests/unit/finding-schema.test.ts b/tests/unit/finding-schema.test.ts index 350b927..a18eb79 100644 --- a/tests/unit/finding-schema.test.ts +++ b/tests/unit/finding-schema.test.ts @@ -75,3 +75,51 @@ describe("claimed_fixed_recurred tag", () => { expect(() => FindingSchema.parse({ ...base, claimed_fixed_recurred: { iter: 0 } })).toThrow(); }); }); + +describe("FindingSchema policy effects", () => { + it("keeps legacy findings parseable with policy_effects absent", () => { + expect(FindingSchema.parse(base).policy_effects).toBeUndefined(); + }); + + it("preserves a server-authored material policy effect", () => { + const parsed = FindingSchema.parse({ + ...base, + policy_effects: [ + { + pass_id: "judgment.confidence", + order: 140, + action: "demoted", + before: "WARN", + after: "INFO", + reason_code: "below-confidence-floor", + source_signatures: ["sig-a", "sig-b"], + }, + ], + }); + expect(parsed.policy_effects?.[0]?.pass_id).toBe("judgment.confidence"); + }); + + it("rejects malicious prose and unsorted lineage inside policy_effects", () => { + const effect = { + pass_id: "judgment.confidence", + order: 140, + action: "demoted", + before: "WARN", + after: "INFO", + reason_code: "below-confidence-floor", + source_signatures: ["sig-a"], + }; + expect( + FindingSchema.safeParse({ + ...base, + policy_effects: [{ ...effect, reason_code: "the reviewer asked for a demotion" }], + }).success, + ).toBe(false); + expect( + FindingSchema.safeParse({ + ...base, + policy_effects: [{ ...effect, source_signatures: ["sig-b", "sig-a"] }], + }).success, + ).toBe(false); + }); +}); diff --git a/tests/unit/pending-report.test.ts b/tests/unit/pending-report.test.ts index 4a1f412..d94ff7c 100644 --- a/tests/unit/pending-report.test.ts +++ b/tests/unit/pending-report.test.ts @@ -17,6 +17,40 @@ const baseFinding = { consensus: "singleton" as const, }; +const policyPassIds = [ + "evidence.fact-location", + "evidence.self-refutation", + "judgment.hypothetical", + "evidence.grounding-token", + "judgment.grounding-llm", + "evidence.redaction-placeholder", + "judgment.critic", + "scope.diff", + "scope.delta", + "scope.session", + "history.fp-signature", + "history.cycle-rejected", + "history.fp-cluster", + "judgment.confidence", + "judgment.reputation", + "history.region-rejected", + "judgment.test-security", + "judgment.docs-cap", +] as const; + +const policyPasses = policyPassIds.map((pass_id) => ({ + pass_id, + status: "ran" as const, + considered: 0, + opportunities: 0, + would_apply: 0, + applied: 0, + protected: 0, + blocking_removed: 0, + blocking_preserved: 0, + dropped: 0, +})); + describe("PendingReportSchema", () => { it("accepts a minimal PASS report with no findings", () => { const r: PendingReport = { @@ -117,4 +151,70 @@ describe("PendingReportSchema", () => { }; expect(() => PendingReportSchema.parse(r)).not.toThrow(); }); + + it("accepts an optional complete policy summary without changing the outer literal", () => { + const report = { + schema: "reviewgate.pending.v1" as const, + run_id: "x", + iter: 1, + max_iter: 3, + verdict: "PASS" as const, + counts: { critical: 0, warn: 0, info: 0 }, + reviewers: [], + findings: [], + policy_summary: { + catalog_version: "reviewgate.policy-catalog.v1" as const, + status: "complete" as const, + passes: policyPasses, + policy_trace_ref: "audit/2026/08/10/policy/trace.json", + policy_trace_sha256: "a".repeat(64), + }, + cost_usd_total: 0, + duration_ms_total: 0, + generated_at: "x", + git: { sha: "x", branch: "x", dirty_files: [] }, + }; + const parsed = PendingReportSchema.parse(report); + expect(parsed.schema).toBe("reviewgate.pending.v1"); + expect(parsed.policy_summary?.status).toBe("complete"); + }); + + it("rejects inconsistent complete/ref/hash policy summary states", () => { + const base = { + schema: "reviewgate.pending.v1" as const, + run_id: "x", + iter: 1, + max_iter: 3, + verdict: "PASS" as const, + counts: { critical: 0, warn: 0, info: 0 }, + reviewers: [], + findings: [], + cost_usd_total: 0, + duration_ms_total: 0, + generated_at: "x", + git: { sha: "x", branch: "x", dirty_files: [] }, + }; + expect( + PendingReportSchema.safeParse({ + ...base, + policy_summary: { + catalog_version: "reviewgate.policy-catalog.v1", + status: "complete", + passes: policyPasses, + }, + }).success, + ).toBe(false); + expect( + PendingReportSchema.safeParse({ + ...base, + policy_summary: { + catalog_version: "reviewgate.policy-catalog.v1", + status: "overflow", + passes: policyPasses, + policy_trace_ref: "trace.json", + policy_trace_sha256: "b".repeat(64), + }, + }).success, + ).toBe(false); + }); }); diff --git a/tests/unit/policy-catalog.test.ts b/tests/unit/policy-catalog.test.ts new file mode 100644 index 0000000..703a858 --- /dev/null +++ b/tests/unit/policy-catalog.test.ts @@ -0,0 +1,249 @@ +import { describe, expect, it } from "bun:test"; +import { + POLICY_CATALOG_VERSION, + POLICY_PASSES, + POLICY_PASS_IDS, + POLICY_PROTECTION_CODES, + POLICY_STAGES, + type PolicyPassCatalogEntry, +} from "../../src/core/policy/catalog.ts"; + +const COMMON_REASONS: ReadonlySet = new Set([ + "ineligible-starting-state", + "predicate-miss", + "configured-off", + "stage-precondition-miss", +]); + +describe("policy catalog", () => { + it("keeps the closed pass inventory in production execution order", () => { + expect(POLICY_PASSES.map((pass) => [pass.order, pass.id])).toEqual([ + [10, "evidence.fact-location"], + [20, "evidence.self-refutation"], + [30, "judgment.hypothetical"], + [40, "evidence.grounding-token"], + [50, "judgment.grounding-llm"], + [60, "evidence.redaction-placeholder"], + [70, "judgment.critic"], + [80, "scope.diff"], + [90, "scope.delta"], + [100, "scope.session"], + [110, "history.fp-signature"], + [120, "history.cycle-rejected"], + [130, "history.fp-cluster"], + [140, "judgment.confidence"], + [150, "judgment.reputation"], + [160, "history.region-rejected"], + [170, "judgment.test-security"], + [180, "judgment.docs-cap"], + ]); + }); + + it("keeps both explanatory stages in execution order", () => { + expect(POLICY_STAGES.map((stage) => [stage.order, stage.id])).toEqual([ + [65, "aggregation.cluster"], + [190, "verdict.compute"], + ]); + }); + + it("keeps the closed applied-reason and protection contract", () => { + expect( + POLICY_PASSES.map((pass) => ({ + id: pass.id, + reasons: pass.reason_codes.filter((reason) => !COMMON_REASONS.has(reason)), + protections: pass.protection_codes, + })), + ).toEqual([ + { + id: "evidence.fact-location", + reasons: ["location-out-of-range", "evidence-line-reanchored"], + protections: [], + }, + { + id: "evidence.self-refutation", + reasons: ["terminal-self-refutation"], + protections: ["security-correctness-floor", "deterministic-ground-truth"], + }, + { + id: "judgment.hypothetical", + reasons: ["hypothetical-critical"], + protections: ["security-correctness-floor", "deterministic-ground-truth"], + }, + { + id: "evidence.grounding-token", + reasons: ["cited-token-absent"], + protections: ["security-correctness-floor"], + }, + { + id: "judgment.grounding-llm", + reasons: ["judge-ungrounded"], + protections: ["security-correctness-floor"], + }, + { + id: "evidence.redaction-placeholder", + reasons: ["placeholder-code-hallucination"], + protections: ["security-correctness-floor", "secret-evidence-backstop"], + }, + { + id: "judgment.critic", + reasons: ["critic-likely-fp"], + protections: [ + "claimed-fixed-pin", + "self-refutation-visibility", + "security-correctness-floor", + "corroborated-majority", + "corroborated-unanimous", + "high-precision-reviewer", + ], + }, + { + id: "scope.diff", + reasons: ["outside-changed-file", "outside-changed-lines", "preexisting-harness-config"], + protections: ["out-of-diff-blocking-hatch"], + }, + { + id: "scope.delta", + reasons: ["outside-delta-scope"], + protections: [ + "claimed-fixed-pin", + "security-correctness-floor", + "critical-floor", + "out-of-diff-blocking-hatch", + ], + }, + { + id: "scope.session", + reasons: ["foreign-to-session"], + protections: ["out-of-diff-blocking-hatch"], + }, + { id: "history.fp-signature", reasons: ["active-fp-signature"], protections: [] }, + { + id: "history.cycle-rejected", + reasons: ["cycle-signature-rejected"], + protections: ["critical-floor", "security-correctness-floor"], + }, + { id: "history.fp-cluster", reasons: ["active-fp-cluster"], protections: [] }, + { + id: "judgment.confidence", + reasons: ["below-confidence-floor"], + protections: [ + "claimed-fixed-pin", + "security-correctness-floor", + "corroborated-majority", + "corroborated-unanimous", + "high-precision-reviewer", + ], + }, + { + id: "judgment.reputation", + reasons: ["unreliable-reviewer"], + protections: [ + "claimed-fixed-pin", + "security-floor", + "correctness-demote-disabled", + "corroborated-majority", + "corroborated-unanimous", + "critical-floor", + ], + }, + { + id: "history.region-rejected", + reasons: ["rejected-region-overlap"], + protections: [ + "claimed-fixed-pin", + "insufficient-distinct-rejections", + "category-change", + "severity-increase", + "critical-floor", + "security-correctness-floor", + ], + }, + { + id: "judgment.test-security", + reasons: ["test-only-security"], + protections: ["mixed-category-cluster"], + }, + { + id: "judgment.docs-cap", + reasons: ["docs-critical-cap"], + protections: ["security-correctness-floor"], + }, + ]); + }); + + it("is a versioned JSON-data catalog with no executable entries", () => { + expect(POLICY_CATALOG_VERSION).toBe("reviewgate.policy-catalog.v1"); + expect(POLICY_PASS_IDS).toHaveLength(18); + expect(JSON.parse(JSON.stringify(POLICY_PASSES))).toEqual(POLICY_PASSES); + expect(JSON.parse(JSON.stringify(POLICY_STAGES))).toEqual(POLICY_STAGES); + }); + + it("declares no protection code outside the closed per-pass contract", () => { + const used = [...new Set(POLICY_PASSES.flatMap((pass) => [...pass.protection_codes]))].sort(); + expect([...POLICY_PROTECTION_CODES].sort()).toEqual(used); + }); + + it("keeps every static transition inside its pass action/reason contract", () => { + for (const pass of POLICY_PASSES) { + const transitionActions = [...new Set(pass.material_transitions.map((row) => row.action))] + .sort() + .join(","); + const materialActions = pass.actions + .filter((action) => action !== "protected") + .sort() + .join(","); + expect(transitionActions).toBe(materialActions); + expect( + pass.material_transitions.every( + (row) => + pass.reason_codes.some((reason) => reason === row.reason_code) && + !COMMON_REASONS.has(row.reason_code), + ), + ).toBe(true); + expect(new Set(pass.material_transitions.map((row) => JSON.stringify(row))).size).toBe( + pass.material_transitions.length, + ); + expect(pass.actions.some((action) => action === "protected")).toBe( + pass.protection_codes.length > 0, + ); + expect(pass.protection_rules.length > 0).toBe(pass.protection_codes.length > 0); + expect( + pass.protection_codes.every((code) => + pass.protection_rules.some((rule) => rule.protected_by === code), + ), + ).toBe(true); + expect( + pass.protection_rules.every( + (rule) => + pass.protection_codes.some((code) => code === rule.protected_by) && + pass.reason_codes.some((reason) => reason === rule.reason_code) && + !COMMON_REASONS.has(rule.reason_code), + ), + ).toBe(true); + expect(new Set(pass.protection_rules.map((rule) => JSON.stringify(rule))).size).toBe( + pass.protection_rules.length, + ); + } + }); + + it("keeps protection guards bound to their production severities", () => { + const passes: readonly PolicyPassCatalogEntry[] = POLICY_PASSES; + const critic = passes.find((pass) => pass.id === "judgment.critic"); + const cycle = passes.find((pass) => pass.id === "history.cycle-rejected"); + expect( + critic?.protection_rules.some( + (rule) => rule.protected_by === "high-precision-reviewer" && rule.before === "INFO", + ), + ).toBe(false); + expect( + critic?.protection_rules.some( + (rule) => rule.protected_by === "self-refutation-visibility" && rule.before === "INFO", + ), + ).toBe(true); + expect( + cycle?.protection_rules.some( + (rule) => rule.protected_by === "critical-floor" && rule.before === "WARN", + ), + ).toBe(false); + }); +}); diff --git a/tests/unit/policy-trace-schema.test.ts b/tests/unit/policy-trace-schema.test.ts new file mode 100644 index 0000000..15cd58a --- /dev/null +++ b/tests/unit/policy-trace-schema.test.ts @@ -0,0 +1,1000 @@ +import { describe, expect, it } from "bun:test"; +import { + PolicyEffectSchema, + PolicyEvaluationSchema, + PolicyPassSummarySchema, + PolicyStageEvaluationSchema, + PolicySummarySchema, + PolicyTraceFinalSchema, + PolicyTraceSchema, +} from "../../src/schemas/policy-trace.ts"; + +const PASS_IDS = [ + "evidence.fact-location", + "evidence.self-refutation", + "judgment.hypothetical", + "evidence.grounding-token", + "judgment.grounding-llm", + "evidence.redaction-placeholder", + "judgment.critic", + "scope.diff", + "scope.delta", + "scope.session", + "history.fp-signature", + "history.cycle-rejected", + "history.fp-cluster", + "judgment.confidence", + "judgment.reputation", + "history.region-rejected", + "judgment.test-security", + "judgment.docs-cap", +] as const; + +function emptyRanSummary(passId: (typeof PASS_IDS)[number]) { + return { + pass_id: passId, + status: "ran" as const, + considered: 0, + opportunities: 0, + would_apply: 0, + applied: 0, + protected: 0, + blocking_removed: 0, + blocking_preserved: 0, + dropped: 0, + }; +} + +const validEffect = { + pass_id: "judgment.confidence", + order: 140, + action: "demoted", + before: "WARN", + after: "INFO", + reason_code: "below-confidence-floor", + source_signatures: ["sig-a", "sig-b"], +}; + +const validEvaluation = { + pass_id: "judgment.confidence", + order: 140, + result: "applied", + before: "WARN", + after: "INFO", + reason_code: "below-confidence-floor", + source_signatures: ["sig-a"], + final_signature: "sig-a", +}; + +function emptyTrace() { + return { + schema: "reviewgate.policy-trace.v1" as const, + catalog_version: "reviewgate.policy-catalog.v1" as const, + run_id: "run-1", + iter: 1, + ablated: [], + raw_response_sha256: ["a".repeat(64)], + passes: PASS_IDS.map(emptyRanSummary), + evaluations: [], + stages: [ + { + stage_id: "verdict.compute" as const, + order: 190, + reason_code: "no-blocking-findings" as const, + input_signatures: [], + verdict: "PASS" as const, + }, + ], + final: { + verdict: "PASS" as const, + counts: { critical: 0, warn: 0, info: 0 }, + finding_signatures: [], + }, + }; +} + +function passesWithOnly( + passId: (typeof PASS_IDS)[number], + summary: ReturnType, +) { + return PASS_IDS.map((candidate) => + candidate === passId + ? summary + : { + pass_id: candidate, + status: "not-run" as const, + reason_code: "stage-precondition-miss" as const, + }, + ); +} + +function traceWithSingleFinal(severity: "CRITICAL" | "WARN" | "INFO") { + const trace = emptyTrace(); + const counts = { + critical: severity === "CRITICAL" ? 1 : 0, + warn: severity === "WARN" ? 1 : 0, + info: severity === "INFO" ? 1 : 0, + }; + const blocking = severity !== "INFO"; + return { + ...trace, + passes: PASS_IDS.map((passId) => ({ + ...emptyRanSummary(passId), + considered: 1, + })), + evaluations: PASS_IDS.map((passId, index) => ({ + pass_id: passId, + order: (index + 1) * 10, + result: "no-opportunity" as const, + before: severity, + after: severity, + reason_code: "ineligible-starting-state" as const, + source_signatures: ["sig-a"], + final_signature: "sig-a", + })), + stages: [ + { + stage_id: "aggregation.cluster" as const, + order: 65, + reason_code: "singleton" as const, + input_signatures: ["sig-a"], + output_signature: "sig-a", + }, + { + stage_id: "verdict.compute" as const, + order: 190, + reason_code: blocking ? ("blocking-present" as const) : ("no-blocking-findings" as const), + input_signatures: blocking ? ["sig-a"] : [], + verdict: blocking ? ("SOFT-PASS" as const) : ("PASS" as const), + }, + ], + final: { + verdict: blocking ? ("SOFT-PASS" as const) : ("PASS" as const), + counts, + finding_signatures: ["sig-a"], + }, + }; +} + +describe("PolicyEffectSchema", () => { + it("accepts a catalog-valid material effect", () => { + expect(PolicyEffectSchema.parse(validEffect).reason_code).toBe("below-confidence-floor"); + }); + + it("rejects unknown pass, protection, and action values", () => { + expect( + PolicyEffectSchema.safeParse({ ...validEffect, pass_id: "judgment.unknown" }).success, + ).toBe(false); + expect(PolicyEffectSchema.safeParse({ ...validEffect, action: "rewritten" }).success).toBe( + false, + ); + expect( + PolicyEffectSchema.safeParse({ + ...validEffect, + action: "protected", + after: "WARN", + protected_by: "reviewer-said-so", + }).success, + ).toBe(false); + }); + + it("rejects reviewer-controlled prose as a reason code", () => { + expect( + PolicyEffectSchema.safeParse({ + ...validEffect, + reason_code: "The reviewer said this looked suspicious in the diff", + }).success, + ).toBe(false); + }); + + it("rejects a globally known reason, protection, or action on the wrong pass", () => { + expect( + PolicyEffectSchema.safeParse({ ...validEffect, reason_code: "docs-critical-cap" }).success, + ).toBe(false); + expect( + PolicyEffectSchema.safeParse({ ...validEffect, action: "suppressed", after: "INFO" }).success, + ).toBe(false); + expect( + PolicyEffectSchema.safeParse({ + ...validEffect, + action: "protected", + after: "WARN", + protected_by: "mixed-category-cluster", + }).success, + ).toBe(false); + }); + + it("requires source signatures to be sorted and deduplicated", () => { + expect( + PolicyEffectSchema.safeParse({ ...validEffect, source_signatures: ["sig-b", "sig-a"] }) + .success, + ).toBe(false); + expect( + PolicyEffectSchema.safeParse({ ...validEffect, source_signatures: ["sig-a", "sig-a"] }) + .success, + ).toBe(false); + expect( + PolicyEffectSchema.safeParse({ ...validEffect, source_signatures: ["sig-a", "sig-b"] }) + .success, + ).toBe(true); + }); + + it("enforces action-specific after and protection states", () => { + expect(PolicyEffectSchema.safeParse({ ...validEffect, after: "WARN" }).success).toBe(false); + expect(PolicyEffectSchema.safeParse({ ...validEffect, after: null }).success).toBe(false); + expect( + PolicyEffectSchema.safeParse({ + pass_id: "judgment.critic", + order: 70, + action: "dropped", + before: "INFO", + after: null, + reason_code: "critic-likely-fp", + source_signatures: ["sig-a"], + }).success, + ).toBe(true); + expect( + PolicyEffectSchema.safeParse({ + ...validEffect, + action: "protected", + after: "WARN", + protected_by: "high-precision-reviewer", + }).success, + ).toBe(true); + }); + + it("rejects pass-specific reason/action/transition mismatches", () => { + expect( + PolicyEffectSchema.safeParse({ + pass_id: "evidence.fact-location", + order: 10, + action: "reanchored", + before: "WARN", + after: "WARN", + reason_code: "location-out-of-range", + source_signatures: ["sig-a"], + }).success, + ).toBe(false); + expect( + PolicyEffectSchema.safeParse({ + pass_id: "judgment.docs-cap", + order: 180, + action: "capped", + before: "CRITICAL", + after: "INFO", + reason_code: "docs-critical-cap", + source_signatures: ["sig-a"], + }).success, + ).toBe(false); + expect( + PolicyEffectSchema.safeParse({ + pass_id: "judgment.critic", + order: 70, + action: "dropped", + before: "WARN", + after: null, + reason_code: "critic-likely-fp", + source_signatures: ["sig-a"], + }).success, + ).toBe(false); + expect( + PolicyEffectSchema.safeParse({ + pass_id: "judgment.docs-cap", + order: 180, + action: "capped", + before: "CRITICAL", + after: "WARN", + reason_code: "docs-critical-cap", + source_signatures: ["sig-a"], + }).success, + ).toBe(true); + }); + + it("binds each protection code to its exact starting severity", () => { + const criticProtection = { + pass_id: "judgment.critic", + order: 70, + action: "protected", + before: "WARN", + after: "WARN", + reason_code: "critic-likely-fp", + protected_by: "high-precision-reviewer", + source_signatures: ["sig-a"], + } as const; + expect(PolicyEffectSchema.safeParse(criticProtection).success).toBe(true); + expect( + PolicyEffectSchema.safeParse({ + ...criticProtection, + before: "INFO", + after: "INFO", + }).success, + ).toBe(false); + + const selfRefutationVisibility = { + ...criticProtection, + before: "INFO", + after: "INFO", + protected_by: "self-refutation-visibility", + } as const; + expect(PolicyEffectSchema.safeParse(selfRefutationVisibility).success).toBe(true); + expect( + PolicyEffectSchema.safeParse({ + ...selfRefutationVisibility, + before: "CRITICAL", + after: "CRITICAL", + }).success, + ).toBe(false); + expect( + PolicyEffectSchema.safeParse({ + ...criticProtection, + before: "INFO", + after: "INFO", + protected_by: "claimed-fixed-pin", + }).success, + ).toBe(true); + }); +}); + +describe("PolicyEvaluationSchema", () => { + it("accepts an applied evaluation and rejects mismatched result semantics", () => { + expect(PolicyEvaluationSchema.safeParse(validEvaluation).success).toBe(true); + expect( + PolicyEvaluationSchema.safeParse({ + ...validEvaluation, + result: "no-match", + reason_code: "below-confidence-floor", + after: "WARN", + }).success, + ).toBe(false); + expect( + PolicyEvaluationSchema.safeParse({ + ...validEvaluation, + result: "protected", + after: "WARN", + }).success, + ).toBe(false); + }); + + it("validates order, per-pass reason, protection, and lineage signatures", () => { + expect(PolicyEvaluationSchema.safeParse({ ...validEvaluation, order: 150 }).success).toBe( + false, + ); + expect( + PolicyEvaluationSchema.safeParse({ ...validEvaluation, reason_code: "unreliable-reviewer" }) + .success, + ).toBe(false); + expect( + PolicyEvaluationSchema.safeParse({ + ...validEvaluation, + result: "protected", + after: "WARN", + protected_by: "mixed-category-cluster", + }).success, + ).toBe(false); + expect( + PolicyEvaluationSchema.safeParse({ + ...validEvaluation, + source_signatures: ["sig-b", "sig-a"], + }).success, + ).toBe(false); + }); + + it("rejects applied transitions that the pass cannot produce", () => { + expect( + PolicyEvaluationSchema.safeParse({ + pass_id: "judgment.docs-cap", + order: 180, + result: "applied", + before: "CRITICAL", + after: "INFO", + reason_code: "docs-critical-cap", + source_signatures: ["sig-a"], + final_signature: "sig-a", + }).success, + ).toBe(false); + expect( + PolicyEvaluationSchema.safeParse({ + pass_id: "judgment.critic", + order: 70, + result: "applied", + before: "WARN", + after: null, + reason_code: "critic-likely-fp", + source_signatures: ["sig-a"], + }).success, + ).toBe(false); + }); + + it("accepts a critical match protected before a suppression can apply", () => { + expect( + PolicyEvaluationSchema.safeParse({ + pass_id: "history.cycle-rejected", + order: 120, + result: "protected", + before: "CRITICAL", + after: "CRITICAL", + reason_code: "cycle-signature-rejected", + protected_by: "critical-floor", + source_signatures: ["sig-a"], + final_signature: "sig-a", + }).success, + ).toBe(true); + }); + + it("rejects a protection code at a severity where that guard cannot fire", () => { + const protectedCycle = { + pass_id: "history.cycle-rejected", + order: 120, + result: "protected", + before: "WARN", + after: "WARN", + reason_code: "cycle-signature-rejected", + protected_by: "critical-floor", + source_signatures: ["sig-a"], + final_signature: "sig-a", + } as const; + expect(PolicyEvaluationSchema.safeParse(protectedCycle).success).toBe(false); + expect( + PolicyEvaluationSchema.safeParse({ + ...protectedCycle, + before: "CRITICAL", + after: "CRITICAL", + }).success, + ).toBe(true); + }); + + it("accepts the critic claimed-fixed pin that preserves an INFO finding", () => { + expect( + PolicyEvaluationSchema.safeParse({ + pass_id: "judgment.critic", + order: 70, + result: "protected", + before: "INFO", + after: "INFO", + reason_code: "critic-likely-fp", + protected_by: "claimed-fixed-pin", + source_signatures: ["sig-a"], + final_signature: "sig-a", + }).success, + ).toBe(true); + }); +}); + +describe("PolicyPassSummarySchema", () => { + const active = { + pass_id: "judgment.confidence", + status: "ran", + considered: 1, + opportunities: 1, + would_apply: 1, + applied: 1, + protected: 0, + blocking_removed: 1, + blocking_preserved: 0, + dropped: 0, + }; + + it("accepts a valid complete counter row", () => { + expect(PolicyPassSummarySchema.safeParse(active).success).toBe(true); + }); + + it("rejects applied counts greater than would-apply counts", () => { + expect(PolicyPassSummarySchema.safeParse({ ...active, applied: 2 }).success).toBe(false); + }); + + it("rejects every impossible counter relationship", () => { + const invalid = [ + { ...active, considered: 0 }, + { ...active, opportunities: 0 }, + { ...active, protected: 1 }, + { ...active, blocking_removed: 2 }, + { ...active, blocking_preserved: 1 }, + { ...active, dropped: 2 }, + ]; + for (const row of invalid) expect(PolicyPassSummarySchema.safeParse(row).success).toBe(false); + }); + + it("rejects counters that use an action absent from the pass catalog", () => { + expect( + PolicyPassSummarySchema.safeParse({ + ...active, + pass_id: "history.fp-signature", + applied: 0, + protected: 1, + blocking_removed: 0, + blocking_preserved: 1, + }).success, + ).toBe(false); + expect( + PolicyPassSummarySchema.safeParse({ + ...active, + pass_id: "judgment.docs-cap", + dropped: 1, + }).success, + ).toBe(false); + expect( + PolicyPassSummarySchema.safeParse({ + ...active, + pass_id: "judgment.docs-cap", + blocking_removed: 1, + }).success, + ).toBe(false); + }); + + it("rejects a pass summary that misclassifies an applied blocking outcome", () => { + expect( + PolicyPassSummarySchema.safeParse({ + ...active, + pass_id: "history.fp-signature", + blocking_removed: 0, + blocking_preserved: 1, + }).success, + ).toBe(false); + }); + + it("forbids counters and free-form reasons on an inactive pass summary", () => { + expect( + PolicyPassSummarySchema.safeParse({ + pass_id: "judgment.confidence", + status: "not-run", + reason_code: "configured-off", + considered: 0, + }).success, + ).toBe(false); + expect( + PolicyPassSummarySchema.safeParse({ + pass_id: "judgment.confidence", + status: "error", + reason_code: "provider returned an odd answer", + }).success, + ).toBe(false); + }); +}); + +describe("PolicyStageEvaluationSchema", () => { + it("accepts closed cluster and verdict stage rows", () => { + expect( + PolicyStageEvaluationSchema.safeParse({ + stage_id: "aggregation.cluster", + order: 65, + reason_code: "clustered", + input_signatures: ["sig-a", "sig-b"], + output_signature: "sig-a", + }).success, + ).toBe(true); + expect( + PolicyStageEvaluationSchema.safeParse({ + stage_id: "verdict.compute", + order: 190, + reason_code: "blocking-present", + input_signatures: ["sig-a"], + verdict: "SOFT-PASS", + }).success, + ).toBe(true); + }); + + it("rejects unknown reason prose and inconsistent stage fields", () => { + expect( + PolicyStageEvaluationSchema.safeParse({ + stage_id: "verdict.compute", + order: 190, + reason_code: "reviewer sounded confident", + input_signatures: [], + verdict: "PASS", + }).success, + ).toBe(false); + expect( + PolicyStageEvaluationSchema.safeParse({ + stage_id: "aggregation.cluster", + order: 65, + reason_code: "singleton", + input_signatures: ["sig-a", "sig-b"], + output_signature: "sig-c", + }).success, + ).toBe(false); + }); + + it("binds each verdict reason to its only valid verdict", () => { + expect( + PolicyStageEvaluationSchema.safeParse({ + stage_id: "verdict.compute", + order: 190, + reason_code: "no-blocking-findings", + input_signatures: [], + verdict: "FAIL", + }).success, + ).toBe(false); + expect( + PolicyStageEvaluationSchema.safeParse({ + stage_id: "verdict.compute", + order: 190, + reason_code: "blocking-present", + input_signatures: ["sig-a"], + verdict: "PASS", + }).success, + ).toBe(false); + expect( + PolicyStageEvaluationSchema.safeParse({ + stage_id: "verdict.compute", + order: 190, + reason_code: "hard-critical", + input_signatures: ["sig-a"], + verdict: "SOFT-PASS", + }).success, + ).toBe(false); + }); +}); + +describe("PolicyTraceFinalSchema", () => { + it("preserves deterministic production finding order while rejecting duplicates", () => { + const final = { + verdict: "PASS", + counts: { critical: 0, warn: 0, info: 2 }, + finding_signatures: ["sig-z", "sig-a"], + }; + expect(PolicyTraceFinalSchema.safeParse(final).success).toBe(true); + expect( + PolicyTraceFinalSchema.safeParse({ + ...final, + finding_signatures: ["sig-a", "sig-a"], + }).success, + ).toBe(false); + }); +}); + +describe("PolicySummarySchema", () => { + const passes = PASS_IDS.map(emptyRanSummary); + const hash = "b".repeat(64); + + it("requires an exact ordered 18-pass summary", () => { + expect( + PolicySummarySchema.safeParse({ + catalog_version: "reviewgate.policy-catalog.v1", + status: "not-run", + passes, + }).success, + ).toBe(true); + expect( + PolicySummarySchema.safeParse({ + catalog_version: "reviewgate.policy-catalog.v1", + status: "not-run", + passes: passes.slice(0, -1), + }).success, + ).toBe(false); + expect( + PolicySummarySchema.safeParse({ + catalog_version: "reviewgate.policy-catalog.v1", + status: "not-run", + passes: [passes[1], passes[0], ...passes.slice(2)], + }).success, + ).toBe(false); + }); + + it("requires ref and hash only for a complete trace", () => { + expect( + PolicySummarySchema.safeParse({ + catalog_version: "reviewgate.policy-catalog.v1", + status: "complete", + passes, + policy_trace_ref: "audit/2026/08/10/policy/trace.json", + policy_trace_sha256: hash, + }).success, + ).toBe(true); + expect( + PolicySummarySchema.safeParse({ + catalog_version: "reviewgate.policy-catalog.v1", + status: "complete", + passes, + }).success, + ).toBe(false); + expect( + PolicySummarySchema.safeParse({ + catalog_version: "reviewgate.policy-catalog.v1", + status: "error", + passes, + policy_trace_ref: "trace.json", + policy_trace_sha256: hash, + }).success, + ).toBe(false); + }); +}); + +describe("PolicyTraceSchema", () => { + it("accepts a complete empty-finding trace", () => { + expect(PolicyTraceSchema.safeParse(emptyTrace()).success).toBe(true); + }); + + it("rejects missing pass rows, malformed hashes, and duplicate ablations", () => { + const trace = emptyTrace(); + expect( + PolicyTraceSchema.safeParse({ ...trace, passes: trace.passes.slice(0, -1) }).success, + ).toBe(false); + expect( + PolicyTraceSchema.safeParse({ ...trace, raw_response_sha256: ["A".repeat(64)] }).success, + ).toBe(false); + expect( + PolicyTraceSchema.safeParse({ + ...trace, + ablated: ["judgment.confidence", "judgment.confidence"], + }).success, + ).toBe(false); + }); + + it("rejects summaries that disagree with per-finding evaluations", () => { + const trace = emptyTrace(); + const confidenceIndex = PASS_IDS.indexOf("judgment.confidence"); + const passes = [...trace.passes]; + passes[confidenceIndex] = { + ...emptyRanSummary("judgment.confidence"), + considered: 1, + opportunities: 1, + would_apply: 1, + applied: 1, + blocking_removed: 0, + }; + expect( + PolicyTraceSchema.safeParse({ + ...trace, + passes, + evaluations: [validEvaluation], + stages: [ + { + stage_id: "aggregation.cluster", + order: 65, + reason_code: "singleton", + input_signatures: ["sig-a"], + output_signature: "sig-a", + }, + { + stage_id: "verdict.compute", + order: 190, + reason_code: "no-blocking-findings", + input_signatures: [], + verdict: "PASS", + }, + ], + final: { + verdict: "PASS", + counts: { critical: 0, warn: 0, info: 1 }, + finding_signatures: ["sig-a"], + }, + }).success, + ).toBe(false); + }); + + it("rejects a missing verdict stage or cluster outputs inconsistent with final findings", () => { + const trace = emptyTrace(); + expect(PolicyTraceSchema.safeParse({ ...trace, stages: [] }).success).toBe(false); + expect( + PolicyTraceSchema.safeParse({ + ...trace, + final: { + verdict: "PASS", + counts: { critical: 0, warn: 0, info: 1 }, + finding_signatures: ["sig-a"], + }, + }).success, + ).toBe(false); + }); + + it("cross-checks applied and would-apply evaluations against the ablation set", () => { + const applied = traceWithSingleFinal("INFO"); + const appliedSummary = { + ...emptyRanSummary("judgment.confidence"), + considered: 1, + opportunities: 1, + would_apply: 1, + applied: 1, + blocking_removed: 1, + }; + expect( + PolicyTraceSchema.safeParse({ + ...applied, + ablated: ["judgment.confidence"], + passes: passesWithOnly("judgment.confidence", appliedSummary), + evaluations: [validEvaluation], + }).success, + ).toBe(false); + + const wouldApply = traceWithSingleFinal("WARN"); + const wouldApplyEvaluation = { + ...validEvaluation, + result: "would-apply", + after: "WARN", + } as const; + const wouldApplySummary = { + ...emptyRanSummary("judgment.confidence"), + considered: 1, + opportunities: 1, + would_apply: 1, + blocking_preserved: 1, + }; + expect( + PolicyTraceSchema.safeParse({ + ...wouldApply, + passes: passesWithOnly("judgment.confidence", wouldApplySummary), + evaluations: [wouldApplyEvaluation], + }).success, + ).toBe(false); + expect( + PolicyTraceSchema.safeParse({ + ...wouldApply, + ablated: ["judgment.confidence"], + passes: passesWithOnly("judgment.confidence", wouldApplySummary), + evaluations: [wouldApplyEvaluation], + }).success, + ).toBe(true); + }); + + it("keeps a post-cluster critic drop traceable without treating it as a final finding", () => { + const trace = emptyTrace(); + const criticSummary = { + ...emptyRanSummary("judgment.critic"), + considered: 1, + opportunities: 1, + would_apply: 1, + applied: 1, + dropped: 1, + }; + const droppedEvaluation = { + pass_id: "judgment.critic" as const, + order: 70, + result: "applied" as const, + before: "INFO" as const, + after: null, + reason_code: "critic-likely-fp" as const, + source_signatures: ["sig-a"], + }; + const stages = [ + { + stage_id: "aggregation.cluster" as const, + order: 65, + reason_code: "singleton" as const, + input_signatures: ["sig-a"], + output_signature: "sig-a", + }, + ...trace.stages, + ]; + const droppedTrace = { + ...trace, + passes: passesWithOnly("judgment.critic", criticSummary), + evaluations: [droppedEvaluation], + stages, + }; + expect(PolicyTraceSchema.safeParse(droppedTrace).success).toBe(true); + expect( + PolicyTraceSchema.safeParse({ + ...droppedTrace, + stages: trace.stages, + }).success, + ).toBe(false); + }); + + it("binds the final verdict to blocking counts, signatures, and the verdict stage", () => { + const warnTrace = traceWithSingleFinal("WARN"); + expect(PolicyTraceSchema.safeParse(warnTrace).success).toBe(true); + expect( + PolicyTraceSchema.safeParse({ + ...warnTrace, + stages: [ + warnTrace.stages[0], + { + stage_id: "verdict.compute", + order: 190, + reason_code: "no-blocking-findings", + input_signatures: [], + verdict: "PASS", + }, + ], + final: { ...warnTrace.final, verdict: "PASS" }, + }).success, + ).toBe(false); + expect( + PolicyTraceSchema.safeParse({ + ...warnTrace, + final: { ...warnTrace.final, verdict: "ERROR" }, + }).success, + ).toBe(false); + }); + + it("binds verdict reasons to severity counts and excludes unrepresentable ERROR traces", () => { + const warnTrace = traceWithSingleFinal("WARN"); + expect( + PolicyTraceSchema.safeParse({ + ...warnTrace, + stages: [ + warnTrace.stages[0], + { + stage_id: "verdict.compute", + order: 190, + reason_code: "hard-critical", + input_signatures: ["sig-a"], + verdict: "FAIL", + }, + ], + final: { ...warnTrace.final, verdict: "FAIL" }, + }).success, + ).toBe(false); + + const criticalTrace = traceWithSingleFinal("CRITICAL"); + expect( + PolicyTraceSchema.safeParse({ + ...criticalTrace, + stages: [ + criticalTrace.stages[0], + { + stage_id: "verdict.compute", + order: 190, + reason_code: "corroborated-warn", + input_signatures: ["sig-a"], + verdict: "FAIL", + }, + ], + final: { ...criticalTrace.final, verdict: "FAIL" }, + }).success, + ).toBe(false); + + const empty = emptyTrace(); + expect( + PolicyTraceSchema.safeParse({ + ...empty, + final: { ...empty.final, verdict: "ERROR" }, + }).success, + ).toBe(false); + }); + + it("requires the verdict stage to enumerate every final blocking signature", () => { + const single = traceWithSingleFinal("WARN"); + const trace = { + ...single, + passes: single.passes.map((summary) => ({ ...summary, considered: 2 })), + evaluations: single.evaluations.flatMap((evaluation) => [ + evaluation, + { + ...evaluation, + source_signatures: ["sig-b"], + final_signature: "sig-b", + }, + ]), + stages: [ + { + stage_id: "aggregation.cluster" as const, + order: 65, + reason_code: "singleton" as const, + input_signatures: ["sig-a"], + output_signature: "sig-a", + }, + { + stage_id: "aggregation.cluster" as const, + order: 65, + reason_code: "singleton" as const, + input_signatures: ["sig-b"], + output_signature: "sig-b", + }, + { + stage_id: "verdict.compute" as const, + order: 190, + reason_code: "blocking-present" as const, + input_signatures: ["sig-a", "sig-b"], + verdict: "SOFT-PASS" as const, + }, + ], + final: { + verdict: "SOFT-PASS" as const, + counts: { critical: 0, warn: 2, info: 0 }, + finding_signatures: ["sig-a", "sig-b"], + }, + }; + expect(PolicyTraceSchema.safeParse(trace).success).toBe(true); + expect( + PolicyTraceSchema.safeParse({ + ...trace, + stages: [ + trace.stages[0], + trace.stages[1], + { ...trace.stages[2], input_signatures: ["sig-a"] }, + ], + }).success, + ).toBe(false); + }); +}); diff --git a/tests/unit/run-summary-schema.test.ts b/tests/unit/run-summary-schema.test.ts index d078580..319f258 100644 --- a/tests/unit/run-summary-schema.test.ts +++ b/tests/unit/run-summary-schema.test.ts @@ -27,6 +27,7 @@ const valid = { describe("RunSummarySchema", () => { it("validates a complete run summary", () => { expect(RunSummarySchema.parse(valid).providers[0]?.provider).toBe("codex"); + expect(RunSummarySchema.parse(valid).policy_trace_status).toBeUndefined(); }); it("accepts an empty (skipped/cache) summary", () => { expect( @@ -48,4 +49,43 @@ describe("RunSummarySchema", () => { RunSummarySchema.parse({ ...valid, providers: [{ ...valid.providers[0], provider: "x" }] }), ).toThrow(); }); + + it("accepts a complete content-addressed policy trace", () => { + const parsed = RunSummarySchema.parse({ + ...valid, + policy_trace_status: "complete", + policy_trace_ref: "audit/2026/08/10/policy/trace.json", + policy_trace_sha256: "a".repeat(64), + }); + expect(parsed.policy_trace_status).toBe("complete"); + }); + + it("rejects inconsistent complete/ref/hash states", () => { + expect(RunSummarySchema.safeParse({ ...valid, policy_trace_status: "complete" }).success).toBe( + false, + ); + expect( + RunSummarySchema.safeParse({ + ...valid, + policy_trace_status: "error", + policy_trace_ref: "trace.json", + policy_trace_sha256: "a".repeat(64), + }).success, + ).toBe(false); + expect( + RunSummarySchema.safeParse({ + ...valid, + policy_trace_ref: "trace.json", + policy_trace_sha256: "a".repeat(64), + }).success, + ).toBe(false); + expect( + RunSummarySchema.safeParse({ + ...valid, + policy_trace_status: "complete", + policy_trace_ref: "trace.json", + policy_trace_sha256: "A".repeat(64), + }).success, + ).toBe(false); + }); }); From ec63f88f7c8117c5b0f94cff89e99cc8393e42c8 Mon Sep 17 00:00:00 2001 From: Codevena Date: Mon, 10 Aug 2026 03:58:29 +0200 Subject: [PATCH 23/55] fix(policy): close authoritative trace invariants --- src/core/policy/catalog.ts | 9 +- src/schemas/policy-trace.ts | 170 +++++++++++++---- tests/unit/policy-catalog.test.ts | 14 ++ tests/unit/policy-trace-schema.test.ts | 247 ++++++++++++++++++++++++- 4 files changed, 401 insertions(+), 39 deletions(-) diff --git a/src/core/policy/catalog.ts b/src/core/policy/catalog.ts index 3dc8ae0..908d05e 100644 --- a/src/core/policy/catalog.ts +++ b/src/core/policy/catalog.ts @@ -37,6 +37,7 @@ export const POLICY_REASON_CODES = [ "predicate-miss", "configured-off", "stage-precondition-miss", + "instrumentation-error", "location-out-of-range", "evidence-line-reanchored", "terminal-self-refutation", @@ -230,7 +231,7 @@ export const POLICY_PASSES = [ before: "WARN", }, ], - opportunity: "blocking, non-deterministic finding", + opportunity: "blocking finding", depends_on: ["evidence.fact-location"], overlaps_with: [ "evidence.fact-location", @@ -269,7 +270,7 @@ export const POLICY_PASSES = [ before: "CRITICAL", }, ], - opportunity: "CRITICAL, non-deterministic finding", + opportunity: "CRITICAL finding", depends_on: ["evidence.self-refutation"], overlaps_with: [ "evidence.fact-location", @@ -870,7 +871,7 @@ export const POLICY_PASSES = [ before: "WARN", }, ], - opportunity: "blocking, uncorroborated finding while the confidence floor is positive", + opportunity: "blocking finding while the confidence floor is positive", depends_on: ["history.fp-cluster"], overlaps_with: ["judgment.critic", "judgment.reputation"], ablatable: true, @@ -972,7 +973,7 @@ export const POLICY_PASSES = [ before: "WARN", }, ], - opportunity: "blocking, uncorroborated finding while unreliable reviewers exist", + opportunity: "blocking finding while unreliable reviewers exist", depends_on: ["judgment.confidence"], overlaps_with: ["judgment.critic", "judgment.confidence"], ablatable: true, diff --git a/src/schemas/policy-trace.ts b/src/schemas/policy-trace.ts index 0325c4b..deae3d2 100644 --- a/src/schemas/policy-trace.ts +++ b/src/schemas/policy-trace.ts @@ -33,12 +33,14 @@ const EvaluationResultSchema = z.enum([ const TraceVerdictSchema = z.enum(["PASS", "SOFT-PASS", "FAIL", "ERROR"]); const StageVerdictSchema = z.enum(["PASS", "SOFT-PASS", "FAIL"]); -const INACTIVE_REASON_CODES = new Set(["configured-off", "stage-precondition-miss"]); +const NOT_RUN_REASON_CODES = new Set(["configured-off", "stage-precondition-miss"]); +const PASS_ERROR_REASON_CODE = "instrumentation-error" as const; const NON_MATERIAL_REASON_CODES = new Set([ "ineligible-starting-state", "predicate-miss", "configured-off", "stage-precondition-miss", + PASS_ERROR_REASON_CODE, ]); const VERDICT_BY_REASON = { "hard-critical": "FAIL", @@ -356,8 +358,11 @@ export const PolicyPassSummarySchema = z const pass = policyPass(summary.pass_id); if (summary.status !== "ran") { - if (!INACTIVE_REASON_CODES.has(summary.reason_code)) { - addIssue(ctx, ["reason_code"], "inactive summaries require a closed inactive reason"); + if (summary.status === "not-run" && !NOT_RUN_REASON_CODES.has(summary.reason_code)) { + addIssue(ctx, ["reason_code"], "not-run summaries require a closed inactivity reason"); + } + if (summary.status === "error" && summary.reason_code !== PASS_ERROR_REASON_CODE) { + addIssue(ctx, ["reason_code"], `error summaries require ${PASS_ERROR_REASON_CODE}`); } return; } @@ -517,6 +522,13 @@ export const PolicyStageEvaluationSchema = PolicyStageEvaluationObjectSchema.sup export type PolicyStageEvaluation = z.infer; +const PolicyFinalFindingSeveritySchema = z + .object({ + signature: z.string().min(1), + severity: PolicySeveritySchema, + }) + .strict(); + export const PolicyTraceFinalSchema = z .object({ verdict: TraceVerdictSchema, @@ -528,6 +540,7 @@ export const PolicyTraceFinalSchema = z }) .strict(), finding_signatures: UniqueSignaturesSchema, + finding_severities: z.array(PolicyFinalFindingSeveritySchema), }) .strict() .superRefine((final, ctx) => { @@ -535,6 +548,37 @@ export const PolicyTraceFinalSchema = z if (count !== final.finding_signatures.length) { addIssue(ctx, ["finding_signatures"], "final counts must match final finding signatures"); } + if (final.finding_severities.length !== final.finding_signatures.length) { + addIssue( + ctx, + ["finding_severities"], + "final severity evidence must match final finding cardinality", + ); + } + for (const [index, signature] of final.finding_signatures.entries()) { + if (final.finding_severities[index]?.signature !== signature) { + addIssue( + ctx, + ["finding_severities", index, "signature"], + "final severity evidence must preserve finding signature order", + ); + } + } + const derivedCounts = { critical: 0, warn: 0, info: 0 }; + for (const finding of final.finding_severities) { + if (finding.severity === "CRITICAL") derivedCounts.critical += 1; + else if (finding.severity === "WARN") derivedCounts.warn += 1; + else derivedCounts.info += 1; + } + for (const severity of ["critical", "warn", "info"] as const) { + if (final.counts[severity] !== derivedCounts[severity]) { + addIssue( + ctx, + ["counts", severity], + `${severity} count disagrees with final severity evidence`, + ); + } + } const blocking = final.counts.critical + final.counts.warn; if (final.verdict === "PASS" && blocking !== 0) { addIssue(ctx, ["verdict"], "PASS requires zero blocking findings"); @@ -602,6 +646,17 @@ export const PolicySummarySchema = z .superRefine((summary, ctx) => { validateOrderedPassRows(summary.passes, ctx, ["passes"]); validateArtifactState(summary, ctx); + if (summary.status === "not-run") { + for (const [index, pass] of summary.passes.entries()) { + if (pass.status !== "not-run") { + addIssue( + ctx, + ["passes", index, "status"], + "a not-run policy summary requires every pass to be not-run", + ); + } + } + } }); export type PolicySummary = z.infer; @@ -645,7 +700,6 @@ export const PolicyTraceSchema = PolicyTraceObjectSchema.superRefine((trace, ctx let priorEvaluationOrder = -1; const evaluationsByPass = new Map(); const finalSignatures = new Set(trace.final.finding_signatures); - const droppedEvaluations: Array<{ index: number; source_signatures: string[] }> = []; for (const [index, evaluation] of trace.evaluations.entries()) { if (evaluation.order < priorEvaluationOrder) { addIssue(ctx, ["evaluations", index, "order"], "evaluations must remain in catalog order"); @@ -664,9 +718,6 @@ export const PolicyTraceSchema = PolicyTraceObjectSchema.superRefine((trace, ctx "would-apply requires the pass to be ablated", ); } - if (evaluation.result === "applied" && evaluation.after === null) { - droppedEvaluations.push({ index, source_signatures: evaluation.source_signatures }); - } if ( evaluation.final_signature !== undefined && !finalSignatures.has(evaluation.final_signature) @@ -721,6 +772,7 @@ export const PolicyTraceSchema = PolicyTraceObjectSchema.superRefine((trace, ctx let priorStageOrder = -1; const clusterOutputs: string[] = []; const clusterOutputSet = new Set(); + const clusterOutputByInput = new Map(); let verdictRows = 0; let verdictStage: PolicyStageEvaluation | undefined; let verdictStageIndex = -1; @@ -735,6 +787,17 @@ export const PolicyTraceSchema = PolicyTraceObjectSchema.superRefine((trace, ctx } clusterOutputs.push(stage.output_signature); clusterOutputSet.add(stage.output_signature); + for (const [inputIndex, input] of stage.input_signatures.entries()) { + if (clusterOutputByInput.has(input)) { + addIssue( + ctx, + ["stages", index, "input_signatures", inputIndex], + "a cluster input must map to exactly one output", + ); + } else { + clusterOutputByInput.set(input, stage.output_signature); + } + } } if (stage.stage_id === "verdict.compute") { verdictRows += 1; @@ -754,48 +817,91 @@ export const PolicyTraceSchema = PolicyTraceObjectSchema.superRefine((trace, ctx if (!isOrderedSubsequence(trace.final.finding_signatures, clusterOutputs)) { addIssue(ctx, ["stages"], "final finding signatures must preserve cluster output order"); } - const droppedSources = new Set( - droppedEvaluations.flatMap((evaluation) => evaluation.source_signatures), - ); - for (const [index, signature] of clusterOutputs.entries()) { - if (!finalSignatures.has(signature) && !droppedSources.has(signature)) { - addIssue(ctx, ["stages", index], "a non-final cluster output requires a later applied drop"); + + const droppedOutputs = new Map(); + for (const [index, evaluation] of trace.evaluations.entries()) { + const lineageOutputs = new Set(); + for (const [sourceIndex, source] of evaluation.source_signatures.entries()) { + const output = clusterOutputByInput.get(source); + if (output === undefined) { + addIssue( + ctx, + ["evaluations", index, "source_signatures", sourceIndex], + "evaluation lineage must reference an aggregation cluster input", + ); + } else { + lineageOutputs.add(output); + } } - } - for (const dropped of droppedEvaluations) { - const matchingOutputs = dropped.source_signatures.filter((signature) => - clusterOutputSet.has(signature), - ); - if (matchingOutputs.length !== 1) { + if (lineageOutputs.size !== 1) { addIssue( ctx, - ["evaluations", dropped.index, "source_signatures"], - "a dropped lineage requires exactly one aggregation cluster output", + ["evaluations", index, "source_signatures"], + "evaluation lineage must resolve to exactly one cluster output", ); + continue; } - } - if (verdictStage !== undefined) { - const blockingCount = trace.final.counts.critical + trace.final.counts.warn; - if (verdictStage.input_signatures.length !== blockingCount) { + const [output] = lineageOutputs; + if (output === undefined) continue; + const appliedDrop = evaluation.result === "applied" && evaluation.after === null; + const survives = finalSignatures.has(output); + + if (appliedDrop) { + if (survives) { + addIssue( + ctx, + ["evaluations", index, "source_signatures"], + "an applied-drop cluster output cannot remain in final findings", + ); + } + if (droppedOutputs.has(output)) { + addIssue( + ctx, + ["evaluations", index, "source_signatures"], + "a cluster output can have only one applied drop", + ); + } else { + droppedOutputs.set(output, index); + } + } + + if (survives && evaluation.final_signature !== output) { addIssue( ctx, - ["stages", verdictStageIndex, "input_signatures"], - "verdict inputs must equal the final blocking count", + ["evaluations", index, "final_signature"], + "a surviving evaluation must name its resolved cluster output", ); } - if (!verdictStage.input_signatures.every((signature) => finalSignatures.has(signature))) { + if (!survives && evaluation.final_signature !== undefined) { addIssue( ctx, - ["stages", verdictStageIndex, "input_signatures"], - "verdict inputs must reference final findings", + ["evaluations", index, "final_signature"], + "a non-surviving evaluation cannot name a final signature", ); } - if (!isOrderedSubsequence(verdictStage.input_signatures, trace.final.finding_signatures)) { + } + + for (const [index, signature] of clusterOutputs.entries()) { + if (!finalSignatures.has(signature) && !droppedOutputs.has(signature)) { + addIssue(ctx, ["stages", index], "a non-final cluster output requires a later applied drop"); + } + } + + if (verdictStage !== undefined) { + const blockingSignatures = trace.final.finding_severities + .filter((finding) => finding.severity !== "INFO") + .map((finding) => finding.signature); + if ( + verdictStage.input_signatures.length !== blockingSignatures.length || + verdictStage.input_signatures.some( + (signature, index) => signature !== blockingSignatures[index], + ) + ) { addIssue( ctx, ["stages", verdictStageIndex, "input_signatures"], - "verdict inputs must preserve final finding order", + "verdict inputs must exactly equal ordered final blocking signatures", ); } if (verdictStage.reason_code === "hard-critical" && trace.final.counts.critical === 0) { diff --git a/tests/unit/policy-catalog.test.ts b/tests/unit/policy-catalog.test.ts index 703a858..bbe03f1 100644 --- a/tests/unit/policy-catalog.test.ts +++ b/tests/unit/policy-catalog.test.ts @@ -171,6 +171,20 @@ describe("policy catalog", () => { ]); }); + it("keeps protection guards outside opportunity eligibility", () => { + const opportunities = Object.fromEntries( + POLICY_PASSES.map((pass) => [pass.id, pass.opportunity]), + ); + expect(opportunities["evidence.self-refutation"]).toBe("blocking finding"); + expect(opportunities["judgment.hypothetical"]).toBe("CRITICAL finding"); + expect(opportunities["judgment.confidence"]).toBe( + "blocking finding while the confidence floor is positive", + ); + expect(opportunities["judgment.reputation"]).toBe( + "blocking finding while unreliable reviewers exist", + ); + }); + it("is a versioned JSON-data catalog with no executable entries", () => { expect(POLICY_CATALOG_VERSION).toBe("reviewgate.policy-catalog.v1"); expect(POLICY_PASS_IDS).toHaveLength(18); diff --git a/tests/unit/policy-trace-schema.test.ts b/tests/unit/policy-trace-schema.test.ts index 15cd58a..0deaeb1 100644 --- a/tests/unit/policy-trace-schema.test.ts +++ b/tests/unit/policy-trace-schema.test.ts @@ -89,6 +89,7 @@ function emptyTrace() { verdict: "PASS" as const, counts: { critical: 0, warn: 0, info: 0 }, finding_signatures: [], + finding_severities: [], }, }; } @@ -152,6 +153,72 @@ function traceWithSingleFinal(severity: "CRITICAL" | "WARN" | "INFO") { verdict: blocking ? ("SOFT-PASS" as const) : ("PASS" as const), counts, finding_signatures: ["sig-a"], + finding_severities: [{ signature: "sig-a", severity }], + }, + }; +} + +function traceWithWarnAndInfoFinals() { + const trace = emptyTrace(); + return { + ...trace, + passes: PASS_IDS.map((passId) => ({ + ...emptyRanSummary(passId), + considered: 2, + })), + evaluations: PASS_IDS.flatMap((passId, index) => [ + { + pass_id: passId, + order: (index + 1) * 10, + result: "no-opportunity" as const, + before: "INFO" as const, + after: "INFO" as const, + reason_code: "ineligible-starting-state" as const, + source_signatures: ["sig-info"], + final_signature: "sig-info", + }, + { + pass_id: passId, + order: (index + 1) * 10, + result: "no-opportunity" as const, + before: "WARN" as const, + after: "WARN" as const, + reason_code: "ineligible-starting-state" as const, + source_signatures: ["sig-warn"], + final_signature: "sig-warn", + }, + ]), + stages: [ + { + stage_id: "aggregation.cluster" as const, + order: 65, + reason_code: "singleton" as const, + input_signatures: ["sig-info"], + output_signature: "sig-info", + }, + { + stage_id: "aggregation.cluster" as const, + order: 65, + reason_code: "singleton" as const, + input_signatures: ["sig-warn"], + output_signature: "sig-warn", + }, + { + stage_id: "verdict.compute" as const, + order: 190, + reason_code: "blocking-present" as const, + input_signatures: ["sig-warn"], + verdict: "SOFT-PASS" as const, + }, + ], + final: { + verdict: "SOFT-PASS" as const, + counts: { critical: 0, warn: 1, info: 1 }, + finding_signatures: ["sig-info", "sig-warn"], + finding_severities: [ + { signature: "sig-info", severity: "INFO" as const }, + { signature: "sig-warn", severity: "WARN" as const }, + ], }, }; } @@ -549,6 +616,27 @@ describe("PolicyPassSummarySchema", () => { }).success, ).toBe(false); }); + + it("binds inactive pass reasons to not-run versus instrumentation error", () => { + const notRun = { + pass_id: "judgment.confidence", + status: "not-run", + reason_code: "configured-off", + }; + expect(PolicyPassSummarySchema.safeParse(notRun).success).toBe(true); + expect( + PolicyPassSummarySchema.safeParse({ + ...notRun, + status: "error", + reason_code: "instrumentation-error", + }).success, + ).toBe(true); + expect(PolicyPassSummarySchema.safeParse({ ...notRun, status: "error" }).success).toBe(false); + expect( + PolicyPassSummarySchema.safeParse({ ...notRun, reason_code: "instrumentation-error" }) + .success, + ).toBe(false); + }); }); describe("PolicyStageEvaluationSchema", () => { @@ -631,6 +719,10 @@ describe("PolicyTraceFinalSchema", () => { verdict: "PASS", counts: { critical: 0, warn: 0, info: 2 }, finding_signatures: ["sig-z", "sig-a"], + finding_severities: [ + { signature: "sig-z", severity: "INFO" }, + { signature: "sig-a", severity: "INFO" }, + ], }; expect(PolicyTraceFinalSchema.safeParse(final).success).toBe(true); expect( @@ -640,6 +732,39 @@ describe("PolicyTraceFinalSchema", () => { }).success, ).toBe(false); }); + + it("requires ordered one-to-one severity evidence and derives final counts from it", () => { + const final = { + verdict: "SOFT-PASS" as const, + counts: { critical: 0, warn: 1, info: 1 }, + finding_signatures: ["sig-warn", "sig-info"], + finding_severities: [ + { signature: "sig-warn", severity: "WARN" as const }, + { signature: "sig-info", severity: "INFO" as const }, + ], + }; + + expect(PolicyTraceFinalSchema.safeParse(final).success).toBe(true); + expect( + PolicyTraceFinalSchema.safeParse({ + verdict: final.verdict, + counts: final.counts, + finding_signatures: final.finding_signatures, + }).success, + ).toBe(false); + expect( + PolicyTraceFinalSchema.safeParse({ + ...final, + finding_severities: [final.finding_severities[1], final.finding_severities[0]], + }).success, + ).toBe(false); + expect( + PolicyTraceFinalSchema.safeParse({ + ...final, + counts: { critical: 0, warn: 2, info: 0 }, + }).success, + ).toBe(false); + }); }); describe("PolicySummarySchema", () => { @@ -650,21 +775,21 @@ describe("PolicySummarySchema", () => { expect( PolicySummarySchema.safeParse({ catalog_version: "reviewgate.policy-catalog.v1", - status: "not-run", + status: "error", passes, }).success, ).toBe(true); expect( PolicySummarySchema.safeParse({ catalog_version: "reviewgate.policy-catalog.v1", - status: "not-run", + status: "error", passes: passes.slice(0, -1), }).success, ).toBe(false); expect( PolicySummarySchema.safeParse({ catalog_version: "reviewgate.policy-catalog.v1", - status: "not-run", + status: "error", passes: [passes[1], passes[0], ...passes.slice(2)], }).success, ).toBe(false); @@ -697,6 +822,28 @@ describe("PolicySummarySchema", () => { }).success, ).toBe(false); }); + + it("requires a top-level not-run summary to contain only not-run pass rows", () => { + expect( + PolicySummarySchema.safeParse({ + catalog_version: "reviewgate.policy-catalog.v1", + status: "not-run", + passes, + }).success, + ).toBe(false); + + expect( + PolicySummarySchema.safeParse({ + catalog_version: "reviewgate.policy-catalog.v1", + status: "not-run", + passes: PASS_IDS.map((pass_id) => ({ + pass_id, + status: "not-run", + reason_code: "stage-precondition-miss", + })), + }).success, + ).toBe(true); + }); }); describe("PolicyTraceSchema", () => { @@ -757,6 +904,7 @@ describe("PolicyTraceSchema", () => { verdict: "PASS", counts: { critical: 0, warn: 0, info: 1 }, finding_signatures: ["sig-a"], + finding_severities: [{ signature: "sig-a", severity: "INFO" }], }, }).success, ).toBe(false); @@ -772,6 +920,7 @@ describe("PolicyTraceSchema", () => { verdict: "PASS", counts: { critical: 0, warn: 0, info: 1 }, finding_signatures: ["sig-a"], + finding_severities: [{ signature: "sig-a", severity: "INFO" }], }, }).success, ).toBe(false); @@ -870,6 +1019,80 @@ describe("PolicyTraceSchema", () => { ).toBe(false); }); + it("rejects a dropped cluster output that remains in the final findings", () => { + const trace = emptyTrace(); + expect( + PolicyTraceSchema.safeParse({ + ...trace, + passes: passesWithOnly("judgment.critic", { + ...emptyRanSummary("judgment.critic"), + considered: 1, + opportunities: 1, + would_apply: 1, + applied: 1, + dropped: 1, + }), + evaluations: [ + { + pass_id: "judgment.critic", + order: 70, + result: "applied", + before: "INFO", + after: null, + reason_code: "critic-likely-fp", + source_signatures: ["sig-a"], + }, + ], + stages: [ + { + stage_id: "aggregation.cluster", + order: 65, + reason_code: "singleton", + input_signatures: ["sig-a"], + output_signature: "sig-a", + }, + ...trace.stages, + ], + final: { + verdict: "PASS", + counts: { critical: 0, warn: 0, info: 1 }, + finding_signatures: ["sig-a"], + finding_severities: [{ signature: "sig-a", severity: "INFO" }], + }, + }).success, + ).toBe(false); + }); + + it("requires each evaluation lineage and final signature to resolve to one cluster output", () => { + const trace = traceWithWarnAndInfoFinals(); + expect( + PolicyTraceSchema.safeParse({ + ...trace, + evaluations: trace.evaluations.map((evaluation, index) => + index === 0 ? { ...evaluation, final_signature: "sig-warn" } : evaluation, + ), + }).success, + ).toBe(false); + }); + + it("rejects a cluster input assigned to more than one output", () => { + const trace = traceWithWarnAndInfoFinals(); + expect( + PolicyTraceSchema.safeParse({ + ...trace, + stages: [ + trace.stages[0], + { + ...trace.stages[1], + reason_code: "clustered", + input_signatures: ["sig-info", "sig-warn"], + }, + trace.stages[2], + ], + }).success, + ).toBe(false); + }); + it("binds the final verdict to blocking counts, signatures, and the verdict stage", () => { const warnTrace = traceWithSingleFinal("WARN"); expect(PolicyTraceSchema.safeParse(warnTrace).success).toBe(true); @@ -983,6 +1206,10 @@ describe("PolicyTraceSchema", () => { verdict: "SOFT-PASS" as const, counts: { critical: 0, warn: 2, info: 0 }, finding_signatures: ["sig-a", "sig-b"], + finding_severities: [ + { signature: "sig-a", severity: "WARN" as const }, + { signature: "sig-b", severity: "WARN" as const }, + ], }, }; expect(PolicyTraceSchema.safeParse(trace).success).toBe(true); @@ -997,4 +1224,18 @@ describe("PolicyTraceSchema", () => { }).success, ).toBe(false); }); + + it("rejects a verdict stage that swaps an INFO signature for a blocking signature", () => { + const trace = traceWithWarnAndInfoFinals(); + expect( + PolicyTraceSchema.safeParse({ + ...trace, + stages: [ + trace.stages[0], + trace.stages[1], + { ...trace.stages[2], input_signatures: ["sig-info"] }, + ], + }).success, + ).toBe(false); + }); }); From 5942e441cf8222bf6edd0ef3353f354d107ab6f0 Mon Sep 17 00:00:00 2001 From: Codevena Date: Mon, 10 Aug 2026 04:26:17 +0200 Subject: [PATCH 24/55] feat(policy): add fail-open transition recorder --- src/core/policy/response-hashes.ts | 52 +++ src/core/policy/trace.ts | 453 ++++++++++++++++++ tests/unit/policy-response-hashes.test.ts | 56 +++ tests/unit/policy-trace-recorder.test.ts | 545 ++++++++++++++++++++++ 4 files changed, 1106 insertions(+) create mode 100644 src/core/policy/response-hashes.ts create mode 100644 src/core/policy/trace.ts create mode 100644 tests/unit/policy-response-hashes.test.ts create mode 100644 tests/unit/policy-trace-recorder.test.ts diff --git a/src/core/policy/response-hashes.ts b/src/core/policy/response-hashes.ts new file mode 100644 index 0000000..4926b44 --- /dev/null +++ b/src/core/policy/response-hashes.ts @@ -0,0 +1,52 @@ +import { createHash } from "node:crypto"; + +export interface OrderedResponseHash { + readonly kind: string; + readonly ordinal: number; + readonly sha256: string; +} + +function compareByteOrder(left: string, right: string): number { + if (left < right) return -1; + if (left > right) return 1; + return 0; +} + +/** + * Holds response digests in logical call order. The raw response is hashed at + * the boundary and is never retained by this object. + */ +export class OrderedResponseHashes { + readonly #entries = new Map(); + + record(kind: string, ordinal: number, rawText?: string): void { + if (rawText === undefined) return; + if (kind.length === 0) throw new Error("response hash kind must not be empty"); + if (!Number.isSafeInteger(ordinal) || ordinal < 0) { + throw new Error("response hash ordinal must be a non-negative safe integer"); + } + + const identity = JSON.stringify([kind, ordinal]); + if (this.#entries.has(identity)) { + throw new Error(`duplicate response hash call identity: ${kind}:${ordinal}`); + } + + this.#entries.set(identity, { + kind, + ordinal, + sha256: createHash("sha256").update(Buffer.from(rawText, "utf8")).digest("hex"), + }); + } + + entries(): OrderedResponseHash[] { + return [...this.#entries.values()] + .sort( + (left, right) => left.ordinal - right.ordinal || compareByteOrder(left.kind, right.kind), + ) + .map((entry) => ({ ...entry })); + } + + values(): string[] { + return this.entries().map((entry) => entry.sha256); + } +} diff --git a/src/core/policy/trace.ts b/src/core/policy/trace.ts new file mode 100644 index 0000000..c7fb82f --- /dev/null +++ b/src/core/policy/trace.ts @@ -0,0 +1,453 @@ +import type { Finding } from "../../schemas/finding.ts"; +import { + type PolicyEffect, + PolicyEffectSchema, + PolicyEffectsSchema, + type PolicyEvaluation, + PolicyEvaluationSchema, + type PolicyPassSummary, + PolicyPassSummarySchema, + type PolicyStageEvaluation, + PolicyStageEvaluationSchema, + type PolicyTrace, + PolicyTraceSchema, +} from "../../schemas/policy-trace.ts"; +import { + POLICY_PASSES, + POLICY_STAGES, + type PolicyEffectAction, + type PolicyPassId, + type PolicyProtectionCode, + type PolicyReasonCode, + type PolicyStageId, +} from "./catalog.ts"; + +type RanPolicyPassSummary = Extract; +type TraceVerdict = PolicyTrace["final"]["verdict"]; + +const PASS_BY_ID = new Map(POLICY_PASSES.map((pass) => [pass.id, pass])); +const STAGE_BY_ID = new Map(POLICY_STAGES.map((stage) => [stage.id, stage])); + +export interface TransitionInput { + readonly runtime?: PolicyRuntime; + readonly passId: PolicyPassId; + readonly finding: Finding; + readonly opportunity: boolean; + readonly matched: boolean; + readonly reasonCode: PolicyReasonCode; + readonly action: PolicyEffectAction; + readonly protectedBy?: PolicyProtectionCode; + readonly sourceSignatures?: readonly string[]; + readonly proposed: () => Finding | null; +} + +export interface RecordPolicyStageInput { + readonly stageId: PolicyStageId; + readonly reasonCode: PolicyReasonCode; + readonly inputSignatures: readonly string[]; + readonly outputSignature?: string; + readonly verdict?: Exclude; +} + +export interface FinalizePolicyTraceInput { + readonly rawResponseSha256: readonly string[]; + readonly verdict: TraceVerdict; + readonly finalFindings: readonly Pick[]; +} + +export interface PolicyRuntime { + readonly telemetryError: boolean; + transition(input: TransitionInput): Finding | null; + summary(passId: PolicyPassId): PolicyPassSummary; + evaluations(): PolicyEvaluation[]; + recordStage(input: RecordPolicyStageInput): void; + linkFinal(inputSignatures: readonly string[], finalSignature: string): void; + finalize(input: FinalizePolicyTraceInput): PolicyTrace | null; +} + +export interface StartPolicyTraceInput { + readonly runId: string; + readonly iter: number; + readonly ablated: readonly PolicyPassId[] | ReadonlySet; +} + +function compareByteOrder(left: string, right: string): number { + if (left < right) return -1; + if (left > right) return 1; + return 0; +} + +function sortedUnique(values: readonly string[]): string[] { + return [...new Set(values)].sort(compareByteOrder); +} + +function uniqueInOrder(values: readonly string[]): string[] { + return [...new Set(values)]; +} + +function effectIdentity(effect: PolicyEffect): string { + return JSON.stringify([ + effect.pass_id, + effect.action, + effect.before, + effect.after, + effect.reason_code, + effect.protected_by ?? null, + effect.source_signatures, + ]); +} + +export function mergePolicyEffects( + ...groups: Array +): PolicyEffect[] { + const byIdentity = new Map(); + for (const effect of groups.flatMap((group) => group ?? [])) { + const parsed = PolicyEffectSchema.parse(effect); + byIdentity.set(effectIdentity(parsed), parsed); + } + + const merged = [...byIdentity.entries()] + .sort( + ([leftIdentity, left], [rightIdentity, right]) => + left.order - right.order || compareByteOrder(leftIdentity, rightIdentity), + ) + .map(([, effect]) => effect); + return PolicyEffectsSchema.parse(merged); +} + +function emptySummary(passId: PolicyPassId): RanPolicyPassSummary { + return { + pass_id: passId, + status: "ran", + considered: 0, + opportunities: 0, + would_apply: 0, + applied: 0, + protected: 0, + blocking_removed: 0, + blocking_preserved: 0, + dropped: 0, + }; +} + +function copyEvaluation(evaluation: PolicyEvaluation): PolicyEvaluation { + return { + ...evaluation, + source_signatures: [...evaluation.source_signatures], + }; +} + +function countFinalFindings( + findings: readonly Pick[], +): PolicyTrace["final"]["counts"] { + const counts = { critical: 0, warn: 0, info: 0 }; + for (const finding of findings) { + if (finding.severity === "CRITICAL") counts.critical += 1; + else if (finding.severity === "WARN") counts.warn += 1; + else counts.info += 1; + } + return counts; +} + +export class PolicyTraceRecorder implements PolicyRuntime { + readonly #runId: string; + readonly #iter: number; + readonly #ablated: ReadonlySet; + readonly #summaries = new Map(); + readonly #evaluations: PolicyEvaluation[] = []; + readonly #stages: PolicyStageEvaluation[] = []; + readonly #finalBySource = new Map(); + #telemetryError = false; + + private constructor(input: StartPolicyTraceInput) { + this.#runId = input.runId; + this.#iter = input.iter; + this.#ablated = new Set(input.ablated); + for (const pass of POLICY_PASSES) this.#summaries.set(pass.id, emptySummary(pass.id)); + } + + static start(input: StartPolicyTraceInput): PolicyTraceRecorder { + return new PolicyTraceRecorder(input); + } + + get telemetryError(): boolean { + return this.#telemetryError; + } + + transition(input: TransitionInput): Finding | null { + // These are production inputs. Read them and calculate the production result + // before entering the fail-open telemetry boundary so their errors propagate. + const finding = input.finding; + const opportunity = input.opportunity; + const matched = input.matched; + const protectedBy = input.protectedBy; + const productionResult = matched && protectedBy === undefined ? input.proposed() : finding; + const ablated = this.#ablated.has(input.passId); + + try { + if (!opportunity) { + if (matched) throw new Error("a matched policy transition requires an opportunity"); + this.#recordEvaluation({ + pass_id: input.passId, + result: "no-opportunity", + before: finding.severity, + after: finding.severity, + reason_code: "ineligible-starting-state", + source_signatures: this.#sourceSignatures(input, finding), + }); + return finding; + } + + if (!matched) { + this.#recordEvaluation({ + pass_id: input.passId, + result: "no-match", + before: finding.severity, + after: finding.severity, + reason_code: "predicate-miss", + source_signatures: this.#sourceSignatures(input, finding), + }); + return finding; + } + + const sourceSignatures = this.#sourceSignatures(input, finding); + if (protectedBy !== undefined) { + const effect = PolicyEffectSchema.parse({ + pass_id: input.passId, + order: this.#passOrder(input.passId), + action: "protected", + before: finding.severity, + after: finding.severity, + reason_code: input.reasonCode, + protected_by: protectedBy, + source_signatures: sourceSignatures, + }); + this.#recordEvaluation( + { + pass_id: input.passId, + result: "protected", + before: finding.severity, + after: finding.severity, + reason_code: input.reasonCode, + protected_by: protectedBy, + source_signatures: sourceSignatures, + }, + effect, + ); + return { + ...finding, + policy_effects: mergePolicyEffects(finding.policy_effects, [effect]), + }; + } + + if (ablated) { + this.#recordEvaluation({ + pass_id: input.passId, + result: "would-apply", + before: finding.severity, + after: finding.severity, + reason_code: input.reasonCode, + source_signatures: sourceSignatures, + }); + return finding; + } + + const after = productionResult?.severity ?? null; + const effect = PolicyEffectSchema.parse({ + pass_id: input.passId, + order: this.#passOrder(input.passId), + action: input.action, + before: finding.severity, + after, + reason_code: input.reasonCode, + source_signatures: sourceSignatures, + }); + this.#recordEvaluation( + { + pass_id: input.passId, + result: "applied", + before: finding.severity, + after, + reason_code: input.reasonCode, + source_signatures: sourceSignatures, + }, + effect, + ); + if (productionResult === null) return null; + return { + ...productionResult, + policy_effects: mergePolicyEffects( + finding.policy_effects, + productionResult.policy_effects, + [effect], + ), + }; + } catch { + this.#telemetryError = true; + return ablated ? finding : productionResult; + } + } + + summary(passId: PolicyPassId): PolicyPassSummary { + const summary = this.#summaries.get(passId); + if (summary === undefined) throw new Error(`unknown policy pass: ${passId}`); + return { ...summary }; + } + + evaluations(): PolicyEvaluation[] { + return this.#evaluations.map(copyEvaluation); + } + + recordStage(input: RecordPolicyStageInput): void { + try { + const stage = STAGE_BY_ID.get(input.stageId); + if (stage === undefined) throw new Error(`unknown policy stage: ${input.stageId}`); + const candidate = { + stage_id: input.stageId, + order: stage.order, + reason_code: input.reasonCode, + input_signatures: uniqueInOrder(input.inputSignatures), + ...(input.outputSignature === undefined ? {} : { output_signature: input.outputSignature }), + ...(input.verdict === undefined ? {} : { verdict: input.verdict }), + }; + this.#stages.push(PolicyStageEvaluationSchema.parse(candidate)); + } catch { + this.#telemetryError = true; + } + } + + linkFinal(inputSignatures: readonly string[], finalSignature: string): void { + try { + const sources = sortedUnique(inputSignatures); + if (sources.length === 0 || !sources.includes(finalSignature)) { + throw new Error("final signature must be one of the cluster inputs"); + } + for (const source of sources) { + const existing = this.#finalBySource.get(source); + if (existing !== undefined && existing !== finalSignature) { + throw new Error(`policy lineage ${source} already links to ${existing}`); + } + } + for (const source of sources) this.#finalBySource.set(source, finalSignature); + } catch { + this.#telemetryError = true; + } + } + + finalize(input: FinalizePolicyTraceInput): PolicyTrace | null { + if (this.#telemetryError) return null; + try { + const finalSignatures = new Set(input.finalFindings.map((finding) => finding.signature)); + const evaluations = this.#evaluations + .map((evaluation) => { + const linked = new Set( + evaluation.source_signatures + .map((source) => this.#finalBySource.get(source)) + .filter((signature): signature is string => signature !== undefined), + ); + const [finalSignature] = linked; + return { + ...copyEvaluation(evaluation), + ...(linked.size === 1 && + finalSignature !== undefined && + finalSignatures.has(finalSignature) + ? { final_signature: finalSignature } + : {}), + }; + }) + .sort((left, right) => left.order - right.order); + const stages = this.#stages + .map((stage, index) => ({ stage, index })) + .sort((left, right) => left.stage.order - right.stage.order || left.index - right.index) + .map(({ stage }) => ({ ...stage, input_signatures: [...stage.input_signatures] })); + const final = { + verdict: input.verdict, + counts: countFinalFindings(input.finalFindings), + finding_signatures: input.finalFindings.map((finding) => finding.signature), + finding_severities: input.finalFindings.map((finding) => ({ + signature: finding.signature, + severity: finding.severity, + })), + }; + return PolicyTraceSchema.parse({ + schema: "reviewgate.policy-trace.v1", + catalog_version: "reviewgate.policy-catalog.v1", + run_id: this.#runId, + iter: this.#iter, + ablated: POLICY_PASSES.filter((pass) => this.#ablated.has(pass.id)).map((pass) => pass.id), + raw_response_sha256: [...input.rawResponseSha256], + passes: POLICY_PASSES.map((pass) => this.summary(pass.id)), + evaluations, + stages, + final, + }); + } catch { + this.#telemetryError = true; + return null; + } + } + + #passOrder(passId: PolicyPassId): number { + const pass = PASS_BY_ID.get(passId); + if (pass === undefined) throw new Error(`unknown policy pass: ${passId}`); + return pass.order; + } + + #sourceSignatures(input: TransitionInput, finding: Finding): string[] { + return sortedUnique(input.sourceSignatures ?? [finding.signature]); + } + + #recordEvaluation( + input: Omit, + effect?: PolicyEffect, + ): void { + if (effect !== undefined) PolicyEffectSchema.parse(effect); + const evaluation = PolicyEvaluationSchema.parse({ + ...input, + order: this.#passOrder(input.pass_id), + }); + const current = this.#summaries.get(evaluation.pass_id); + if (current === undefined) throw new Error(`unknown policy pass: ${evaluation.pass_id}`); + const next: RanPolicyPassSummary = { ...current, considered: current.considered + 1 }; + + if (evaluation.result !== "no-opportunity") next.opportunities += 1; + if ( + evaluation.result === "would-apply" || + evaluation.result === "protected" || + evaluation.result === "applied" + ) { + next.would_apply += 1; + if (evaluation.result === "protected") next.protected += 1; + if (evaluation.result === "applied") { + next.applied += 1; + if (evaluation.after === null) next.dropped += 1; + } + if (evaluation.before !== "INFO") { + if ( + evaluation.result === "applied" && + (evaluation.after === null || evaluation.after === "INFO") + ) { + next.blocking_removed += 1; + } else if (evaluation.after !== null && evaluation.after !== "INFO") { + next.blocking_preserved += 1; + } + } + } + + const parsedSummary = PolicyPassSummarySchema.parse(next); + if (parsedSummary.status !== "ran") throw new Error("recorded summary unexpectedly inactive"); + this.#summaries.set(evaluation.pass_id, parsedSummary); + this.#evaluations.push(evaluation); + } +} + +export function transitionFinding(input: TransitionInput): Finding | null { + // Access telemetry predicates before any runtime-owned fail-open boundary. + void input.opportunity; + const matched = input.matched; + const protectedBy = input.protectedBy; + if (input.runtime === undefined) { + return matched && protectedBy === undefined ? input.proposed() : input.finding; + } + return input.runtime.transition(input); +} diff --git a/tests/unit/policy-response-hashes.test.ts b/tests/unit/policy-response-hashes.test.ts new file mode 100644 index 0000000..3cafd69 --- /dev/null +++ b/tests/unit/policy-response-hashes.test.ts @@ -0,0 +1,56 @@ +import { describe, expect, it } from "bun:test"; +import { OrderedResponseHashes } from "../../src/core/policy/response-hashes.ts"; + +describe("OrderedResponseHashes", () => { + it("hashes UTF-8 response bytes in logical ordinal order", () => { + const hashes = new OrderedResponseHashes(); + + hashes.record("critic", 2, "abc"); + hashes.record("reviewer", 0, ""); + hashes.record("grounding", 1, "Grüße 🚪"); + + expect(hashes.values()).toEqual([ + "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "4df010e4ad94311d48de48a08a3fe623b46e5535377c83bd73eb7f1b7ad63355", + "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad", + ]); + }); + + it("retains only call identity and SHA-256, never raw response text", () => { + const hashes = new OrderedResponseHashes(); + + hashes.record("reviewer", 0, "raw secret-shaped reviewer response"); + + expect(hashes.entries()).toEqual([ + { + kind: "reviewer", + ordinal: 0, + sha256: "ca072f2419f612818cc0be807a46c9a293797889a1cbd9454737bb5d0e9d92df", + }, + ]); + expect(JSON.stringify(hashes.entries())).not.toContain("secret-shaped"); + }); + + it("distinguishes an empty successful response from no response", () => { + const hashes = new OrderedResponseHashes(); + const recordCall = (ordinal: number, call: () => string): void => { + const response = call(); + hashes.record("critic", ordinal, response); + }; + + expect(() => + recordCall(0, () => { + throw new Error("provider failed before returning a response"); + }), + ).toThrow("provider failed before returning a response"); + hashes.record("critic", 1, undefined); + hashes.record("critic", 2, ""); + + expect(hashes.entries()).toHaveLength(1); + expect(hashes.entries()[0]).toEqual({ + kind: "critic", + ordinal: 2, + sha256: "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + }); + }); +}); diff --git a/tests/unit/policy-trace-recorder.test.ts b/tests/unit/policy-trace-recorder.test.ts new file mode 100644 index 0000000..91e3204 --- /dev/null +++ b/tests/unit/policy-trace-recorder.test.ts @@ -0,0 +1,545 @@ +import { describe, expect, it } from "bun:test"; +import { + PolicyTraceRecorder, + type TransitionInput, + mergePolicyEffects, + transitionFinding, +} from "../../src/core/policy/trace.ts"; +import type { Finding } from "../../src/schemas/finding.ts"; +import type { PolicyEffect } from "../../src/schemas/policy-trace.ts"; + +const warnFinding = { + id: "F-001", + signature: "sig-confidence", + severity: "WARN", + category: "quality", + rule_id: "naming", + file: "src/x.ts", + line_start: 1, + line_end: 1, + message: "name is unclear", + details: "rename the value", + reviewer: { provider: "codex", model: "m", persona: "quality" }, + confidence: 0.2, + consensus: "singleton", +} satisfies Finding; + +const infoFinding = { + ...warnFinding, + id: "F-002", + signature: "sig-info", + severity: "INFO", + confidence: 0.9, +} satisfies Finding; + +function stripPolicyEffects(finding: Finding | null): Finding | null { + if (finding === null) return null; + const { policy_effects: _policyEffects, ...productionFinding } = finding; + return productionFinding as Finding; +} + +describe("transitionFinding production boundary", () => { + it("preserves the exact legacy mutation result when no runtime is supplied", () => { + const proposed = { ...warnFinding, severity: "INFO" as const, low_confidence: true }; + + const applied = transitionFinding({ + passId: "judgment.confidence", + finding: warnFinding, + opportunity: true, + matched: true, + reasonCode: "below-confidence-floor", + action: "demoted", + proposed: () => proposed, + }); + const missed = transitionFinding({ + passId: "judgment.confidence", + finding: warnFinding, + opportunity: true, + matched: false, + reasonCode: "below-confidence-floor", + action: "demoted", + proposed: () => { + throw new Error("a missed proposal must stay lazy"); + }, + }); + const protectedFinding = transitionFinding({ + passId: "judgment.confidence", + finding: warnFinding, + opportunity: true, + matched: true, + reasonCode: "below-confidence-floor", + action: "demoted", + protectedBy: "high-precision-reviewer", + proposed: () => { + throw new Error("a protected proposal must stay lazy"); + }, + }); + + expect(applied).toBe(proposed); + expect(missed).toBe(warnFinding); + expect(protectedFinding).toBe(warnFinding); + }); + + it("keeps opportunity, predicate, and proposal failures outside telemetry isolation", () => { + const opportunityError = new Error("opportunity failed"); + const predicateError = new Error("predicate failed"); + const proposalError = new Error("proposal failed"); + const runtime = PolicyTraceRecorder.start({ runId: "run-errors", iter: 1, ablated: [] }); + const base = { + runtime, + passId: "judgment.confidence", + finding: warnFinding, + opportunity: true, + matched: true, + reasonCode: "below-confidence-floor", + action: "demoted", + proposed: () => ({ ...warnFinding, severity: "INFO" as const }), + } satisfies TransitionInput; + const opportunityInput = { ...base } as TransitionInput; + const predicateInput = { ...base } as TransitionInput; + Object.defineProperty(opportunityInput, "opportunity", { + get: () => { + throw opportunityError; + }, + }); + Object.defineProperty(predicateInput, "matched", { + get: () => { + throw predicateError; + }, + }); + + expect(() => transitionFinding(opportunityInput)).toThrow(opportunityError); + expect(() => transitionFinding(predicateInput)).toThrow(predicateError); + expect(() => + transitionFinding({ + ...base, + proposed: () => { + throw proposalError; + }, + }), + ).toThrow(proposalError); + expect(runtime.telemetryError).toBe(false); + }); + + it("never lets an inconsistent telemetry opportunity suppress a matched production result", () => { + const runtime = PolicyTraceRecorder.start({ + runId: "run-inconsistent-opportunity", + iter: 1, + ablated: [], + }); + const proposed = { ...warnFinding, severity: "INFO" as const, low_confidence: true }; + + const after = transitionFinding({ + runtime, + passId: "judgment.confidence", + finding: warnFinding, + opportunity: false, + matched: true, + reasonCode: "below-confidence-floor", + action: "demoted", + proposed: () => proposed, + }); + + expect(after).toBe(proposed); + expect(runtime.telemetryError).toBe(true); + }); +}); + +describe("PolicyTraceRecorder terminal evaluations", () => { + it("records no-opportunity without evaluating the proposal", () => { + const runtime = PolicyTraceRecorder.start({ + runId: "run-no-opportunity", + iter: 1, + ablated: [], + }); + + const after = transitionFinding({ + runtime, + passId: "judgment.confidence", + finding: warnFinding, + opportunity: false, + matched: false, + reasonCode: "below-confidence-floor", + action: "demoted", + proposed: () => { + throw new Error("no-opportunity must not propose"); + }, + }); + + expect(after).toBe(warnFinding); + expect(runtime.summary("judgment.confidence")).toEqual({ + pass_id: "judgment.confidence", + status: "ran", + considered: 1, + opportunities: 0, + would_apply: 0, + applied: 0, + protected: 0, + blocking_removed: 0, + blocking_preserved: 0, + dropped: 0, + }); + expect(runtime.evaluations()).toEqual([ + { + pass_id: "judgment.confidence", + order: 140, + result: "no-opportunity", + before: "WARN", + after: "WARN", + reason_code: "ineligible-starting-state", + source_signatures: ["sig-confidence"], + }, + ]); + }); + + it("increments opportunity for an eligible predicate miss", () => { + const runtime = PolicyTraceRecorder.start({ runId: "run-no-match", iter: 1, ablated: [] }); + + const after = transitionFinding({ + runtime, + passId: "judgment.confidence", + finding: warnFinding, + opportunity: true, + matched: false, + reasonCode: "below-confidence-floor", + action: "demoted", + proposed: () => { + throw new Error("no-match must not propose"); + }, + }); + + expect(after).toBe(warnFinding); + expect(runtime.summary("judgment.confidence")).toMatchObject({ + considered: 1, + opportunities: 1, + would_apply: 0, + applied: 0, + }); + expect(runtime.evaluations()[0]?.result).toBe("no-match"); + expect(runtime.evaluations()[0]?.reason_code).toBe("predicate-miss"); + }); + + it("records an applied transition and exact blocking counters", () => { + const runtime = PolicyTraceRecorder.start({ runId: "run-applied", iter: 1, ablated: [] }); + const proposed = { ...warnFinding, severity: "INFO" as const, low_confidence: true }; + + const after = transitionFinding({ + runtime, + passId: "judgment.confidence", + finding: warnFinding, + opportunity: true, + matched: true, + reasonCode: "below-confidence-floor", + action: "demoted", + proposed: () => proposed, + }); + + expect(stripPolicyEffects(after)).toEqual(proposed); + expect(after?.policy_effects).toEqual([ + { + pass_id: "judgment.confidence", + order: 140, + action: "demoted", + before: "WARN", + after: "INFO", + reason_code: "below-confidence-floor", + source_signatures: ["sig-confidence"], + }, + ]); + expect(runtime.summary("judgment.confidence")).toEqual({ + pass_id: "judgment.confidence", + status: "ran", + considered: 1, + opportunities: 1, + would_apply: 1, + applied: 1, + protected: 0, + blocking_removed: 1, + blocking_preserved: 0, + dropped: 0, + }); + }); + + it("records a protected material effect while preserving production fields", () => { + const runtime = PolicyTraceRecorder.start({ runId: "run-protected", iter: 1, ablated: [] }); + + const after = transitionFinding({ + runtime, + passId: "judgment.confidence", + finding: warnFinding, + opportunity: true, + matched: true, + reasonCode: "below-confidence-floor", + action: "demoted", + protectedBy: "high-precision-reviewer", + proposed: () => { + throw new Error("protected transition must not propose"); + }, + }); + + expect(stripPolicyEffects(after)).toEqual(warnFinding); + expect(after?.policy_effects).toEqual([ + { + pass_id: "judgment.confidence", + order: 140, + action: "protected", + before: "WARN", + after: "WARN", + reason_code: "below-confidence-floor", + protected_by: "high-precision-reviewer", + source_signatures: ["sig-confidence"], + }, + ]); + expect(runtime.summary("judgment.confidence")).toMatchObject({ + considered: 1, + opportunities: 1, + would_apply: 1, + applied: 0, + protected: 1, + blocking_removed: 0, + blocking_preserved: 1, + }); + }); + + it("keeps an explicitly ablated match unchanged and records would-apply", () => { + const runtime = PolicyTraceRecorder.start({ + runId: "run-ablated", + iter: 1, + ablated: ["judgment.confidence"], + }); + let proposedCalls = 0; + + const after = transitionFinding({ + runtime, + passId: "judgment.confidence", + finding: warnFinding, + opportunity: true, + matched: true, + reasonCode: "below-confidence-floor", + action: "demoted", + proposed: () => { + proposedCalls += 1; + return { ...warnFinding, severity: "INFO", low_confidence: true }; + }, + }); + + expect(after).toBe(warnFinding); + expect(proposedCalls).toBe(1); + expect(after?.policy_effects).toBeUndefined(); + expect(runtime.evaluations()[0]?.result).toBe("would-apply"); + expect(runtime.summary("judgment.confidence")).toEqual({ + pass_id: "judgment.confidence", + status: "ran", + considered: 1, + opportunities: 1, + would_apply: 1, + applied: 0, + protected: 0, + blocking_removed: 0, + blocking_preserved: 1, + dropped: 0, + }); + }); + + it("records an applied drop without retaining a visible effect", () => { + const runtime = PolicyTraceRecorder.start({ runId: "run-drop", iter: 1, ablated: [] }); + + const after = transitionFinding({ + runtime, + passId: "judgment.critic", + finding: infoFinding, + opportunity: true, + matched: true, + reasonCode: "critic-likely-fp", + action: "dropped", + proposed: () => null, + }); + + expect(after).toBeNull(); + expect(runtime.evaluations()[0]).toMatchObject({ + result: "applied", + before: "INFO", + after: null, + }); + expect(runtime.summary("judgment.critic")).toMatchObject({ + considered: 1, + opportunities: 1, + would_apply: 1, + applied: 1, + blocking_removed: 0, + blocking_preserved: 0, + dropped: 1, + }); + }); + + it("records an applied re-anchor as blocking-preserving", () => { + const runtime = PolicyTraceRecorder.start({ runId: "run-reanchor", iter: 1, ablated: [] }); + const proposed = { + ...warnFinding, + line_start: 8, + line_end: 8, + anchor_repaired: true, + } satisfies Finding; + + const after = transitionFinding({ + runtime, + passId: "evidence.fact-location", + finding: warnFinding, + opportunity: true, + matched: true, + reasonCode: "evidence-line-reanchored", + action: "reanchored", + proposed: () => proposed, + }); + + expect(stripPolicyEffects(after)).toEqual(proposed); + expect(after?.policy_effects?.[0]?.action).toBe("reanchored"); + expect(runtime.summary("evidence.fact-location")).toMatchObject({ + applied: 1, + blocking_removed: 0, + blocking_preserved: 1, + }); + }); +}); + +describe("PolicyTraceRecorder fail-open telemetry", () => { + it("returns the exact proposed result when recorder or effect attachment fails", () => { + const runtime = PolicyTraceRecorder.start({ + runId: "run-recorder-error", + iter: 1, + ablated: [], + }); + const proposed = { ...warnFinding, reviewer: { ...warnFinding.reviewer } }; + + const after = transitionFinding({ + runtime, + passId: "judgment.confidence", + finding: warnFinding, + opportunity: true, + matched: true, + reasonCode: "below-confidence-floor", + action: "demoted", + proposed: () => proposed, + }); + + expect(after).toBe(proposed); + expect(after?.policy_effects).toBeUndefined(); + expect(runtime.telemetryError).toBe(true); + }); + + it("preserves a null production drop when telemetry recording fails", () => { + const runtime = PolicyTraceRecorder.start({ runId: "run-drop-error", iter: 1, ablated: [] }); + + const after = transitionFinding({ + runtime, + passId: "judgment.critic", + finding: infoFinding, + opportunity: true, + matched: true, + reasonCode: "critic-likely-fp", + action: "dropped", + sourceSignatures: [], + proposed: () => null, + }); + + expect(after).toBeNull(); + expect(runtime.telemetryError).toBe(true); + }); +}); + +describe("policy effect merging", () => { + it("deduplicates identical effects and restores ascending catalog order", () => { + const earlier = { + pass_id: "evidence.fact-location", + order: 10, + action: "reanchored", + before: "WARN", + after: "WARN", + reason_code: "evidence-line-reanchored", + source_signatures: ["sig-confidence"], + } satisfies PolicyEffect; + const later = { + pass_id: "judgment.confidence", + order: 140, + action: "demoted", + before: "WARN", + after: "INFO", + reason_code: "below-confidence-floor", + source_signatures: ["sig-confidence"], + } satisfies PolicyEffect; + + expect(mergePolicyEffects([later], [earlier, later], undefined)).toEqual([earlier, later]); + }); +}); + +describe("PolicyTraceRecorder finalization", () => { + it("links cluster lineage and derives ordered final severity evidence", () => { + const runtime = PolicyTraceRecorder.start({ runId: "run-final", iter: 2, ablated: [] }); + const finalWarn = { + ...warnFinding, + id: "F-010", + signature: "sig-z-final", + } satisfies Finding; + const finalInfo = { + ...infoFinding, + id: "F-011", + signature: "sig-a-final", + } satisfies Finding; + + transitionFinding({ + runtime, + passId: "judgment.confidence", + finding: { ...warnFinding, signature: "sig-member" }, + opportunity: true, + matched: false, + reasonCode: "below-confidence-floor", + action: "demoted", + sourceSignatures: ["sig-member"], + proposed: () => { + throw new Error("predicate miss must stay lazy"); + }, + }); + runtime.recordStage({ + stageId: "aggregation.cluster", + reasonCode: "clustered", + inputSignatures: ["sig-z-final", "sig-member"], + outputSignature: "sig-z-final", + }); + runtime.recordStage({ + stageId: "aggregation.cluster", + reasonCode: "singleton", + inputSignatures: ["sig-a-final"], + outputSignature: "sig-a-final", + }); + runtime.linkFinal(["sig-member", "sig-z-final"], "sig-z-final"); + runtime.linkFinal(["sig-a-final"], "sig-a-final"); + runtime.recordStage({ + stageId: "verdict.compute", + reasonCode: "blocking-present", + inputSignatures: ["sig-z-final"], + verdict: "SOFT-PASS", + }); + + const trace = runtime.finalize({ + rawResponseSha256: ["a".repeat(64)], + verdict: "SOFT-PASS", + finalFindings: [finalWarn, finalInfo], + }); + + expect(trace?.evaluations[0]?.final_signature).toBe("sig-z-final"); + expect(trace?.stages.map((stage) => [stage.order, stage.stage_id])).toEqual([ + [65, "aggregation.cluster"], + [65, "aggregation.cluster"], + [190, "verdict.compute"], + ]); + expect(trace?.final).toEqual({ + verdict: "SOFT-PASS", + counts: { critical: 0, warn: 1, info: 1 }, + finding_signatures: ["sig-z-final", "sig-a-final"], + finding_severities: [ + { signature: "sig-z-final", severity: "WARN" }, + { signature: "sig-a-final", severity: "INFO" }, + ], + }); + }); +}); From 8d2d414b781970c0873e66f5452c5577e0ec80b3 Mon Sep 17 00:00:00 2001 From: Codevena Date: Mon, 10 Aug 2026 04:37:09 +0200 Subject: [PATCH 25/55] fix(policy): enforce global response hash ordinals --- src/core/policy/response-hashes.ts | 19 +++++-------------- tests/unit/policy-response-hashes.test.ts | 17 +++++++++++++++++ 2 files changed, 22 insertions(+), 14 deletions(-) diff --git a/src/core/policy/response-hashes.ts b/src/core/policy/response-hashes.ts index 4926b44..5a691f3 100644 --- a/src/core/policy/response-hashes.ts +++ b/src/core/policy/response-hashes.ts @@ -6,18 +6,12 @@ export interface OrderedResponseHash { readonly sha256: string; } -function compareByteOrder(left: string, right: string): number { - if (left < right) return -1; - if (left > right) return 1; - return 0; -} - /** * Holds response digests in logical call order. The raw response is hashed at * the boundary and is never retained by this object. */ export class OrderedResponseHashes { - readonly #entries = new Map(); + readonly #entries = new Map(); record(kind: string, ordinal: number, rawText?: string): void { if (rawText === undefined) return; @@ -26,12 +20,11 @@ export class OrderedResponseHashes { throw new Error("response hash ordinal must be a non-negative safe integer"); } - const identity = JSON.stringify([kind, ordinal]); - if (this.#entries.has(identity)) { - throw new Error(`duplicate response hash call identity: ${kind}:${ordinal}`); + if (this.#entries.has(ordinal)) { + throw new Error(`duplicate response hash ordinal: ${ordinal}`); } - this.#entries.set(identity, { + this.#entries.set(ordinal, { kind, ordinal, sha256: createHash("sha256").update(Buffer.from(rawText, "utf8")).digest("hex"), @@ -40,9 +33,7 @@ export class OrderedResponseHashes { entries(): OrderedResponseHash[] { return [...this.#entries.values()] - .sort( - (left, right) => left.ordinal - right.ordinal || compareByteOrder(left.kind, right.kind), - ) + .sort((left, right) => left.ordinal - right.ordinal) .map((entry) => ({ ...entry })); } diff --git a/tests/unit/policy-response-hashes.test.ts b/tests/unit/policy-response-hashes.test.ts index 3cafd69..e56e7ab 100644 --- a/tests/unit/policy-response-hashes.test.ts +++ b/tests/unit/policy-response-hashes.test.ts @@ -31,6 +31,23 @@ describe("OrderedResponseHashes", () => { expect(JSON.stringify(hashes.entries())).not.toContain("secret-shaped"); }); + it("rejects a duplicate global ordinal even when call kinds differ", () => { + const hashes = new OrderedResponseHashes(); + + hashes.record("reviewer", 0, "first logical response"); + + expect(() => hashes.record("critic", 0, "different response at the same position")).toThrow( + "duplicate response hash ordinal: 0", + ); + expect(hashes.entries()).toEqual([ + { + kind: "reviewer", + ordinal: 0, + sha256: "c177496f3a8c709a22592ff68cd8ac6f41db2a48c26cf8a1241a6e079a3909c7", + }, + ]); + }); + it("distinguishes an empty successful response from no response", () => { const hashes = new OrderedResponseHashes(); const recordCall = (ordinal: number, call: () => string): void => { From 59eab8e5f5692d49bdd601372f7392e05e0ca805 Mon Sep 17 00:00:00 2001 From: Codevena Date: Mon, 10 Aug 2026 05:02:23 +0200 Subject: [PATCH 26/55] feat(policy): trace pre-aggregation decisions --- src/core/critic.ts | 16 +- src/core/fact-check.ts | 88 ++- src/core/grounding.ts | 95 +++- src/core/hypothetical-demote.ts | 39 +- src/core/self-refutation.ts | 40 +- tests/unit/critic-runner.test.ts | 31 ++ tests/unit/fact-check-reanchor.test.ts | 20 + tests/unit/fact-check.test.ts | 19 +- tests/unit/grounding-judge.test.ts | 60 ++ tests/unit/grounding.test.ts | 21 + tests/unit/hypothetical-demote.test.ts | 23 +- .../policy-preaggregation-contracts.test.ts | 527 ++++++++++++++++++ tests/unit/self-refutation.test.ts | 27 + 13 files changed, 947 insertions(+), 59 deletions(-) create mode 100644 tests/unit/policy-preaggregation-contracts.test.ts diff --git a/src/core/critic.ts b/src/core/critic.ts index d1ec892..dc71d2c 100644 --- a/src/core/critic.ts +++ b/src/core/critic.ts @@ -1,4 +1,5 @@ // src/core/critic.ts +import { createHash } from "node:crypto"; import { neutralizeInjectionMarkers } from "../diff/sanitizer.ts"; import type { CompleteOptions, ProviderAdapter } from "../providers/adapter-base.ts"; import type { Finding } from "../schemas/finding.ts"; @@ -12,6 +13,7 @@ export interface CriticVerdict { export interface CriticRunResult { map: Map; info: { provider: string; status: "ran" | "error" | "empty" | "misconfigured"; verdicts: number }; + rawResponseSha256?: string; } export function buildCriticPrompt(findings: Finding[]): string { @@ -59,6 +61,7 @@ export async function runCritic( const attemptLimit = Number.isSafeInteger(maxAttempts) && maxAttempts > 0 ? maxAttempts : 1; const prompt = buildCriticPrompt(findings); let finalStatus: "error" | "empty" = "empty"; + let rawResponseSha256: string | undefined; for (let attempt = 1; attempt <= attemptLimit; attempt++) { try { // Force reasoning OFF: the critic is a keep/demote classification that needs @@ -67,16 +70,25 @@ export async function runCritic( // coverage gaps in the alpha.12 benchmark). Providers that don't support the // flag ignore it. const text = await adapter.complete(prompt, { ...opts, disableReasoning: true }); + rawResponseSha256 = createHash("sha256").update(Buffer.from(text, "utf8")).digest("hex"); const map = parseCriticOutput(text); if (map.size > 0) { - return { map, info: { provider, status: "ran", verdicts: map.size } }; + return { + map, + info: { provider, status: "ran", verdicts: map.size }, + rawResponseSha256, + }; } finalStatus = "empty"; } catch { finalStatus = "error"; } } - return { map: new Map(), info: { provider, status: finalStatus, verdicts: 0 } }; + return { + map: new Map(), + info: { provider, status: finalStatus, verdicts: 0 }, + ...(rawResponseSha256 === undefined ? {} : { rawResponseSha256 }), + }; } // Locate the real `{"verdicts":[...]}` payload inside arbitrary model output. diff --git a/src/core/fact-check.ts b/src/core/fact-check.ts index 505d454..547c4b5 100644 --- a/src/core/fact-check.ts +++ b/src/core/fact-check.ts @@ -3,6 +3,7 @@ import { dirname, isAbsolute, join, relative } from "node:path"; import { neutralizeFences, neutralizeInjectionMarkers } from "../diff/sanitizer.ts"; import type { Finding } from "../schemas/finding.ts"; import { safeReadContained } from "../utils/safe-read.ts"; +import { type PolicyRuntime, transitionFinding } from "./policy/trace.ts"; // Deterministic finding fact-check — no LLM, no network. Two independent production // field reports hit the same trust-killer: a single reviewer emitted a 0.97/1.00 @@ -126,31 +127,57 @@ export function validateFindingFacts( findings: Finding[], repoRoot: string, deletedPaths: Set, + runtime?: PolicyRuntime, ): Finding[] { + const runtimeInput = runtime === undefined ? {} : { runtime }; let repoReal: string; try { repoReal = realpathSync(repoRoot); } catch { - return findings; // can't establish a safe root → demote nothing + if (runtime === undefined) return findings; // can't establish a safe root → demote nothing + return findings.map( + (f) => + transitionFinding({ + ...runtimeInput, + passId: "evidence.fact-location", + finding: f, + opportunity: false, + matched: false, + reasonCode: "location-out-of-range", + action: "demoted", + proposed: () => f, + }) ?? f, + ); } return findings.map((f) => { + const noOpportunity = (): Finding => + transitionFinding({ + ...runtimeInput, + passId: "evidence.fact-location", + finding: f, + opportunity: false, + matched: false, + reasonCode: "location-out-of-range", + action: "demoted", + proposed: () => f, + }) ?? f; const file = f.file; - if (!file || file === "." || deletedPaths.has(file)) return f; - if (f.line_start < 1) return f; + if (!file || file === "." || deletedPaths.has(file)) return noOpportunity(); + if (f.line_start < 1) return noOpportunity(); // Reject a path that escapes the repo BEFORE touching the filesystem. const abs = join(repoRoot, file); const rel = relative(repoRoot, abs); - if (rel.startsWith("..") || isAbsolute(rel)) return f; + if (rel.startsWith("..") || isAbsolute(rel)) return noOpportunity(); // Realpath-contain the PARENT directory (catches intermediate-symlink escape that // a final-component lstat would miss); then validate the leaf inside it. let parentReal: string; try { parentReal = realpathSync(dirname(abs)); } catch { - return f; // parent unresolved (absent dir / perm) → can't prove anything + return noOpportunity(); // parent unresolved (absent dir / perm) → can't prove anything } const parentRel = relative(repoReal, parentReal); - if (parentRel.startsWith("..") || isAbsolute(parentRel)) return f; // escapes repo + if (parentRel.startsWith("..") || isAbsolute(parentRel)) return noOpportunity(); // escapes repo const leaf = join(parentReal, file.slice(file.lastIndexOf("/") + 1)); // Open with O_NOFOLLOW so a symlink-swapped leaf fails CLOSED (ELOOP) instead of // following OUT of the repo, then fstat + read THROUGH the same fd — no path @@ -162,17 +189,17 @@ export function validateFindingFacts( try { fd = openSync(leaf, constants.O_RDONLY | constants.O_NOFOLLOW); } catch { - return f; + return noOpportunity(); } let text: string; try { const st = fstatSync(fd); - if (!st.isFile() || st.size > MAX_READ_BYTES) return f; // dir/special/oversize → skip + if (!st.isFile() || st.size > MAX_READ_BYTES) return noOpportunity(); // dir/special/oversize → skip const buf = Buffer.alloc(st.size); if (st.size > 0) readSync(fd, buf, 0, st.size, 0); text = buf.toString("utf8"); } catch { - return f; // unreadable (e.g. binary perms) → fail-safe + return noOpportunity(); // unreadable (e.g. binary perms) → fail-safe } finally { try { closeSync(fd); @@ -181,16 +208,53 @@ export function validateFindingFacts( } } const lines = lineCount(text); - if (f.line_start <= lines) return f; // cited line exists → real finding, untouched + if (f.line_start <= lines) { + return ( + transitionFinding({ + ...runtimeInput, + passId: "evidence.fact-location", + finding: f, + opportunity: true, + matched: false, + reasonCode: "location-out-of-range", + action: "demoted", + proposed: () => f, + }) ?? f + ); + } // Out of range. Before calling it a fabrication, consult the reviewer's OWN quoted evidence — // f.evidence_line is UNTRUSTED reviewer-supplied input, so this is a match against real file // content already read above, never a trust decision made from the quote alone: a quote that // matches a real line of THIS file proves the reviewer read the code and mis-numbered it, // which is not what this pass exists to catch. const repaired = reanchorByEvidence(f, text); - if (repaired !== null) return repaired; + if (repaired !== null) { + return ( + transitionFinding({ + ...runtimeInput, + passId: "evidence.fact-location", + finding: f, + opportunity: true, + matched: true, + reasonCode: "evidence-line-reanchored", + action: "reanchored", + proposed: () => repaired, + }) ?? f + ); + } const note = `\n\n[reviewgate fact-check] cited location ${file}:${f.line_start} does not exist in the working tree (file has ${lines} line${lines === 1 ? "" : "s"}) — almost certainly hallucinated; demoted to advisory. Verify before treating as real.`; - return demote(f, note); + return ( + transitionFinding({ + ...runtimeInput, + passId: "evidence.fact-location", + finding: f, + opportunity: true, + matched: true, + reasonCode: "location-out-of-range", + action: "demoted", + proposed: () => demote(f, note), + }) ?? f + ); }); } diff --git a/src/core/grounding.ts b/src/core/grounding.ts index 81b095e..ea2ec0c 100644 --- a/src/core/grounding.ts +++ b/src/core/grounding.ts @@ -1,7 +1,9 @@ +import { createHash } from "node:crypto"; import { neutralizeInjectionMarkers, sanitizeDiff } from "../diff/sanitizer.ts"; import type { CompleteOptions, ProviderAdapter } from "../providers/adapter-base.ts"; import type { Finding } from "../schemas/finding.ts"; import { safeJsonParse } from "../utils/safe-json.ts"; +import { type PolicyRuntime, transitionFinding } from "./policy/trace.ts"; // S6 grounding (layer 1) — deterministic, no LLM. A reviewer occasionally fabricates // a CRITICAL by inventing a code fact (field report 2026-06-03: F-003 claimed a @@ -82,22 +84,45 @@ function isSecurityOrCorrectness(f: Finding): boolean { // HIGH-precision — ANY absent one is almost certainly fabricated, so it triggers. Dotted/ // backtick code refs are LOWER-precision (a real finding may cite a present core symbol plus // an incidental absent one), so they only trigger when ALL are absent. -export function groundFindings(findings: Finding[], corpus: string): Finding[] { +export function groundFindings( + findings: Finding[], + corpus: string, + runtime?: PolicyRuntime, +): Finding[] { + const runtimeInput = runtime === undefined ? {} : { runtime }; return findings.map((f) => { - if (f.severity !== "CRITICAL" || isSecurityOrCorrectness(f)) return f; - const { cssVars, codeRefs } = citedTokens(`${f.message} ${f.details}`); - if (cssVars.length === 0 && codeRefs.length === 0) return f; + const critical = f.severity === "CRITICAL"; + const { cssVars, codeRefs } = critical + ? citedTokens(`${f.message} ${f.details}`) + : { cssVars: [], codeRefs: [] }; + const opportunity = critical && (cssVars.length > 0 || codeRefs.length > 0); const cssAbsent = cssVars.filter((t) => !corpus.includes(t)); const refsAbsent = codeRefs.filter((t) => !corpus.includes(t)); const allRefsAbsent = codeRefs.length > 0 && refsAbsent.length === codeRefs.length; - if (cssAbsent.length === 0 && !allRefsAbsent) return f; + const matched = opportunity && (cssAbsent.length > 0 || allRefsAbsent); const absent = [...cssAbsent, ...(allRefsAbsent ? refsAbsent : [])]; - const note = `\n\n↓ grounding: cites ${absent - .map((t) => `\`${t}\``) - .join( - ", ", - )} not found in the reviewed code — likely fabricated; demoted to advisory. Verify before treating as real.`; - return groundingDemote(f, note); + const protectedBy = + matched && isSecurityOrCorrectness(f) ? ("security-correctness-floor" as const) : undefined; + return ( + transitionFinding({ + ...runtimeInput, + passId: "evidence.grounding-token", + finding: f, + opportunity, + matched, + reasonCode: "cited-token-absent", + action: "demoted", + ...(protectedBy === undefined ? {} : { protectedBy }), + proposed: () => { + const note = `\n\n↓ grounding: cites ${absent + .map((t) => `\`${t}\``) + .join( + ", ", + )} not found in the reviewed code — likely fabricated; demoted to advisory. Verify before treating as real.`; + return groundingDemote(f, note); + }, + }) ?? f + ); }); } @@ -214,7 +239,11 @@ export async function judgeGrounding( opts: CompleteOptions, findings: Finding[], corpus: string, -): Promise<{ map: Map; status: GroundingJudgeStatus }> { +): Promise<{ + map: Map; + status: GroundingJudgeStatus; + rawResponseSha256?: string; +}> { const criticals = findings.filter((f) => f.severity === "CRITICAL"); if (criticals.length === 0) return { map: new Map(), status: "skipped" }; if (typeof adapter.complete !== "function") return { map: new Map(), status: "misconfigured" }; @@ -224,8 +253,9 @@ export async function judgeGrounding( } catch { return { map: new Map(), status: "error" }; } + const rawResponseSha256 = createHash("sha256").update(Buffer.from(text, "utf8")).digest("hex"); const map = parseGroundingOutput(text); - return { map, status: map.size > 0 ? "ran" : "empty" }; + return { map, status: map.size > 0 ? "ran" : "empty", rawResponseSha256 }; } // Demote-only, CRITICAL-only, fail-safe. A CRITICAL the judge marked grounded:false → @@ -241,17 +271,36 @@ export async function judgeGrounding( export function applyGroundingJudgeVerdicts( findings: Finding[], map: Map, + runtime?: PolicyRuntime, ): Finding[] { + const runtimeInput = runtime === undefined ? {} : { runtime }; return findings.map((f) => { - if (f.severity !== "CRITICAL" || isSecurityOrCorrectness(f)) return f; - const v = map.get(f.signature); - if (!v || v.grounded !== false) return f; - // Bound the UNTRUSTED judge reason so truncation lands on it (not on the finding's own - // details) and the note stays well within the 2000-char cap; groundingDemote caps again - // as a backstop. - const note = `\n\n↓ grounding judge: the claim is not supported by the reviewed code${ - v.reason ? ` — ${v.reason.slice(0, 300)}` : "" - }; likely fabricated, demoted to advisory.`; - return groundingDemote(f, note); + const critical = f.severity === "CRITICAL"; + const verdict = critical ? map.get(f.signature) : undefined; + const opportunity = critical && verdict !== undefined; + const matched = opportunity && verdict.grounded === false; + const protectedBy = + matched && isSecurityOrCorrectness(f) ? ("security-correctness-floor" as const) : undefined; + return ( + transitionFinding({ + ...runtimeInput, + passId: "judgment.grounding-llm", + finding: f, + opportunity, + matched, + reasonCode: "judge-ungrounded", + action: "demoted", + ...(protectedBy === undefined ? {} : { protectedBy }), + proposed: () => { + // Bound the UNTRUSTED judge reason so truncation lands on it (not on the finding's own + // details) and the note stays well within the 2000-char cap; groundingDemote caps again + // as a backstop. + const note = `\n\n↓ grounding judge: the claim is not supported by the reviewed code${ + verdict?.reason ? ` — ${verdict.reason.slice(0, 300)}` : "" + }; likely fabricated, demoted to advisory.`; + return groundingDemote(f, note); + }, + }) ?? f + ); }); } diff --git a/src/core/hypothetical-demote.ts b/src/core/hypothetical-demote.ts index 4e0e387..3fd343a 100644 --- a/src/core/hypothetical-demote.ts +++ b/src/core/hypothetical-demote.ts @@ -1,4 +1,5 @@ import type { Finding } from "../schemas/finding.ts"; +import { type PolicyRuntime, transitionFinding } from "./policy/trace.ts"; // #2 severity floor (field report 2026-06-17 non-convergence): a CRITICAL must describe a // PRESENT, demonstrable defect. A reviewer that concedes the code is "currently safe" yet raises @@ -51,16 +52,36 @@ const NOTE = * (currently-safe / hypothetical / future-conditional) and asserts no present-defect backstop. * Demote-only, positive-signal, security/correctness-exempt, fail-safe. */ -export function demoteHypotheticalCriticals(findings: Finding[], enabled = true): Finding[] { +export function demoteHypotheticalCriticals( + findings: Finding[], + enabled = true, + runtime?: PolicyRuntime, +): Finding[] { if (!enabled) return findings; + const runtimeInput = runtime === undefined ? {} : { runtime }; return findings.map((f) => { - if (f.severity !== "CRITICAL") return f; // CRITICAL-only - if (f.deterministic) return f; // check-tier ground truth — never demote - // Never soften the hard-veto categories on an untrusted text signal (mirror self-refutation). - if (f.category === "security" || f.category === "correctness") return f; - const text = `${f.message}\n${f.details}\n${f.suggested_fix ?? ""}`; - if (!HYPOTHETICAL.test(text)) return f; // positive marker required - if (PRESENT_DEFECT.test(text)) return f; // also asserts a present defect → stays CRITICAL - return demote(f, NOTE); + const opportunity = f.severity === "CRITICAL"; + const text = opportunity ? `${f.message}\n${f.details}\n${f.suggested_fix ?? ""}` : ""; + const matched = opportunity && HYPOTHETICAL.test(text) && !PRESENT_DEFECT.test(text); + const protectedBy = !matched + ? undefined + : f.deterministic + ? ("deterministic-ground-truth" as const) + : f.category === "security" || f.category === "correctness" + ? ("security-correctness-floor" as const) + : undefined; + return ( + transitionFinding({ + ...runtimeInput, + passId: "judgment.hypothetical", + finding: f, + opportunity, + matched, + reasonCode: "hypothetical-critical", + action: "demoted", + ...(protectedBy === undefined ? {} : { protectedBy }), + proposed: () => demote(f, NOTE), + }) ?? f + ); }); } diff --git a/src/core/self-refutation.ts b/src/core/self-refutation.ts index 8ddc33d..eb318bc 100644 --- a/src/core/self-refutation.ts +++ b/src/core/self-refutation.ts @@ -1,4 +1,5 @@ import type { Finding } from "../schemas/finding.ts"; +import { type PolicyRuntime, transitionFinding } from "./policy/trace.ts"; // Deterministic self-refutation filter — no LLM, no network. A reviewer frequently // investigates a concern, narrates the analysis, and CONCLUDES the code is fine ("This @@ -114,18 +115,35 @@ const NOTE = * "No defect", "Safe.") to INFO (advisory). Demote-only, positive-signal, fail-safe. * Skips deterministic check-tier findings and findings already at INFO (idempotent). */ -export function demoteSelfRefuting(findings: Finding[], enabled = true): Finding[] { +export function demoteSelfRefuting( + findings: Finding[], + enabled = true, + runtime?: PolicyRuntime, +): Finding[] { if (!enabled) return findings; + const runtimeInput = runtime === undefined ? {} : { runtime }; return findings.map((f) => { - if (f.severity === "INFO") return f; // already advisory — idempotent no-op - if (f.deterministic) return f; // check-tier ground truth — never demote - // Never soften a security/correctness finding on the reviewer's own untrusted prose - // (dogfood DoD: a confused/injected reviewer could retract a real vuln). Hard-veto - // categories stay blocking — the agent dispositions them with a decision instead. - if (f.category === "security" || f.category === "correctness") return f; - if (isSelfRefutingText(f.message) || isSelfRefutingText(f.details)) { - return demote(f, NOTE); - } - return f; + const opportunity = f.severity !== "INFO"; + const matched = opportunity && (isSelfRefutingText(f.message) || isSelfRefutingText(f.details)); + const protectedBy = !matched + ? undefined + : f.deterministic + ? ("deterministic-ground-truth" as const) + : f.category === "security" || f.category === "correctness" + ? ("security-correctness-floor" as const) + : undefined; + return ( + transitionFinding({ + ...runtimeInput, + passId: "evidence.self-refutation", + finding: f, + opportunity, + matched, + reasonCode: "terminal-self-refutation", + action: "demoted", + ...(protectedBy === undefined ? {} : { protectedBy }), + proposed: () => demote(f, NOTE), + }) ?? f + ); }); } diff --git a/tests/unit/critic-runner.test.ts b/tests/unit/critic-runner.test.ts index 3a4ed33..35d3e35 100644 --- a/tests/unit/critic-runner.test.ts +++ b/tests/unit/critic-runner.test.ts @@ -31,6 +31,37 @@ function mkFinding(over: Partial = {}): Finding { const OPTS: CompleteOptions = { model: "m" }; describe("runCritic", () => { + it("returns only the SHA-256 of the successful raw critic response", async () => { + const raw = '{"verdicts":[{"signature":"sig-hash","verdict":"keep"}]}'; + const result = await runCritic({ complete: async () => raw }, "codex", OPTS, [ + mkFinding({ signature: "sig-hash" }), + ]); + + expect(result.rawResponseSha256).toBe( + "ff83aa82e9f5568766a85df650d31931478f29a3af43523571c266882344d312", + ); + expect(Object.values(result)).not.toContain(raw); + }); + + it("hashes an empty successful critic response but not a run with only thrown calls", async () => { + const empty = await runCritic({ complete: async () => "" }, "codex", OPTS, [mkFinding()]); + const failed = await runCritic( + { + complete: async () => { + throw new Error("boom"); + }, + }, + "codex", + OPTS, + [mkFinding()], + ); + + expect(empty.rawResponseSha256).toBe( + "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + ); + expect(failed.rawResponseSha256).toBeUndefined(); + }); + it("uses complete() and returns the critic verdict map", async () => { const adapter: Pick = { complete: async () => diff --git a/tests/unit/fact-check-reanchor.test.ts b/tests/unit/fact-check-reanchor.test.ts index 13c5372..dd73151 100644 --- a/tests/unit/fact-check-reanchor.test.ts +++ b/tests/unit/fact-check-reanchor.test.ts @@ -10,6 +10,7 @@ import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { validateFindingFacts } from "../../src/core/fact-check.ts"; +import { PolicyTraceRecorder } from "../../src/core/policy/trace.ts"; import type { Finding } from "../../src/schemas/finding.ts"; // Temp dirs created by repo(), removed after the run whether tests pass or fail. @@ -56,10 +57,12 @@ describe("validateFindingFacts — mis-anchored vs fabricated", () => { // WITH it: CRITICAL kept at the quoted line 3 + anchor_repaired -> 1 blocking. it("re-anchors an out-of-range finding whose evidence_line matches a real line", () => { const dir = repo(FIVE_LINES); + const runtime = PolicyTraceRecorder.start({ runId: "fact-reanchor", iter: 1, ablated: [] }); const out = validateFindingFacts( [mkFinding({ line_start: 67, line_end: 67, evidence_line: EVIDENCE })], dir, new Set(), + runtime, ); expect(out[0]?.severity).toBe("CRITICAL"); expect(out[0]?.line_start).toBe(3); @@ -67,6 +70,23 @@ describe("validateFindingFacts — mis-anchored vs fabricated", () => { expect(out[0]?.anchor_repaired).toBe(true); expect(out[0]?.fact_invalid).toBeUndefined(); expect(out[0]?.details).toContain("re-anchored"); + expect(out[0]?.policy_effects?.[0]).toEqual({ + pass_id: "evidence.fact-location", + order: 10, + action: "reanchored", + before: "CRITICAL", + after: "CRITICAL", + reason_code: "evidence-line-reanchored", + source_signatures: ["sig1"], + }); + expect(runtime.summary("evidence.fact-location")).toMatchObject({ + considered: 1, + opportunities: 1, + would_apply: 1, + applied: 1, + blocking_removed: 0, + blocking_preserved: 1, + }); }); // GUARD 2 (passes on current code — MUTATION-CHECKED in Step 3). diff --git a/tests/unit/fact-check.test.ts b/tests/unit/fact-check.test.ts index 6e602e8..06498e9 100644 --- a/tests/unit/fact-check.test.ts +++ b/tests/unit/fact-check.test.ts @@ -4,6 +4,7 @@ import { mkdtempSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { validateFindingFacts } from "../../src/core/fact-check.ts"; +import { PolicyTraceRecorder } from "../../src/core/policy/trace.ts"; import type { Finding } from "../../src/schemas/finding.ts"; function mkFinding(over: Partial = {}): Finding { @@ -33,8 +34,9 @@ function repo(): string { } describe("validateFindingFacts", () => { - it("demotes a CRITICAL citing a line in an EMPTY file (line out of range)", () => { + it("demotes a security CRITICAL citing an EMPTY file and records no category protection", () => { const dir = repo(); + const runtime = PolicyTraceRecorder.start({ runId: "fact-security", iter: 1, ablated: [] }); const out = validateFindingFacts( [ mkFinding({ @@ -47,9 +49,24 @@ describe("validateFindingFacts", () => { ], dir, new Set(), + runtime, ); expect(out[0]).toMatchObject({ severity: "INFO", fact_invalid: true }); expect(out[0]?.details).toContain("fact-check"); + expect(runtime.summary("evidence.fact-location")).toMatchObject({ + considered: 1, + opportunities: 1, + would_apply: 1, + applied: 1, + protected: 0, + blocking_removed: 1, + }); + expect(out[0]?.policy_effects?.[0]).toMatchObject({ + pass_id: "evidence.fact-location", + order: 10, + action: "demoted", + reason_code: "location-out-of-range", + }); }); it("demotes a CRITICAL whose line is beyond the file's length (out of range)", () => { diff --git a/tests/unit/grounding-judge.test.ts b/tests/unit/grounding-judge.test.ts index 3d548cc..ba0da06 100644 --- a/tests/unit/grounding-judge.test.ts +++ b/tests/unit/grounding-judge.test.ts @@ -5,6 +5,7 @@ import { judgeGrounding, parseGroundingOutput, } from "../../src/core/grounding.ts"; +import { PolicyTraceRecorder } from "../../src/core/policy/trace.ts"; import { type Finding, FindingSchema } from "../../src/schemas/finding.ts"; function mk(over: Partial = {}): Finding { @@ -100,6 +101,45 @@ describe("grounding judge (S6 layer 2)", () => { expect(captured).not.toContain("sig-warn"); }); + it("judgeGrounding returns only the SHA-256 of a successful raw response", async () => { + const raw = '{"verdicts":[{"signature":"sig-hash","grounded":true}]}'; + const result = await judgeGrounding( + { complete: async () => raw }, + { model: "x" }, + [mk({ signature: "sig-hash", severity: "CRITICAL" })], + CORPUS, + ); + + expect(result.rawResponseSha256).toBe( + "1a6fad758d64c4f258f1d635bfaead430ba8b8782fc0ce8bf411c6e8319bbc23", + ); + expect(Object.values(result)).not.toContain(raw); + }); + + it("judgeGrounding hashes an empty successful response but not a thrown call", async () => { + const empty = await judgeGrounding( + { complete: async () => "" }, + { model: "x" }, + [mk({ severity: "CRITICAL" })], + CORPUS, + ); + const failed = await judgeGrounding( + { + complete: async () => { + throw new Error("boom"); + }, + }, + { model: "x" }, + [mk({ severity: "CRITICAL" })], + CORPUS, + ); + + expect(empty.rawResponseSha256).toBe( + "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + ); + expect(failed.rawResponseSha256).toBeUndefined(); + }); + it("judgeGrounding is fail-safe: a thrown adapter yields an empty map + error status", async () => { const adapter = { complete: async () => { @@ -159,12 +199,32 @@ describe("grounding judge (S6 layer 2)", () => { }); it("G0: EXEMPTS a correctness CRITICAL even when the judge marks it ungrounded", () => { + const runtime = PolicyTraceRecorder.start({ + runId: "grounding-llm-protected", + iter: 1, + ablated: [], + }); const out = applyGroundingJudgeVerdicts( [mk({ signature: "s1", severity: "CRITICAL", category: "correctness" })], new Map([["s1", { grounded: false, reason: "value not present" }]]), + runtime, ); expect(out[0]?.severity).toBe("CRITICAL"); expect(out[0]?.grounding_demoted).toBeUndefined(); + expect(out[0]?.policy_effects?.[0]).toMatchObject({ + pass_id: "judgment.grounding-llm", + order: 50, + action: "protected", + reason_code: "judge-ungrounded", + protected_by: "security-correctness-floor", + }); + expect(runtime.summary("judgment.grounding-llm")).toMatchObject({ + opportunities: 1, + would_apply: 1, + applied: 0, + protected: 1, + blocking_preserved: 1, + }); }); // A security CRITICAL clustered under a non-security representative must also be exempt — diff --git a/tests/unit/grounding.test.ts b/tests/unit/grounding.test.ts index 14320fd..fca776b 100644 --- a/tests/unit/grounding.test.ts +++ b/tests/unit/grounding.test.ts @@ -1,5 +1,6 @@ import { describe, expect, it } from "bun:test"; import { groundFindings } from "../../src/core/grounding.ts"; +import { PolicyTraceRecorder } from "../../src/core/policy/trace.ts"; import type { Finding } from "../../src/schemas/finding.ts"; function mk(over: Partial = {}): Finding { @@ -122,12 +123,32 @@ describe("groundFindings (S6 layer 1)", () => { // is layer 2's job, which reads the actual code). Otherwise an absent token in // attacker-influenced finding text is a fail-open. it("NEVER demotes a security CRITICAL, even citing an absent token (exempt — fail-open guard)", () => { + const runtime = PolicyTraceRecorder.start({ + runId: "grounding-token-protected", + iter: 1, + ablated: [], + }); const out = groundFindings( [mk({ category: "security", details: "leak via --ghost-token" })], CORPUS, + runtime, ); expect(out[0]?.severity).toBe("CRITICAL"); expect(out[0]?.grounding_demoted).toBeUndefined(); + expect(out[0]?.policy_effects?.[0]).toMatchObject({ + pass_id: "evidence.grounding-token", + order: 40, + action: "protected", + reason_code: "cited-token-absent", + protected_by: "security-correctness-floor", + }); + expect(runtime.summary("evidence.grounding-token")).toMatchObject({ + opportunities: 1, + would_apply: 1, + applied: 0, + protected: 1, + blocking_preserved: 1, + }); }); it("NEVER demotes a correctness CRITICAL citing an absent token (exempt)", () => { diff --git a/tests/unit/hypothetical-demote.test.ts b/tests/unit/hypothetical-demote.test.ts index 182969e..4c34edf 100644 --- a/tests/unit/hypothetical-demote.test.ts +++ b/tests/unit/hypothetical-demote.test.ts @@ -3,6 +3,7 @@ // as currently-safe / hypothetical / future fragility one step to WARN. Fail-safe. import { describe, expect, it } from "bun:test"; import { demoteHypotheticalCriticals } from "../../src/core/hypothetical-demote.ts"; +import { PolicyTraceRecorder } from "../../src/core/policy/trace.ts"; import { demoteSelfRefuting } from "../../src/core/self-refutation.ts"; import type { Finding } from "../../src/schemas/finding.ts"; @@ -56,11 +57,31 @@ describe("demoteHypotheticalCriticals — NEGATIVE (stays CRITICAL)", () => { expect(out?.severity).toBe("CRITICAL"); }); it("EXEMPT security even with a hypothetical marker", () => { - const out = one({ + const finding = f({ category: "security", details: "Currently safe, but a future change could leak the token.", }); + const runtime = PolicyTraceRecorder.start({ + runId: "hypothetical-protected", + iter: 1, + ablated: [], + }); + const out = demoteHypotheticalCriticals([finding], true, runtime)[0]; expect(out?.severity).toBe("CRITICAL"); + expect(out?.policy_effects?.[0]).toMatchObject({ + pass_id: "judgment.hypothetical", + order: 30, + action: "protected", + reason_code: "hypothetical-critical", + protected_by: "security-correctness-floor", + }); + expect(runtime.summary("judgment.hypothetical")).toMatchObject({ + opportunities: 1, + would_apply: 1, + applied: 0, + protected: 1, + blocking_preserved: 1, + }); }); it("EXEMPT correctness even with a hypothetical marker", () => { const out = one({ diff --git a/tests/unit/policy-preaggregation-contracts.test.ts b/tests/unit/policy-preaggregation-contracts.test.ts new file mode 100644 index 0000000..43d8a23 --- /dev/null +++ b/tests/unit/policy-preaggregation-contracts.test.ts @@ -0,0 +1,527 @@ +import { afterAll, describe, expect, it } from "bun:test"; +import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { validateFindingFacts } from "../../src/core/fact-check.ts"; +import { + type GroundingVerdict, + applyGroundingJudgeVerdicts, + groundFindings, +} from "../../src/core/grounding.ts"; +import { demoteHypotheticalCriticals } from "../../src/core/hypothetical-demote.ts"; +import type { + PolicyPassId, + PolicyProtectionCode, + PolicyReasonCode, +} from "../../src/core/policy/catalog.ts"; +import { type PolicyRuntime, PolicyTraceRecorder } from "../../src/core/policy/trace.ts"; +import { demoteSelfRefuting } from "../../src/core/self-refutation.ts"; +import type { Finding } from "../../src/schemas/finding.ts"; + +type NumericSummary = readonly [number, number, number, number, number, number, number, number]; + +const createdDirs: string[] = []; + +afterAll(() => { + for (const dir of createdDirs) rmSync(dir, { recursive: true, force: true }); +}); + +function mkFinding(overrides: Partial = {}): Finding { + return { + id: "F-001", + signature: "sig-policy", + severity: "WARN", + category: "quality", + rule_id: "policy-contract", + file: "src/x.ts", + line_start: 1, + line_end: 1, + message: "A concrete policy finding", + details: "The implementation has a concrete defect.", + reviewer: { provider: "codex", model: "m", persona: "quality" }, + confidence: 0.9, + consensus: "singleton", + ...overrides, + }; +} + +function runtime(runId: string, ablated: readonly PolicyPassId[] = []): PolicyTraceRecorder { + return PolicyTraceRecorder.start({ runId, iter: 1, ablated }); +} + +function stripPolicyEffects(finding: Finding | undefined): Finding | undefined { + if (finding === undefined) return undefined; + const { policy_effects: _policyEffects, ...legacy } = finding; + return legacy as Finding; +} + +function numericSummary(recorder: PolicyRuntime, passId: PolicyPassId): NumericSummary { + const summary = recorder.summary(passId); + expect(summary.status).toBe("ran"); + if (summary.status !== "ran") throw new Error(`${passId} did not run`); + return [ + summary.considered, + summary.opportunities, + summary.would_apply, + summary.applied, + summary.protected, + summary.blocking_removed, + summary.blocking_preserved, + summary.dropped, + ]; +} + +function expectEffect( + finding: Finding | undefined, + expected: { + pass_id: PolicyPassId; + order: number; + action: "demoted" | "protected" | "reanchored"; + before: Finding["severity"]; + after: Finding["severity"]; + reason_code: PolicyReasonCode; + protected_by?: PolicyProtectionCode; + }, +): void { + if (finding === undefined) throw new Error("expected one finding"); + expect(finding?.policy_effects).toEqual([ + { + ...expected, + source_signatures: [finding.signature], + }, + ]); +} + +function factRepo(): string { + const dir = mkdtempSync(join(tmpdir(), "reviewgate-policy-preaggregation-")); + createdDirs.push(dir); + writeFileSync(join(dir, "one-line.ts"), "const present = true;\n"); + return dir; +} + +describe("pre-aggregation policy numeric contracts", () => { + it("records fact-location no-opportunity and predicate-miss rows", () => { + const dir = factRepo(); + const absentRuntime = runtime("fact-absent"); + const validRuntime = runtime("fact-valid"); + + validateFindingFacts( + [mkFinding({ file: "absent.ts", line_start: 9, line_end: 9 })], + dir, + new Set(), + absentRuntime, + ); + validateFindingFacts( + [mkFinding({ file: "one-line.ts", line_start: 1, line_end: 1 })], + dir, + new Set(), + validRuntime, + ); + + expect(numericSummary(absentRuntime, "evidence.fact-location")).toEqual([ + 1, 0, 0, 0, 0, 0, 0, 0, + ]); + expect(numericSummary(validRuntime, "evidence.fact-location")).toEqual([ + 1, 1, 0, 0, 0, 0, 0, 0, + ]); + }); + + it("records fact-location active, ablated, and re-anchor actions without legacy drift", () => { + const dir = factRepo(); + const finding = mkFinding({ + signature: "sig-fact-demote", + file: "one-line.ts", + line_start: 9, + line_end: 9, + }); + const activeRuntime = runtime("fact-active"); + const ablatedRuntime = runtime("fact-ablated", ["evidence.fact-location"]); + + const legacy = validateFindingFacts([finding], dir, new Set()); + const active = validateFindingFacts([finding], dir, new Set(), activeRuntime); + const ablated = validateFindingFacts([finding], dir, new Set(), ablatedRuntime); + + expect(stripPolicyEffects(active[0])).toEqual(legacy[0]); + expect(active[0]).toMatchObject({ severity: "INFO", fact_invalid: true }); + expect(ablated[0]).toEqual(finding); + expect(numericSummary(activeRuntime, "evidence.fact-location")).toEqual([ + 1, 1, 1, 1, 0, 1, 0, 0, + ]); + expect(numericSummary(ablatedRuntime, "evidence.fact-location")).toEqual([ + 1, 1, 1, 0, 0, 0, 1, 0, + ]); + expectEffect(active[0], { + pass_id: "evidence.fact-location", + order: 10, + action: "demoted", + before: "WARN", + after: "INFO", + reason_code: "location-out-of-range", + }); + + const reanchorFinding = mkFinding({ + signature: "sig-fact-reanchor", + severity: "CRITICAL", + file: "one-line.ts", + line_start: 99, + line_end: 99, + evidence_line: "const present = true;", + }); + const reanchorRuntime = runtime("fact-reanchor"); + const reanchored = validateFindingFacts([reanchorFinding], dir, new Set(), reanchorRuntime); + + expect(reanchored[0]).toMatchObject({ + severity: "CRITICAL", + line_start: 1, + line_end: 1, + anchor_repaired: true, + }); + expect(numericSummary(reanchorRuntime, "evidence.fact-location")).toEqual([ + 1, 1, 1, 1, 0, 0, 1, 0, + ]); + expectEffect(reanchored[0], { + pass_id: "evidence.fact-location", + order: 10, + action: "reanchored", + before: "CRITICAL", + after: "CRITICAL", + reason_code: "evidence-line-reanchored", + }); + }); + + it("records self-refutation no-opportunity and predicate-miss rows", () => { + const infoRuntime = runtime("self-info"); + const ordinaryRuntime = runtime("self-ordinary"); + + demoteSelfRefuting([mkFinding({ severity: "INFO", details: "No issue." })], true, infoRuntime); + demoteSelfRefuting([mkFinding({ severity: "WARN" })], true, ordinaryRuntime); + + expect(numericSummary(infoRuntime, "evidence.self-refutation")).toEqual([ + 1, 0, 0, 0, 0, 0, 0, 0, + ]); + expect(numericSummary(ordinaryRuntime, "evidence.self-refutation")).toEqual([ + 1, 1, 0, 0, 0, 0, 0, 0, + ]); + }); + + it("records self-refutation active, ablated, and closed protections without legacy drift", () => { + const finding = mkFinding({ signature: "sig-self", details: "Checked carefully. No issue." }); + const activeRuntime = runtime("self-active"); + const ablatedRuntime = runtime("self-ablated", ["evidence.self-refutation"]); + + const legacy = demoteSelfRefuting([finding]); + const active = demoteSelfRefuting([finding], true, activeRuntime); + const ablated = demoteSelfRefuting([finding], true, ablatedRuntime); + + expect(stripPolicyEffects(active[0])).toEqual(legacy[0]); + expect(active[0]).toMatchObject({ severity: "INFO", self_refuted: true }); + expect(ablated[0]).toEqual(finding); + expect(numericSummary(activeRuntime, "evidence.self-refutation")).toEqual([ + 1, 1, 1, 1, 0, 1, 0, 0, + ]); + expect(numericSummary(ablatedRuntime, "evidence.self-refutation")).toEqual([ + 1, 1, 1, 0, 0, 0, 1, 0, + ]); + expectEffect(active[0], { + pass_id: "evidence.self-refutation", + order: 20, + action: "demoted", + before: "WARN", + after: "INFO", + reason_code: "terminal-self-refutation", + }); + + for (const [protectedFinding, protectedBy] of [ + [ + mkFinding({ + signature: "sig-self-security", + category: "correctness", + details: "Checked carefully. No issue.", + }), + "security-correctness-floor", + ], + [ + mkFinding({ + signature: "sig-self-deterministic", + deterministic: true, + details: "Checked carefully. No issue.", + }), + "deterministic-ground-truth", + ], + ] as const) { + const protectedRuntime = runtime(`self-${protectedBy}`); + const protectedResult = demoteSelfRefuting([protectedFinding], true, protectedRuntime); + expect(stripPolicyEffects(protectedResult[0])).toEqual(protectedFinding); + expect(numericSummary(protectedRuntime, "evidence.self-refutation")).toEqual([ + 1, 1, 1, 0, 1, 0, 1, 0, + ]); + expectEffect(protectedResult[0], { + pass_id: "evidence.self-refutation", + order: 20, + action: "protected", + before: "WARN", + after: "WARN", + reason_code: "terminal-self-refutation", + protected_by: protectedBy, + }); + } + }); + + it("records hypothetical no-opportunity and predicate-miss rows", () => { + const warnRuntime = runtime("hypothetical-warn"); + const presentRuntime = runtime("hypothetical-present"); + + demoteHypotheticalCriticals( + [mkFinding({ severity: "WARN", details: "Currently safe; future change." })], + true, + warnRuntime, + ); + demoteHypotheticalCriticals( + [ + mkFinding({ + severity: "CRITICAL", + details: "Currently safe in theory, but this already fails right now.", + }), + ], + true, + presentRuntime, + ); + + expect(numericSummary(warnRuntime, "judgment.hypothetical")).toEqual([1, 0, 0, 0, 0, 0, 0, 0]); + expect(numericSummary(presentRuntime, "judgment.hypothetical")).toEqual([ + 1, 1, 0, 0, 0, 0, 0, 0, + ]); + }); + + it("records hypothetical active, ablated, and closed protections without legacy drift", () => { + const finding = mkFinding({ + signature: "sig-hypothetical", + severity: "CRITICAL", + details: "This is currently safe, but a future change could break it.", + }); + const activeRuntime = runtime("hypothetical-active"); + const ablatedRuntime = runtime("hypothetical-ablated", ["judgment.hypothetical"]); + + const legacy = demoteHypotheticalCriticals([finding]); + const active = demoteHypotheticalCriticals([finding], true, activeRuntime); + const ablated = demoteHypotheticalCriticals([finding], true, ablatedRuntime); + + expect(stripPolicyEffects(active[0])).toEqual(legacy[0]); + expect(active[0]).toMatchObject({ + severity: "WARN", + hypothetical_demoted: true, + demoted_from_critical: true, + }); + expect(ablated[0]).toEqual(finding); + expect(numericSummary(activeRuntime, "judgment.hypothetical")).toEqual([ + 1, 1, 1, 1, 0, 0, 1, 0, + ]); + expect(numericSummary(ablatedRuntime, "judgment.hypothetical")).toEqual([ + 1, 1, 1, 0, 0, 0, 1, 0, + ]); + expectEffect(active[0], { + pass_id: "judgment.hypothetical", + order: 30, + action: "demoted", + before: "CRITICAL", + after: "WARN", + reason_code: "hypothetical-critical", + }); + + for (const [protectedFinding, protectedBy] of [ + [ + { ...finding, signature: "sig-hypothetical-security", category: "security" }, + "security-correctness-floor", + ], + [ + { ...finding, signature: "sig-hypothetical-deterministic", deterministic: true }, + "deterministic-ground-truth", + ], + ] as const) { + const protectedRuntime = runtime(`hypothetical-${protectedBy}`); + const protectedResult = demoteHypotheticalCriticals( + [protectedFinding], + true, + protectedRuntime, + ); + expect(stripPolicyEffects(protectedResult[0])).toEqual(protectedFinding); + expect(numericSummary(protectedRuntime, "judgment.hypothetical")).toEqual([ + 1, 1, 1, 0, 1, 0, 1, 0, + ]); + expectEffect(protectedResult[0], { + pass_id: "judgment.hypothetical", + order: 30, + action: "protected", + before: "CRITICAL", + after: "CRITICAL", + reason_code: "hypothetical-critical", + protected_by: protectedBy, + }); + } + }); + + it("records token grounding no-opportunity and predicate-miss rows", () => { + const warnRuntime = runtime("token-warn"); + const presentRuntime = runtime("token-present"); + const corpus = ":root { --present-token: #fff; }"; + + groundFindings( + [mkFinding({ severity: "WARN", details: "Missing --absent-token." })], + corpus, + warnRuntime, + ); + groundFindings( + [mkFinding({ severity: "CRITICAL", details: "The --present-token is wrong." })], + corpus, + presentRuntime, + ); + + expect(numericSummary(warnRuntime, "evidence.grounding-token")).toEqual([ + 1, 0, 0, 0, 0, 0, 0, 0, + ]); + expect(numericSummary(presentRuntime, "evidence.grounding-token")).toEqual([ + 1, 1, 0, 0, 0, 0, 0, 0, + ]); + }); + + it("records token grounding active, ablated, and protection rows without legacy drift", () => { + const finding = mkFinding({ + signature: "sig-token", + severity: "CRITICAL", + details: "The --absent-token breaks the theme.", + }); + const activeRuntime = runtime("token-active"); + const ablatedRuntime = runtime("token-ablated", ["evidence.grounding-token"]); + + const legacy = groundFindings([finding], "const present = true;"); + const active = groundFindings([finding], "const present = true;", activeRuntime); + const ablated = groundFindings([finding], "const present = true;", ablatedRuntime); + + expect(stripPolicyEffects(active[0])).toEqual(legacy[0]); + expect(active[0]).toMatchObject({ + severity: "WARN", + grounding_demoted: true, + demoted_from_critical: true, + }); + expect(ablated[0]).toEqual(finding); + expect(numericSummary(activeRuntime, "evidence.grounding-token")).toEqual([ + 1, 1, 1, 1, 0, 0, 1, 0, + ]); + expect(numericSummary(ablatedRuntime, "evidence.grounding-token")).toEqual([ + 1, 1, 1, 0, 0, 0, 1, 0, + ]); + expectEffect(active[0], { + pass_id: "evidence.grounding-token", + order: 40, + action: "demoted", + before: "CRITICAL", + after: "WARN", + reason_code: "cited-token-absent", + }); + + const protectedFinding = { + ...finding, + signature: "sig-token-security", + category: "security" as const, + }; + const protectedRuntime = runtime("token-protected"); + const protectedResult = groundFindings( + [protectedFinding], + "const present = true;", + protectedRuntime, + ); + expect(stripPolicyEffects(protectedResult[0])).toEqual(protectedFinding); + expect(numericSummary(protectedRuntime, "evidence.grounding-token")).toEqual([ + 1, 1, 1, 0, 1, 0, 1, 0, + ]); + expectEffect(protectedResult[0], { + pass_id: "evidence.grounding-token", + order: 40, + action: "protected", + before: "CRITICAL", + after: "CRITICAL", + reason_code: "cited-token-absent", + protected_by: "security-correctness-floor", + }); + }); + + it("records LLM grounding no-opportunity and predicate-miss rows", () => { + const absentRuntime = runtime("llm-absent"); + const groundedRuntime = runtime("llm-grounded"); + const finding = mkFinding({ severity: "CRITICAL" }); + + applyGroundingJudgeVerdicts([finding], new Map(), absentRuntime); + applyGroundingJudgeVerdicts( + [finding], + new Map([[finding.signature, { grounded: true }]]), + groundedRuntime, + ); + + expect(numericSummary(absentRuntime, "judgment.grounding-llm")).toEqual([ + 1, 0, 0, 0, 0, 0, 0, 0, + ]); + expect(numericSummary(groundedRuntime, "judgment.grounding-llm")).toEqual([ + 1, 1, 0, 0, 0, 0, 0, 0, + ]); + }); + + it("records LLM grounding active, ablated, and protection rows without legacy drift", () => { + const finding = mkFinding({ signature: "sig-llm", severity: "CRITICAL" }); + const verdicts = new Map([ + [finding.signature, { grounded: false, reason: "not present" }], + ]); + const activeRuntime = runtime("llm-active"); + const ablatedRuntime = runtime("llm-ablated", ["judgment.grounding-llm"]); + + const legacy = applyGroundingJudgeVerdicts([finding], verdicts); + const active = applyGroundingJudgeVerdicts([finding], verdicts, activeRuntime); + const ablated = applyGroundingJudgeVerdicts([finding], verdicts, ablatedRuntime); + + expect(stripPolicyEffects(active[0])).toEqual(legacy[0]); + expect(active[0]).toMatchObject({ + severity: "WARN", + grounding_demoted: true, + demoted_from_critical: true, + }); + expect(ablated[0]).toEqual(finding); + expect(numericSummary(activeRuntime, "judgment.grounding-llm")).toEqual([ + 1, 1, 1, 1, 0, 0, 1, 0, + ]); + expect(numericSummary(ablatedRuntime, "judgment.grounding-llm")).toEqual([ + 1, 1, 1, 0, 0, 0, 1, 0, + ]); + expectEffect(active[0], { + pass_id: "judgment.grounding-llm", + order: 50, + action: "demoted", + before: "CRITICAL", + after: "WARN", + reason_code: "judge-ungrounded", + }); + + const protectedFinding = { + ...finding, + signature: "sig-llm-correctness", + category: "correctness" as const, + }; + const protectedRuntime = runtime("llm-protected"); + const protectedResult = applyGroundingJudgeVerdicts( + [protectedFinding], + new Map([[protectedFinding.signature, { grounded: false }]]), + protectedRuntime, + ); + expect(stripPolicyEffects(protectedResult[0])).toEqual(protectedFinding); + expect(numericSummary(protectedRuntime, "judgment.grounding-llm")).toEqual([ + 1, 1, 1, 0, 1, 0, 1, 0, + ]); + expectEffect(protectedResult[0], { + pass_id: "judgment.grounding-llm", + order: 50, + action: "protected", + before: "CRITICAL", + after: "CRITICAL", + reason_code: "judge-ungrounded", + protected_by: "security-correctness-floor", + }); + }); +}); diff --git a/tests/unit/self-refutation.test.ts b/tests/unit/self-refutation.test.ts index 8f8ac4b..7d86999 100644 --- a/tests/unit/self-refutation.test.ts +++ b/tests/unit/self-refutation.test.ts @@ -1,5 +1,6 @@ // tests/unit/self-refutation.test.ts import { describe, expect, it } from "bun:test"; +import { PolicyTraceRecorder } from "../../src/core/policy/trace.ts"; import { demoteSelfRefuting } from "../../src/core/self-refutation.ts"; import type { Finding } from "../../src/schemas/finding.ts"; @@ -146,6 +147,32 @@ describe("demoteSelfRefuting — NEGATIVE (must stay blocking)", () => { }); describe("demoteSelfRefuting — guards", () => { + it("records a matching correctness retraction as protected after its opportunity", () => { + const f = mkFinding({ + category: "correctness", + details: "Traced the index math carefully. This is fine.", + }); + const runtime = PolicyTraceRecorder.start({ runId: "self-protected", iter: 1, ablated: [] }); + const out = demoteSelfRefuting([f], true, runtime); + + expect(out[0]?.severity).toBe("WARN"); + expect(out[0]?.self_refuted).toBeUndefined(); + expect(out[0]?.policy_effects?.[0]).toMatchObject({ + pass_id: "evidence.self-refutation", + order: 20, + action: "protected", + reason_code: "terminal-self-refutation", + protected_by: "security-correctness-floor", + }); + expect(runtime.summary("evidence.self-refutation")).toMatchObject({ + opportunities: 1, + would_apply: 1, + applied: 0, + protected: 1, + blocking_preserved: 1, + }); + }); + it("is idempotent (re-running leaves an already-demoted INFO unchanged)", () => { const f = mkFinding({ details: "Looks correct. No issue." }); const once = demoteSelfRefuting([f]); From 1b1ddd76d325ba5d716e21c5f98133084c00b81f Mon Sep 17 00:00:00 2001 From: Codevena Date: Mon, 10 Aug 2026 05:19:02 +0200 Subject: [PATCH 27/55] fix(policy): register INFO fact-location transition --- src/core/policy/catalog.ts | 1 + tests/unit/policy-catalog.test.ts | 11 ++++ .../policy-preaggregation-contracts.test.ts | 60 +++++++++++++++++++ 3 files changed, 72 insertions(+) diff --git a/src/core/policy/catalog.ts b/src/core/policy/catalog.ts index 908d05e..8436df9 100644 --- a/src/core/policy/catalog.ts +++ b/src/core/policy/catalog.ts @@ -156,6 +156,7 @@ export const POLICY_PASSES = [ after: "INFO", }, { reason_code: "location-out-of-range", action: "demoted", before: "WARN", after: "INFO" }, + { reason_code: "location-out-of-range", action: "demoted", before: "INFO", after: "INFO" }, { reason_code: "evidence-line-reanchored", action: "reanchored", diff --git a/tests/unit/policy-catalog.test.ts b/tests/unit/policy-catalog.test.ts index bbe03f1..d3fe687 100644 --- a/tests/unit/policy-catalog.test.ts +++ b/tests/unit/policy-catalog.test.ts @@ -260,4 +260,15 @@ describe("policy catalog", () => { ), ).toBe(false); }); + + it("registers the legacy INFO fact-invalid mutation as an exact closed transition", () => { + const factLocation = POLICY_PASSES.find((pass) => pass.id === "evidence.fact-location"); + + expect(factLocation?.material_transitions).toContainEqual({ + reason_code: "location-out-of-range", + action: "demoted", + before: "INFO", + after: "INFO", + }); + }); }); diff --git a/tests/unit/policy-preaggregation-contracts.test.ts b/tests/unit/policy-preaggregation-contracts.test.ts index 43d8a23..06dbe56 100644 --- a/tests/unit/policy-preaggregation-contracts.test.ts +++ b/tests/unit/policy-preaggregation-contracts.test.ts @@ -189,6 +189,66 @@ describe("pre-aggregation policy numeric contracts", () => { }); }); + it("records an INFO fact-invalid mutation without invalidating trace or ablation", () => { + const dir = factRepo(); + const finding = mkFinding({ + signature: "sig-fact-info", + severity: "INFO", + file: "one-line.ts", + line_start: 9, + line_end: 9, + }); + const activeRuntime = runtime("fact-info-active"); + const ablatedRuntime = runtime("fact-info-ablated", ["evidence.fact-location"]); + + const legacy = validateFindingFacts([finding], dir, new Set()); + const active = validateFindingFacts([finding], dir, new Set(), activeRuntime); + const ablated = validateFindingFacts([finding], dir, new Set(), ablatedRuntime); + + expect(stripPolicyEffects(active[0])).toEqual(legacy[0]); + expect(active[0]).toMatchObject({ severity: "INFO", fact_invalid: true }); + expect(activeRuntime.telemetryError).toBe(false); + expect(numericSummary(activeRuntime, "evidence.fact-location")).toEqual([ + 1, 1, 1, 1, 0, 0, 0, 0, + ]); + expect(activeRuntime.evaluations()).toEqual([ + { + pass_id: "evidence.fact-location", + order: 10, + result: "applied", + before: "INFO", + after: "INFO", + reason_code: "location-out-of-range", + source_signatures: ["sig-fact-info"], + }, + ]); + expectEffect(active[0], { + pass_id: "evidence.fact-location", + order: 10, + action: "demoted", + before: "INFO", + after: "INFO", + reason_code: "location-out-of-range", + }); + + expect(ablated[0]).toEqual(finding); + expect(ablatedRuntime.telemetryError).toBe(false); + expect(numericSummary(ablatedRuntime, "evidence.fact-location")).toEqual([ + 1, 1, 1, 0, 0, 0, 0, 0, + ]); + expect(ablatedRuntime.evaluations()).toEqual([ + { + pass_id: "evidence.fact-location", + order: 10, + result: "would-apply", + before: "INFO", + after: "INFO", + reason_code: "location-out-of-range", + source_signatures: ["sig-fact-info"], + }, + ]); + }); + it("records self-refutation no-opportunity and predicate-miss rows", () => { const infoRuntime = runtime("self-info"); const ordinaryRuntime = runtime("self-ordinary"); From 9a164b1a21140e0ca7a3b2f2dbe78895813fd6f1 Mon Sep 17 00:00:00 2001 From: Codevena Date: Mon, 10 Aug 2026 05:45:07 +0200 Subject: [PATCH 28/55] feat(policy): trace clustering and scope decisions --- src/core/aggregator.ts | 428 ++++++++--- .../unit/policy-aggregator-first-half.test.ts | 667 ++++++++++++++++++ 2 files changed, 987 insertions(+), 108 deletions(-) create mode 100644 tests/unit/policy-aggregator-first-half.test.ts diff --git a/src/core/aggregator.ts b/src/core/aggregator.ts index c4766c8..36f947b 100644 --- a/src/core/aggregator.ts +++ b/src/core/aggregator.ts @@ -4,15 +4,19 @@ import { normalizeRepoPath } from "../diff/repo-path.ts"; import { classify } from "../research/diff-facts.ts"; import type { Consensus, Finding, FindingCategory } from "../schemas/finding.ts"; import type { Verdict } from "../schemas/pending-report.ts"; +import type { PolicyEffect } from "../schemas/policy-trace.ts"; import { compareCodeUnits } from "../utils/compare.ts"; import { isHarnessConfigPath } from "../utils/git.ts"; import type { CriticVerdict } from "./critic.ts"; import { normalizeProviders } from "./decision-outcome.ts"; import { ruleIdToken0 } from "./fp-ledger/clusters.ts"; +import type { PolicyProtectionCode, PolicyReasonCode } from "./policy/catalog.ts"; +import { type PolicyRuntime, mergePolicyEffects, transitionFinding } from "./policy/trace.ts"; export interface AggregateInput { findings: Finding[]; reviewersTotal: number; + policyRuntime?: PolicyRuntime; critic?: Map; // M5 Part A: per-file changed new-file line ranges. When provided and // scopeToDiff !== false, findings outside the changed hunks are demoted to INFO. @@ -245,6 +249,7 @@ interface Cluster { tokens: Set; categories: Set; members: NonNullable; + effects: PolicyEffect[]; } // True if the finding's representative OR any merged member is categorized @@ -307,6 +312,12 @@ function memberOf(f: Finding): NonNullable[number] { }; } +function sourceSignatures(f: Finding): string[] { + return [...new Set([f.signature, ...(f.members?.map((member) => member.signature) ?? [])])].sort( + compareCodeUnits, + ); +} + // Diff-scoping: demote findings that don't anchor to the changed lines to INFO // (advisory, never dropped) so a hallucination on unchanged code can't block. // Two cases: (1) the finding's FILE isn't in the diff at all — the strongest FP @@ -315,9 +326,12 @@ function memberOf(f: Finding): NonNullable[number] { // outside the changed hunks. Paths on both sides are normalized so a reviewer's // "./src/x.ts" matches the canonical "src/x.ts" diff key. function scopeFindings(survivors: Finding[], input: AggregateInput): Finding[] { - if (input.scopeToDiff === false || !input.changedRanges) return survivors; + const enabled = input.scopeToDiff !== false && input.changedRanges !== undefined; + if (!enabled && input.policyRuntime === undefined) return survivors; const normalizedRanges = new Map(); - for (const [k, v] of input.changedRanges) normalizedRanges.set(normalizeRepoPath(k), v); + for (const [k, v] of input.changedRanges ?? []) { + normalizedRanges.set(normalizeRepoPath(k), v); + } const blocking = new Set(input.outOfDiffBlocking ?? []); // Keep details within FindingSchema's 2000-char cap (truncate the original, // never the note) — appending blindly can overflow a finding already at the @@ -328,8 +342,30 @@ function scopeFindings(survivors: Finding[], input: AggregateInput): Finding[] { return { ...f, severity: "INFO" as const, scope_demoted: true, details }; }; return survivors.map((f) => { - if (!f.line_start) return f; // no usable line → keep (conservative) + const opportunity = enabled && f.severity !== "INFO" && Boolean(f.line_start); + if (!enabled || !f.line_start) { + return ( + transitionFinding({ + ...(input.policyRuntime === undefined ? {} : { runtime: input.policyRuntime }), + passId: "scope.diff", + finding: f, + opportunity, + matched: false, + reasonCode: "outside-changed-lines", + action: "demoted", + sourceSignatures: sourceSignatures(f), + proposed: () => f, + }) ?? f + ); + } + const ranges = normalizedRanges.get(normalizeRepoPath(f.file)); + const categories = [f.category, ...(f.members?.map((m) => m.category) ?? [])]; + let matched = false; + let protectedBy: PolicyProtectionCode | undefined; + let reasonCode: PolicyReasonCode = "outside-changed-lines"; + let note = "\n\n↓ outside the changed lines — advisory only."; + if (!ranges) { // I-17: a finding on harness config (.claude/) the diff did NOT touch is // exploration noise — the every-branch "repo-local hooks = RCE" wolf-cry on @@ -338,27 +374,51 @@ function scopeFindings(survivors: Finding[], input: AggregateInput): Finding[] { // .claude change hits the ranges branch below and CAN still block, so // malicious/accidental hook edits stay reviewed (F-003). if (isHarnessConfigPath(normalizeRepoPath(f.file))) { - return demote( - f, - "\n\n↓ pre-existing harness config not changed by this diff — advisory only.", - ); + matched = opportunity; + reasonCode = "preexisting-harness-config"; + note = "\n\n↓ pre-existing harness config not changed by this diff — advisory only."; + } else { + matched = opportunity; + reasonCode = "outside-changed-file"; + note = "\n\n↓ not in the changed files — advisory only."; + if (matched && categories.some((c) => blocking.has(c))) { + protectedBy = "out-of-diff-blocking-hatch"; + } + } + } else if (!rangeOverlapsChanged(f.line_start, f.line_end ?? f.line_start, ranges)) { + matched = opportunity; + if (matched && categories.some((c) => blocking.has(c))) { + protectedBy = "out-of-diff-blocking-hatch"; } - // Category-independent clustering can merge several categories into one - // finding, so honor the escape hatch if ANY merged member category (not just - // the representative's) is configured to stay blocking. - const categories = [f.category, ...(f.members?.map((m) => m.category) ?? [])]; - if (categories.some((c) => blocking.has(c))) return f; - return demote(f, "\n\n↓ not in the changed files — advisory only."); } - if (rangeOverlapsChanged(f.line_start, f.line_end ?? f.line_start, ranges)) return f; - // In-file but outside the changed hunks. Honor the SAME blocking escape hatch - // as the file-absent case above: a reviewer often cites the enclosing - // declaration a few lines above the changed call, so a configured category - // (e.g. security) must be able to stay blocking instead of silently demoting a - // real CRITICAL to INFO (F-033). - const categories = [f.category, ...(f.members?.map((m) => m.category) ?? [])]; - if (categories.some((c) => blocking.has(c))) return f; - return demote(f, "\n\n↓ outside the changed lines — advisory only."); + + const transitioned = + transitionFinding({ + ...(input.policyRuntime === undefined ? {} : { runtime: input.policyRuntime }), + passId: "scope.diff", + finding: f, + opportunity, + matched, + reasonCode, + action: "demoted", + ...(protectedBy === undefined ? {} : { protectedBy }), + sourceSignatures: sourceSignatures(f), + proposed: () => demote(f, note), + }) ?? f; + + // INFO is outside this pass's blocking opportunity denominator, but the legacy + // implementation still stamped the advisory marker when it was outside scope. + if ( + f.severity === "INFO" && + ((ranges === undefined && !categories.some((category) => blocking.has(category))) || + (ranges !== undefined && + !rangeOverlapsChanged(f.line_start, f.line_end ?? f.line_start, ranges) && + !categories.some((category) => blocking.has(category))) || + (ranges === undefined && isHarnessConfigPath(normalizeRepoPath(f.file)))) + ) { + return demote(transitioned, note); + } + return transitioned; }); } @@ -392,9 +452,29 @@ const SECRET_LEAD_WORD = const REDACTION_CODE_HALLUCINATION = /\b(undefined|undeclared|not\s+defined|unused|unresolved|reference\s?error|type\s?error|syntax\s?error|no\s+such\s+(?:variable|symbol|identifier)|cannot\s+find\s+(?:name|module)|can't\s+find\s+(?:name|module)|invalid\s+(?:identifier|cuid|uuid|token|symbol)|not\s+a\s+valid\s+(?:identifier|name|variable)|never\s+(?:declared|defined))\b/i; +function redactionSubjectFields(f: Finding): string[] { + return [f.message, f.suggested_fix ?? ""]; +} + +function hasRedactionPlaceholder(f: Finding): boolean { + return redactionSubjectFields(f).some((field) => field.includes(" REDACTION_CODE_HALLUCINATION.test(field)); +} + +function redactionProtection(f: Finding): PolicyProtectionCode | undefined { + if (f.category === "security") return "security-correctness-floor"; + if (redactionSubjectFields(f).some((field) => SECRET_LEAD_WORD.test(field))) { + return "secret-evidence-backstop"; + } + return undefined; +} + function isRedactionArtifact(f: Finding): boolean { - const fields = [f.message, f.suggested_fix ?? ""]; - if (!fields.some((s) => s.includes(" SECRET_LEAD_WORD.test(s))) return false; // gate 3: secret-word backstop // gate 4 (fail-safe): demote ONLY with a positive code-hallucination signal. No signal → @@ -410,16 +490,34 @@ export function aggregate(input: AggregateInput): AggregateResult { // the cluster instead, and the artifact rides as an INFO member. Demote, NOT drop: see // isRedactionArtifact — a mis-worded real secret leak must stay VISIBLE, not vanish. const demoteRedaction = (f: Finding): Finding => { - if (!isRedactionArtifact(f)) return f; - if (f.severity === "INFO") return { ...f, redaction_demoted: true }; + const legacyMatch = isRedactionArtifact(f); + const opportunity = f.severity !== "INFO" && hasRedactionPlaceholder(f); + const matched = opportunity && hasRedactionHallucinationSignal(f); + const protectedBy = matched ? redactionProtection(f) : undefined; const note = "\n\n↓ targets Reviewgate's own placeholder (a stripped secret, not real code) — advisory only."; - return { - ...f, - severity: "INFO" as const, - redaction_demoted: true, - details: `${f.details.slice(0, 2000 - note.length)}${note}`, - }; + const transitioned = + transitionFinding({ + ...(input.policyRuntime === undefined ? {} : { runtime: input.policyRuntime }), + passId: "evidence.redaction-placeholder", + finding: f, + opportunity, + matched, + reasonCode: "placeholder-code-hallucination", + action: "demoted", + ...(protectedBy === undefined ? {} : { protectedBy }), + sourceSignatures: [f.signature], + proposed: () => ({ + ...f, + severity: "INFO" as const, + redaction_demoted: true, + details: `${f.details.slice(0, 2000 - note.length)}${note}`, + }), + }) ?? f; + // INFO findings are outside the blocking opportunity denominator, but the old + // path still exposed a matching placeholder as redaction-demoted. + if (f.severity === "INFO" && legacyMatch) return { ...transitioned, redaction_demoted: true }; + return transitioned; }; // Canonicalize every finding's path up front so clustering/dedup, the emitted // representative path, AND the diff-scope lookup all agree — otherwise "./x.ts" @@ -479,6 +577,7 @@ export function aggregate(input: AggregateInput): AggregateResult { if (!target.messages.includes(f.message)) target.messages.push(f.message); target.categories.add(f.category); target.members.push(memberOf(f)); + target.effects = mergePolicyEffects(target.effects, f.policy_effects); // Representative = highest severity (most conservative); ties keep the first. // Note: target.tokens is NOT mutated — the seed's tokens stay the cluster's // stable comparison anchor (mutating them would make clustering order-dependent). @@ -495,12 +594,13 @@ export function aggregate(input: AggregateInput): AggregateResult { tokens: fTokens, categories: new Set([f.category]), members: [memberOf(f)], + effects: mergePolicyEffects(f.policy_effects), }); } } const deduped: Finding[] = []; - for (const { sample, reviewers, messages, categories, members } of clusters) { + for (const { sample, reviewers, messages, categories, members, effects } of clusters) { const consensus = computeConsensus(reviewers.length, input.reviewersTotal); // Preserve every reviewer's wording so nothing is lost when findings merge. const others = messages.filter((m) => m !== sample.message); @@ -535,15 +635,28 @@ export function aggregate(input: AggregateInput): AggregateResult { // exactly the merge the repair made possible. const anchorRepaired = sample.anchor_repaired === true || members.some((m) => m.anchor_repaired === true); - deduped.push({ + const mergedEffects = mergePolicyEffects(effects); + const representative: Finding = { ...sample, details: details.slice(0, 2000), confirmed_by: reviewers, consensus, members, + ...(mergedEffects.length > 0 || sample.policy_effects !== undefined + ? { policy_effects: mergedEffects } + : {}), ...(demotedFromCritical ? { demoted_from_critical: true } : {}), ...(anchorRepaired ? { anchor_repaired: true } : {}), + }; + deduped.push(representative); + const inputSignatures = sourceSignatures(representative); + input.policyRuntime?.recordStage({ + stageId: "aggregation.cluster", + reasonCode: inputSignatures.length === 1 ? "singleton" : "clustered", + inputSignatures, + outputSignature: representative.signature, }); + input.policyRuntime?.linkFinal(inputSignatures, representative.signature); } // §4.3 Fix-Verification — pin claimed-fixed recurrences UP FRONT (before any @@ -588,26 +701,31 @@ export function aggregate(input: AggregateInput): AggregateResult { const survivors: Finding[] = []; const criticDropped: Finding[] = []; for (const f of taggedFindings) { - // §4.3: a pinned recurrence keeps its blocking severity — skip the critic demote. - if (pinned.has(f.signature)) { - survivors.push(f); - continue; - } - // #1: a self-refuted finding (T1) is already demoted to advisory INFO. The critic's - // INFO+likely_fp → DROP would erase it, violating self-refutation's "demote-to-INFO, - // never drop — stays visible/attributable" fail-safe contract end-to-end. Keep it as a - // visible advisory survivor (it is already non-blocking, so nothing is gained by dropping). - if (f.self_refuted === true) { - survivors.push(f); - continue; - } // Scan the representative AND every merged member signature (mirror the // fp_ledger_match pass): the critic may have keyed its verdict on a member's // signature, not the promoted representative's — checking only f.signature // would let that likely_fp leak through with full blocking weight. const critSigs = [f.signature, ...(f.members?.map((m) => m.signature) ?? [])]; - const cv = critic && critSigs.map((s) => critic.get(s)).find((v) => v?.verdict === "likely_fp"); - if (cv?.verdict === "likely_fp") { + const criticRows = critic ? critSigs.map((signature) => critic.get(signature)) : []; + const cv = + criticRows.find((verdict) => verdict?.verdict === "likely_fp") ?? + criticRows.find((verdict) => verdict !== undefined); + const opportunity = cv !== undefined; + const matched = cv?.verdict === "likely_fp"; + let protectedBy: PolicyProtectionCode | undefined; + + if (matched && pinned.has(f.signature)) { + protectedBy = "claimed-fixed-pin"; + } else if (matched && f.self_refuted === true) { + // #1: a self-refuted finding (T1) is already advisory INFO. The critic's + // INFO+likely_fp → DROP must not erase its visible attribution. + protectedBy = "self-refutation-visibility"; + } + + let isSecurityProtected = false; + let isCorroborated = false; + let highPrecisionProtected = false; + if (matched && protectedBy === undefined) { // CRITICAL-only by measurement. A WARN floor (Slice B, 2026-08-05) sat here until // 2026-08-07 and was REVERTED: replayed over the whole recorded corpus it fired 3 times, // protected a false positive all 3 times, and protected 0 true positives. The one time the @@ -618,41 +736,62 @@ export function aggregate(input: AggregateInput): AggregateResult { // Accepted cost: an uncorroborated WARN security finding from an unproven reviewer, called // likely_fp, now goes to INFO with no downstream gate (protected_high_precision below is // cold-start-inert). Evidence: docs/dev/2026-08-07-slice-b-critic-floor-counterfactual.md. - const isSecurityProtected = f.severity === "CRITICAL" && touchesSecurityOrCorrectness(f); + isSecurityProtected = f.severity === "CRITICAL" && touchesSecurityOrCorrectness(f); // A single adversarial critic must not override GROUP agreement. Both // unanimous AND majority are corroborated consensus — the verdict gate // treats them identically (warnFail), and the confidence- and reputation- // demote tiers already exempt majority. Mirror that here so the critic // can't silently flip a corroborated FAIL into a SOFT-PASS. - const isCorroborated = f.consensus === "unanimous" || f.consensus === "majority"; + isCorroborated = f.consensus === "unanimous" || f.consensus === "majority"; // #4: a high-precision reviewer's blocking finding is kept at full severity even when // the critic calls it likely_fp — the dangerous direction is a demoted TRUE positive // (field report F-005). Tag it so the agent sees WHY it stayed blocking; do NOT set // critic_verdict (that renders the dismissive "likely FP" badge). - if (!isSecurityProtected && !isCorroborated && isProtected(f)) { - survivors.push({ ...f, protected_high_precision: true }); - continue; - } - if (!isSecurityProtected && !isCorroborated) { + highPrecisionProtected = !isSecurityProtected && !isCorroborated && isProtected(f); + if (isSecurityProtected) protectedBy = "security-correctness-floor"; + else if (f.consensus === "unanimous") protectedBy = "corroborated-unanimous"; + else if (f.consensus === "majority") protectedBy = "corroborated-majority"; + else if (highPrecisionProtected) protectedBy = "high-precision-reviewer"; + } + + const predictedDemotion = matched ? demoteOneStep(f) : undefined; + const transitioned = transitionFinding({ + ...(input.policyRuntime === undefined ? {} : { runtime: input.policyRuntime }), + passId: "judgment.critic", + finding: f, + opportunity, + matched, + reasonCode: "critic-likely-fp", + action: predictedDemotion?.severity === "drop" ? "dropped" : "demoted", + ...(protectedBy === undefined ? {} : { protectedBy }), + sourceSignatures: sourceSignatures(f), + proposed: () => { const demoted = demoteOneStep(f); - if (demoted.severity === "drop") { - criticDropped.push(f); // INFO likely_fp dropped entirely — keep it attributable - continue; - } - survivors.push({ + if (demoted.severity === "drop") return null; + return { ...f, severity: demoted.severity, // G0: a critic likely_fp that lowers a from-CRITICAL keeps it ≥WARN + decision-required. ...(demoted.demoted_from_critical ? { demoted_from_critical: true } : {}), critic_verdict: "likely_fp", - ...(cv.reason ? { critic_reason: cv.reason } : {}), - }); - continue; - } - survivors.push({ ...f, critic_verdict: "keep" }); + ...(cv?.reason ? { critic_reason: cv.reason } : {}), + }; + }, + }); + + if (transitioned === null) { + criticDropped.push(f); // INFO likely_fp dropped entirely — keep it attributable + continue; + } + if (matched && protectedBy === "high-precision-reviewer") { + survivors.push({ ...transitioned, protected_high_precision: true }); continue; } - survivors.push(f); + if (matched && (isSecurityProtected || isCorroborated)) { + survivors.push({ ...transitioned, critic_verdict: "keep" }); + continue; + } + survivors.push(transitioned); } // M5 Part A — diff-scoping: demote findings outside the changed hunks to INFO @@ -674,34 +813,68 @@ export function aggregate(input: AggregateInput): AggregateResult { // blocking; §4.3 pinned recurrences stay; inert when no deltaScope was computed // (missing/corrupt snapshot, iteration 1, one-shot mode, incomplete diff). const deltaScope = input.deltaScope; - const deltaScoped: Finding[] = deltaScope - ? scoped.map((f) => { - if (f.severity === "INFO") return f; - if (f.claimed_fixed_recurred) return f; - if (touchesSecurityOrCorrectness(f)) return f; - // G0 alignment (adversarial review 2026-07-03): a from-CRITICAL WARN stays - // decision-required — pushing it to INFO here would bypass the SOFT-PASS - // re-arm blocker (run-summary counts the flag only on CRITICAL/WARN). - // Stricter than the sibling structural demotes; costs one decision in a - // rare treadmill case, never hides a possibly-real CRITICAL. - if (f.demoted_from_critical === true) return f; - if (deltaScope.has(normalizeRepoPath(f.file))) return f; - // Honor the SAME cross-file escape hatch as scopeFindings/foreign: a - // category the maintainer configured to stay blocking out-of-diff must - // not be silently demoted by the delta pass either (adversarial review). - const memberCats = [f.category, ...(f.members?.map((m) => m.category) ?? [])]; - const hatch = new Set(input.outOfDiffBlocking ?? []); - if (memberCats.some((c) => hatch.has(c))) return f; - const note = - "\n\n↓ on content already reviewed in an earlier iteration and unchanged since — advisory only (delta scope)."; - return { - ...f, - severity: "INFO" as const, - delta_scope_demoted: true, - details: `${f.details.slice(0, 2000 - note.length)}${note}`, - }; - }) - : scoped; + const deltaScoped: Finding[] = + deltaScope !== null && deltaScope !== undefined + ? scoped.map((f) => { + const opportunity = f.severity !== "INFO"; + const matched = opportunity && !deltaScope.has(normalizeRepoPath(f.file)); + let protectedBy: PolicyProtectionCode | undefined; + if (matched && f.claimed_fixed_recurred) { + protectedBy = "claimed-fixed-pin"; + } else if (matched && touchesSecurityOrCorrectness(f)) { + protectedBy = "security-correctness-floor"; + } else if (matched && f.demoted_from_critical === true) { + // G0 alignment: a from-CRITICAL WARN remains decision-required. + protectedBy = "critical-floor"; + } else if (matched) { + // Honor the same cross-file escape hatch as diff/session scope. + const memberCats = [f.category, ...(f.members?.map((m) => m.category) ?? [])]; + const hatch = new Set(input.outOfDiffBlocking ?? []); + if (memberCats.some((category) => hatch.has(category))) { + protectedBy = "out-of-diff-blocking-hatch"; + } + } + + return ( + transitionFinding({ + ...(input.policyRuntime === undefined ? {} : { runtime: input.policyRuntime }), + passId: "scope.delta", + finding: f, + opportunity, + matched, + reasonCode: "outside-delta-scope", + action: "demoted", + ...(protectedBy === undefined ? {} : { protectedBy }), + sourceSignatures: sourceSignatures(f), + proposed: () => { + const note = + "\n\n↓ on content already reviewed in an earlier iteration and unchanged since — advisory only (delta scope)."; + return { + ...f, + severity: "INFO" as const, + delta_scope_demoted: true, + details: `${f.details.slice(0, 2000 - note.length)}${note}`, + }; + }, + }) ?? f + ); + }) + : input.policyRuntime + ? scoped.map( + (f) => + transitionFinding({ + ...(input.policyRuntime === undefined ? {} : { runtime: input.policyRuntime }), + passId: "scope.delta", + finding: f, + opportunity: false, + matched: false, + reasonCode: "outside-delta-scope", + action: "demoted", + sourceSignatures: sourceSignatures(f), + proposed: () => f, + }) ?? f, + ) + : scoped; // Slice A (P1) — session-ownership demote. A blocking finding on a file FOREIGN to this // session (provably byte-identical to its SessionStart baseline, not tool-owned) is demoted @@ -716,22 +889,61 @@ export function aggregate(input: AggregateInput): AggregateResult { const foreignScoped: Finding[] = foreignFiles && foreignFiles.size > 0 ? deltaScoped.map((f) => { - if (!foreignFiles.has(normalizeRepoPath(f.file))) return f; + const opportunity = f.severity !== "INFO"; + const isForeign = foreignFiles.has(normalizeRepoPath(f.file)); + const matched = opportunity && isForeign; const categories = [f.category, ...(f.members?.map((m) => m.category) ?? [])]; const blocking = new Set(input.outOfDiffBlocking ?? []); - // Escape hatch: keep blocking, but still tag so out-of-scope is available. - if (categories.some((c) => blocking.has(c))) return { ...f, foreign_to_session: true }; - if (f.severity === "INFO") return { ...f, foreign_to_session: true }; - const note = - "\n\n↓ on a file this session did not author (parallel agent / pre-existing) — advisory only."; - return { - ...f, - severity: "INFO" as const, - foreign_to_session: true, - details: `${f.details.slice(0, 2000 - note.length)}${note}`, - }; + const protectedBy = + matched && categories.some((category) => blocking.has(category)) + ? "out-of-diff-blocking-hatch" + : undefined; + const transitioned = + transitionFinding({ + ...(input.policyRuntime === undefined ? {} : { runtime: input.policyRuntime }), + passId: "scope.session", + finding: f, + opportunity, + matched, + reasonCode: "foreign-to-session", + action: "demoted", + ...(protectedBy === undefined ? {} : { protectedBy }), + sourceSignatures: sourceSignatures(f), + proposed: () => { + const note = + "\n\n↓ on a file this session did not author (parallel agent / pre-existing) — advisory only."; + return { + ...f, + severity: "INFO" as const, + foreign_to_session: true, + details: `${f.details.slice(0, 2000 - note.length)}${note}`, + }; + }, + }) ?? f; + // The legacy INFO marker is explanatory rather than a blocking policy + // transition. A protected blocking finding is likewise tagged so the + // out-of-scope disposition remains available. + if (isForeign && (f.severity === "INFO" || protectedBy !== undefined)) { + return { ...transitioned, foreign_to_session: true }; + } + return transitioned; }) - : deltaScoped; + : input.policyRuntime + ? deltaScoped.map( + (f) => + transitionFinding({ + ...(input.policyRuntime === undefined ? {} : { runtime: input.policyRuntime }), + passId: "scope.session", + finding: f, + opportunity: false, + matched: false, + reasonCode: "foreign-to-session", + action: "demoted", + sourceSignatures: sourceSignatures(f), + proposed: () => f, + }) ?? f, + ) + : deltaScoped; // M5 Part B1 — reactive FP-ledger demote: a finding whose representative // signature (or any merged member signature) matches an active/sticky FP entry diff --git a/tests/unit/policy-aggregator-first-half.test.ts b/tests/unit/policy-aggregator-first-half.test.ts new file mode 100644 index 0000000..ac0c923 --- /dev/null +++ b/tests/unit/policy-aggregator-first-half.test.ts @@ -0,0 +1,667 @@ +import { describe, expect, it } from "bun:test"; +import { type AggregateInput, aggregate } from "../../src/core/aggregator.ts"; +import type { + PolicyPassId, + PolicyProtectionCode, + PolicyReasonCode, +} from "../../src/core/policy/catalog.ts"; +import { PolicyTraceRecorder } from "../../src/core/policy/trace.ts"; +import type { Finding } from "../../src/schemas/finding.ts"; + +type NumericSummary = readonly [number, number, number, number, number, number, number, number]; + +function finding(overrides: Partial = {}): Finding { + return { + id: "F-001", + signature: "sig-policy", + severity: "WARN", + category: "quality", + rule_id: "policy-contract", + file: "src/a.ts", + line_start: 10, + line_end: 10, + message: "A concrete policy finding", + details: "The implementation has a concrete defect.", + reviewer: { provider: "codex", model: "m", persona: "quality" }, + confidence: 0.9, + consensus: "singleton", + ...overrides, + }; +} + +function runtime(runId: string, ablated: readonly PolicyPassId[] = []): PolicyTraceRecorder { + return PolicyTraceRecorder.start({ runId, iter: 1, ablated }); +} + +function run( + runId: string, + input: AggregateInput, + ablated: readonly PolicyPassId[] = [], +): { recorder: PolicyTraceRecorder; result: ReturnType } { + const recorder = runtime(runId, ablated); + return { + recorder, + result: aggregate({ ...input, policyRuntime: recorder }), + }; +} + +function numericSummary(recorder: PolicyTraceRecorder, passId: PolicyPassId): NumericSummary { + const summary = recorder.summary(passId); + expect(summary.status).toBe("ran"); + if (summary.status !== "ran") throw new Error(`${passId} did not run`); + return [ + summary.considered, + summary.opportunities, + summary.would_apply, + summary.applied, + summary.protected, + summary.blocking_removed, + summary.blocking_preserved, + summary.dropped, + ]; +} + +function stripPolicyEffects(value: Finding | undefined): Finding | undefined { + if (value === undefined) return undefined; + const { policy_effects: _policyEffects, ...legacy } = value; + return legacy as Finding; +} + +function expectSingleEffect( + value: Finding | undefined, + expected: { + pass_id: PolicyPassId; + order: number; + action: "demoted" | "protected"; + before: Finding["severity"]; + after: Finding["severity"]; + reason_code: PolicyReasonCode; + protected_by?: PolicyProtectionCode; + source_signatures?: string[]; + }, +): void { + if (value === undefined) throw new Error("expected a visible finding"); + const { source_signatures = [value.signature], ...effect } = expected; + expect(value.policy_effects).toEqual([{ ...effect, source_signatures }]); +} + +const NO_OPPORTUNITY = [1, 0, 0, 0, 0, 0, 0, 0] as const; +const PREDICATE_MISS = [1, 1, 0, 0, 0, 0, 0, 0] as const; +const ACTIVE_BLOCKING_REMOVAL = [1, 1, 1, 1, 0, 1, 0, 0] as const; +const ABLATED_BLOCKING_PRESERVED = [1, 1, 1, 0, 0, 0, 1, 0] as const; +const PROTECTED_BLOCKING_PRESERVED = [1, 1, 1, 0, 1, 0, 1, 0] as const; + +describe("aggregator policy numeric contracts, orders 60-100", () => { + it("records redaction no-opportunity, miss, active, ablated, and protected tuples", () => { + const info = run("redaction-info", { + findings: [ + finding({ + signature: "sig-redaction-info", + severity: "INFO", + message: "undefined variable ", + }), + ], + reviewersTotal: 1, + }); + const bland = run("redaction-bland", { + findings: [ + finding({ + signature: "sig-redaction-bland", + message: "exposed value ", + }), + ], + reviewersTotal: 1, + }); + const activeFinding = finding({ + signature: "sig-redaction-active", + message: "undefined variable ", + }); + const active = run("redaction-active", { + findings: [activeFinding], + reviewersTotal: 1, + }); + const ablated = run("redaction-ablated", { findings: [activeFinding], reviewersTotal: 1 }, [ + "evidence.redaction-placeholder", + ]); + const protectedFinding = finding({ + signature: "sig-redaction-protected", + category: "security", + message: "undefined variable ", + }); + const protectedResult = run("redaction-protected", { + findings: [protectedFinding], + reviewersTotal: 1, + }); + + expect(numericSummary(info.recorder, "evidence.redaction-placeholder")).toEqual(NO_OPPORTUNITY); + expect(numericSummary(bland.recorder, "evidence.redaction-placeholder")).toEqual( + PREDICATE_MISS, + ); + expect(numericSummary(active.recorder, "evidence.redaction-placeholder")).toEqual( + ACTIVE_BLOCKING_REMOVAL, + ); + expect(numericSummary(ablated.recorder, "evidence.redaction-placeholder")).toEqual( + ABLATED_BLOCKING_PRESERVED, + ); + expect(numericSummary(protectedResult.recorder, "evidence.redaction-placeholder")).toEqual( + PROTECTED_BLOCKING_PRESERVED, + ); + expect(stripPolicyEffects(active.result.dedupedFindings[0])).toEqual( + aggregate({ findings: [activeFinding], reviewersTotal: 1 }).dedupedFindings[0], + ); + expect(active.result.dedupedFindings[0]).toMatchObject({ + severity: "INFO", + redaction_demoted: true, + }); + expect(ablated.result.dedupedFindings[0]).toEqual({ + ...activeFinding, + id: "F-001", + confirmed_by: ["codex:quality"], + members: [ + { + signature: activeFinding.signature, + provider: "codex", + rule_id: activeFinding.rule_id, + category: activeFinding.category, + confidence: activeFinding.confidence, + }, + ], + }); + expectSingleEffect(active.result.dedupedFindings[0], { + pass_id: "evidence.redaction-placeholder", + order: 60, + action: "demoted", + before: "WARN", + after: "INFO", + reason_code: "placeholder-code-hallucination", + }); + expectSingleEffect(protectedResult.result.dedupedFindings[0], { + pass_id: "evidence.redaction-placeholder", + order: 60, + action: "protected", + before: "WARN", + after: "WARN", + reason_code: "placeholder-code-hallucination", + protected_by: "security-correctness-floor", + }); + }); + + it("records critic no-opportunity, keep, active, ablated, protected, and drop tuples", () => { + const base = finding({ signature: "sig-critic" }); + const absent = run("critic-absent", { + findings: [base], + reviewersTotal: 1, + critic: new Map(), + }); + const keep = run("critic-keep", { + findings: [base], + reviewersTotal: 1, + critic: new Map([[base.signature, { verdict: "keep" }]]), + }); + const active = run("critic-active", { + findings: [base], + reviewersTotal: 1, + critic: new Map([[base.signature, { verdict: "likely_fp" }]]), + }); + const ablated = run( + "critic-ablated", + { + findings: [base], + reviewersTotal: 1, + critic: new Map([[base.signature, { verdict: "likely_fp" }]]), + }, + ["judgment.critic"], + ); + const majorityA = finding({ + signature: "sig-critic-majority-a", + reviewer: { provider: "codex", model: "m", persona: "quality" }, + }); + const majorityB = finding({ + signature: "sig-critic-majority-b", + reviewer: { provider: "gemini", model: "m", persona: "quality" }, + }); + const protectedResult = run("critic-protected", { + findings: [majorityA, majorityB], + reviewersTotal: 3, + critic: new Map([[majorityB.signature, { verdict: "likely_fp" }]]), + }); + const droppedFinding = finding({ signature: "sig-critic-drop", severity: "INFO" }); + const dropped = run("critic-drop", { + findings: [droppedFinding], + reviewersTotal: 1, + critic: new Map([[droppedFinding.signature, { verdict: "likely_fp" }]]), + }); + + expect(numericSummary(absent.recorder, "judgment.critic")).toEqual(NO_OPPORTUNITY); + expect(numericSummary(keep.recorder, "judgment.critic")).toEqual(PREDICATE_MISS); + expect(numericSummary(active.recorder, "judgment.critic")).toEqual(ACTIVE_BLOCKING_REMOVAL); + expect(numericSummary(ablated.recorder, "judgment.critic")).toEqual(ABLATED_BLOCKING_PRESERVED); + expect(numericSummary(protectedResult.recorder, "judgment.critic")).toEqual( + PROTECTED_BLOCKING_PRESERVED, + ); + expect(numericSummary(dropped.recorder, "judgment.critic")).toEqual([1, 1, 1, 1, 0, 0, 0, 1]); + expect(active.result.dedupedFindings[0]).toMatchObject({ + severity: "INFO", + critic_verdict: "likely_fp", + }); + expect(ablated.result.dedupedFindings[0]?.severity).toBe("WARN"); + expect(ablated.result.dedupedFindings[0]?.critic_verdict).toBeUndefined(); + expectSingleEffect(active.result.dedupedFindings[0], { + pass_id: "judgment.critic", + order: 70, + action: "demoted", + before: "WARN", + after: "INFO", + reason_code: "critic-likely-fp", + }); + expectSingleEffect(protectedResult.result.dedupedFindings[0], { + pass_id: "judgment.critic", + order: 70, + action: "protected", + before: "WARN", + after: "WARN", + reason_code: "critic-likely-fp", + protected_by: "corroborated-majority", + source_signatures: [majorityA.signature, majorityB.signature], + }); + expect(dropped.result.dedupedFindings).toEqual([]); + expect(dropped.result.criticDropped.map((item) => item.signature)).toEqual([ + droppedFinding.signature, + ]); + expect(dropped.recorder.evaluations()).toContainEqual({ + pass_id: "judgment.critic", + order: 70, + result: "applied", + before: "INFO", + after: null, + reason_code: "critic-likely-fp", + source_signatures: [droppedFinding.signature], + }); + }); + + it("records a claimed-fixed pin only when critic or delta actually attempts a mutation", () => { + const pinnedFinding = finding({ signature: "sig-claimed-fixed" }); + const claimedFixed = new Map([[pinnedFinding.signature, 2]]); + const criticKeep = run("claimed-critic-keep", { + findings: [pinnedFinding], + reviewersTotal: 1, + claimedFixed, + critic: new Map([[pinnedFinding.signature, { verdict: "keep" }]]), + }); + const criticAttempt = run("claimed-critic-attempt", { + findings: [pinnedFinding], + reviewersTotal: 1, + claimedFixed, + critic: new Map([[pinnedFinding.signature, { verdict: "likely_fp" }]]), + }); + const deltaInside = run("claimed-delta-inside", { + findings: [pinnedFinding], + reviewersTotal: 1, + claimedFixed, + deltaScope: new Set([pinnedFinding.file]), + }); + const deltaAttempt = run("claimed-delta-attempt", { + findings: [pinnedFinding], + reviewersTotal: 1, + claimedFixed, + deltaScope: new Set(["src/other.ts"]), + }); + + expect(numericSummary(criticKeep.recorder, "judgment.critic")).toEqual(PREDICATE_MISS); + expect(criticKeep.result.dedupedFindings[0]?.policy_effects).toBeUndefined(); + expect(numericSummary(criticAttempt.recorder, "judgment.critic")).toEqual( + PROTECTED_BLOCKING_PRESERVED, + ); + expectSingleEffect(criticAttempt.result.dedupedFindings[0], { + pass_id: "judgment.critic", + order: 70, + action: "protected", + before: "WARN", + after: "WARN", + reason_code: "critic-likely-fp", + protected_by: "claimed-fixed-pin", + }); + + expect(numericSummary(deltaInside.recorder, "scope.delta")).toEqual(PREDICATE_MISS); + expect(deltaInside.result.dedupedFindings[0]?.policy_effects).toBeUndefined(); + expect(numericSummary(deltaAttempt.recorder, "scope.delta")).toEqual( + PROTECTED_BLOCKING_PRESERVED, + ); + expectSingleEffect(deltaAttempt.result.dedupedFindings[0], { + pass_id: "scope.delta", + order: 90, + action: "protected", + before: "WARN", + after: "WARN", + reason_code: "outside-delta-scope", + protected_by: "claimed-fixed-pin", + }); + }); + + it("records diff-scope no-opportunity, miss, active, ablated, and protected tuples", () => { + const ranges = new Map([["src/a.ts", [[10, 14]] as Array<[number, number]>]]); + const noLine = run("diff-no-line", { + findings: [finding({ signature: "sig-diff-no-line", line_start: 0, line_end: 0 })], + reviewersTotal: 1, + changedRanges: ranges, + scopeToDiff: true, + }); + const inside = run("diff-inside", { + findings: [finding({ signature: "sig-diff-inside", line_start: 11, line_end: 11 })], + reviewersTotal: 1, + changedRanges: ranges, + scopeToDiff: true, + }); + const outsideFinding = finding({ + signature: "sig-diff-outside", + line_start: 50, + line_end: 50, + }); + const active = run("diff-active", { + findings: [outsideFinding], + reviewersTotal: 1, + changedRanges: ranges, + scopeToDiff: true, + }); + const ablated = run( + "diff-ablated", + { + findings: [outsideFinding], + reviewersTotal: 1, + changedRanges: ranges, + scopeToDiff: true, + }, + ["scope.diff"], + ); + const protectedFinding = finding({ + signature: "sig-diff-protected", + category: "security", + line_start: 50, + line_end: 50, + }); + const protectedResult = run("diff-protected", { + findings: [protectedFinding], + reviewersTotal: 1, + changedRanges: ranges, + scopeToDiff: true, + outOfDiffBlocking: ["security"], + }); + + expect(numericSummary(noLine.recorder, "scope.diff")).toEqual(NO_OPPORTUNITY); + expect(numericSummary(inside.recorder, "scope.diff")).toEqual(PREDICATE_MISS); + expect(numericSummary(active.recorder, "scope.diff")).toEqual(ACTIVE_BLOCKING_REMOVAL); + expect(numericSummary(ablated.recorder, "scope.diff")).toEqual(ABLATED_BLOCKING_PRESERVED); + expect(numericSummary(protectedResult.recorder, "scope.diff")).toEqual( + PROTECTED_BLOCKING_PRESERVED, + ); + expect(active.result.dedupedFindings[0]).toMatchObject({ + severity: "INFO", + scope_demoted: true, + }); + expect(ablated.result.dedupedFindings[0]?.severity).toBe("WARN"); + expect(ablated.result.dedupedFindings[0]?.scope_demoted).toBeUndefined(); + expectSingleEffect(active.result.dedupedFindings[0], { + pass_id: "scope.diff", + order: 80, + action: "demoted", + before: "WARN", + after: "INFO", + reason_code: "outside-changed-lines", + }); + expectSingleEffect(protectedResult.result.dedupedFindings[0], { + pass_id: "scope.diff", + order: 80, + action: "protected", + before: "WARN", + after: "WARN", + reason_code: "outside-changed-lines", + protected_by: "out-of-diff-blocking-hatch", + }); + }); + + it("records delta-scope no-opportunity, miss, active, ablated, and protected tuples", () => { + const noOpportunity = run("delta-info", { + findings: [finding({ signature: "sig-delta-info", severity: "INFO" })], + reviewersTotal: 1, + deltaScope: new Set(["src/a.ts"]), + }); + const inside = run("delta-inside", { + findings: [finding({ signature: "sig-delta-inside" })], + reviewersTotal: 1, + deltaScope: new Set(["src/a.ts"]), + }); + const outsideFinding = finding({ signature: "sig-delta-outside" }); + const active = run("delta-active", { + findings: [outsideFinding], + reviewersTotal: 1, + deltaScope: new Set(["src/other.ts"]), + }); + const ablated = run( + "delta-ablated", + { + findings: [outsideFinding], + reviewersTotal: 1, + deltaScope: new Set(["src/other.ts"]), + }, + ["scope.delta"], + ); + const protectedFinding = finding({ + signature: "sig-delta-protected", + category: "correctness", + }); + const protectedResult = run("delta-protected", { + findings: [protectedFinding], + reviewersTotal: 1, + deltaScope: new Set(["src/other.ts"]), + }); + + expect(numericSummary(noOpportunity.recorder, "scope.delta")).toEqual(NO_OPPORTUNITY); + expect(numericSummary(inside.recorder, "scope.delta")).toEqual(PREDICATE_MISS); + expect(numericSummary(active.recorder, "scope.delta")).toEqual(ACTIVE_BLOCKING_REMOVAL); + expect(numericSummary(ablated.recorder, "scope.delta")).toEqual(ABLATED_BLOCKING_PRESERVED); + expect(numericSummary(protectedResult.recorder, "scope.delta")).toEqual( + PROTECTED_BLOCKING_PRESERVED, + ); + expect(active.result.dedupedFindings[0]).toMatchObject({ + severity: "INFO", + delta_scope_demoted: true, + }); + expect(ablated.result.dedupedFindings[0]?.severity).toBe("WARN"); + expect(ablated.result.dedupedFindings[0]?.delta_scope_demoted).toBeUndefined(); + expectSingleEffect(active.result.dedupedFindings[0], { + pass_id: "scope.delta", + order: 90, + action: "demoted", + before: "WARN", + after: "INFO", + reason_code: "outside-delta-scope", + }); + expectSingleEffect(protectedResult.result.dedupedFindings[0], { + pass_id: "scope.delta", + order: 90, + action: "protected", + before: "WARN", + after: "WARN", + reason_code: "outside-delta-scope", + protected_by: "security-correctness-floor", + }); + }); + + it("records session-scope no-opportunity, miss, active, ablated, and protected tuples", () => { + const noOpportunity = run("session-info", { + findings: [finding({ signature: "sig-session-info", severity: "INFO" })], + reviewersTotal: 1, + foreignFiles: new Set(["src/a.ts"]), + }); + const owned = run("session-owned", { + findings: [finding({ signature: "sig-session-owned" })], + reviewersTotal: 1, + foreignFiles: new Set(["src/foreign.ts"]), + }); + const foreignFinding = finding({ signature: "sig-session-foreign" }); + const active = run("session-active", { + findings: [foreignFinding], + reviewersTotal: 1, + foreignFiles: new Set(["src/a.ts"]), + }); + const ablated = run( + "session-ablated", + { + findings: [foreignFinding], + reviewersTotal: 1, + foreignFiles: new Set(["src/a.ts"]), + }, + ["scope.session"], + ); + const protectedFinding = finding({ + signature: "sig-session-protected", + category: "security", + }); + const protectedResult = run("session-protected", { + findings: [protectedFinding], + reviewersTotal: 1, + foreignFiles: new Set(["src/a.ts"]), + outOfDiffBlocking: ["security"], + }); + + expect(numericSummary(noOpportunity.recorder, "scope.session")).toEqual(NO_OPPORTUNITY); + expect(numericSummary(owned.recorder, "scope.session")).toEqual(PREDICATE_MISS); + expect(numericSummary(active.recorder, "scope.session")).toEqual(ACTIVE_BLOCKING_REMOVAL); + expect(numericSummary(ablated.recorder, "scope.session")).toEqual(ABLATED_BLOCKING_PRESERVED); + expect(numericSummary(protectedResult.recorder, "scope.session")).toEqual( + PROTECTED_BLOCKING_PRESERVED, + ); + expect(active.result.dedupedFindings[0]).toMatchObject({ + severity: "INFO", + foreign_to_session: true, + }); + expect(ablated.result.dedupedFindings[0]?.severity).toBe("WARN"); + expect(ablated.result.dedupedFindings[0]?.foreign_to_session).toBeUndefined(); + expectSingleEffect(active.result.dedupedFindings[0], { + pass_id: "scope.session", + order: 100, + action: "demoted", + before: "WARN", + after: "INFO", + reason_code: "foreign-to-session", + }); + expectSingleEffect(protectedResult.result.dedupedFindings[0], { + pass_id: "scope.session", + order: 100, + action: "protected", + before: "WARN", + after: "WARN", + reason_code: "foreign-to-session", + protected_by: "out-of-diff-blocking-hatch", + }); + }); +}); + +describe("aggregation cluster lineage", () => { + it("propagates a demoted member effect and links every input to the representative", () => { + const artifact = finding({ + signature: "sig-artifact", + message: "undefined variable ", + reviewer: { provider: "codex", model: "m", persona: "quality" }, + }); + const representative = finding({ + signature: "sig-representative", + message: "real defect", + reviewer: { provider: "gemini", model: "m", persona: "quality" }, + }); + const { recorder, result } = run("cluster-lineage", { + findings: [artifact, representative], + reviewersTotal: 2, + }); + const finalFinding = result.dedupedFindings[0]; + if (finalFinding === undefined) throw new Error("expected one final cluster"); + + expect(result.dedupedFindings).toHaveLength(1); + expect(finalFinding.signature).toBe(representative.signature); + expect(finalFinding.members?.map((member) => member.signature).sort()).toEqual([ + artifact.signature, + representative.signature, + ]); + expect(finalFinding.policy_effects).toEqual([ + { + pass_id: "evidence.redaction-placeholder", + order: 60, + action: "demoted", + before: "WARN", + after: "INFO", + reason_code: "placeholder-code-hallucination", + source_signatures: [artifact.signature], + }, + ]); + + recorder.recordStage({ + stageId: "verdict.compute", + reasonCode: "corroborated-warn", + inputSignatures: [finalFinding.signature], + verdict: result.verdict, + }); + const trace = recorder.finalize({ + rawResponseSha256: [], + verdict: result.verdict, + finalFindings: result.dedupedFindings, + }); + if (trace === null) throw new Error("expected a valid finalized policy trace"); + + expect(trace.stages).toEqual([ + { + stage_id: "aggregation.cluster", + order: 65, + reason_code: "clustered", + input_signatures: [artifact.signature, representative.signature], + output_signature: representative.signature, + }, + { + stage_id: "verdict.compute", + order: 190, + reason_code: "corroborated-warn", + input_signatures: [representative.signature], + verdict: "FAIL", + }, + ]); + expect( + trace.evaluations + .filter((evaluation) => evaluation.pass_id === "evidence.redaction-placeholder") + .map((evaluation) => ({ + source: evaluation.source_signatures, + final: evaluation.final_signature, + })), + ).toEqual([ + { source: [artifact.signature], final: representative.signature }, + { source: [representative.signature], final: representative.signature }, + ]); + }); + + it("records one deterministic singleton cluster stage", () => { + const singleton = finding({ signature: "sig-singleton" }); + const { recorder, result } = run("cluster-singleton", { + findings: [singleton], + reviewersTotal: 1, + }); + const finalFinding = result.dedupedFindings[0]; + if (finalFinding === undefined) throw new Error("expected singleton output"); + recorder.recordStage({ + stageId: "verdict.compute", + reasonCode: "blocking-present", + inputSignatures: [finalFinding.signature], + verdict: result.verdict, + }); + + const trace = recorder.finalize({ + rawResponseSha256: [], + verdict: result.verdict, + finalFindings: result.dedupedFindings, + }); + expect(trace?.stages[0]).toEqual({ + stage_id: "aggregation.cluster", + order: 65, + reason_code: "singleton", + input_signatures: [singleton.signature], + output_signature: singleton.signature, + }); + }); +}); From dc2c43ef7c7cc75a45de3509b5d1e7a879aaf1db Mon Sep 17 00:00:00 2001 From: Codevena Date: Mon, 10 Aug 2026 06:08:03 +0200 Subject: [PATCH 29/55] fix(policy): preserve critic floor and cluster cardinality --- src/core/aggregator.ts | 8 +- src/core/policy/catalog.ts | 6 ++ src/core/policy/trace.ts | 25 ++++-- src/schemas/policy-trace.ts | 24 ++++- .../unit/policy-aggregator-first-half.test.ts | 89 +++++++++++++++++++ tests/unit/policy-catalog.test.ts | 1 + tests/unit/policy-trace-recorder.test.ts | 38 ++++++++ tests/unit/policy-trace-schema.test.ts | 50 ++++++++++- 8 files changed, 228 insertions(+), 13 deletions(-) diff --git a/src/core/aggregator.ts b/src/core/aggregator.ts index 36f947b..1141a88 100644 --- a/src/core/aggregator.ts +++ b/src/core/aggregator.ts @@ -652,7 +652,8 @@ export function aggregate(input: AggregateInput): AggregateResult { const inputSignatures = sourceSignatures(representative); input.policyRuntime?.recordStage({ stageId: "aggregation.cluster", - reasonCode: inputSignatures.length === 1 ? "singleton" : "clustered", + reasonCode: members.length === 1 ? "singleton" : "clustered", + memberCount: members.length, inputSignatures, outputSignature: representative.signature, }); @@ -720,6 +721,11 @@ export function aggregate(input: AggregateInput): AggregateResult { // #1: a self-refuted finding (T1) is already advisory INFO. The critic's // INFO+likely_fp → DROP must not erase its visible attribution. protectedBy = "self-refutation-visibility"; + } else if (matched && f.severity === "WARN" && f.demoted_from_critical === true) { + // G0: a prior value-judgment CRITICAL→WARN clamp is already at its floor. + // Treat the critic's attempted second demotion as protection, not an + // applied WARN→WARN mutation (which is neither material nor catalog-valid). + protectedBy = "critical-floor"; } let isSecurityProtected = false; diff --git a/src/core/policy/catalog.ts b/src/core/policy/catalog.ts index 8436df9..3e6233d 100644 --- a/src/core/policy/catalog.ts +++ b/src/core/policy/catalog.ts @@ -415,6 +415,7 @@ export const POLICY_PASSES = [ protection_codes: [ "claimed-fixed-pin", "self-refutation-visibility", + "critical-floor", "security-correctness-floor", "corroborated-majority", "corroborated-unanimous", @@ -461,6 +462,11 @@ export const POLICY_PASSES = [ protected_by: "self-refutation-visibility", before: "INFO", }, + { + reason_code: "critic-likely-fp", + protected_by: "critical-floor", + before: "WARN", + }, { reason_code: "critic-likely-fp", protected_by: "security-correctness-floor", diff --git a/src/core/policy/trace.ts b/src/core/policy/trace.ts index c7fb82f..7bea02e 100644 --- a/src/core/policy/trace.ts +++ b/src/core/policy/trace.ts @@ -41,13 +41,23 @@ export interface TransitionInput { readonly proposed: () => Finding | null; } -export interface RecordPolicyStageInput { - readonly stageId: PolicyStageId; - readonly reasonCode: PolicyReasonCode; - readonly inputSignatures: readonly string[]; - readonly outputSignature?: string; - readonly verdict?: Exclude; -} +export type RecordPolicyStageInput = + | { + readonly stageId: Extract; + readonly reasonCode: PolicyReasonCode; + readonly memberCount: number; + readonly inputSignatures: readonly string[]; + readonly outputSignature: string; + readonly verdict?: never; + } + | { + readonly stageId: Extract; + readonly reasonCode: PolicyReasonCode; + readonly memberCount?: never; + readonly inputSignatures: readonly string[]; + readonly outputSignature?: never; + readonly verdict: Exclude; + }; export interface FinalizePolicyTraceInput { readonly rawResponseSha256: readonly string[]; @@ -306,6 +316,7 @@ export class PolicyTraceRecorder implements PolicyRuntime { stage_id: input.stageId, order: stage.order, reason_code: input.reasonCode, + ...(input.memberCount === undefined ? {} : { member_count: input.memberCount }), input_signatures: uniqueInOrder(input.inputSignatures), ...(input.outputSignature === undefined ? {} : { output_signature: input.outputSignature }), ...(input.verdict === undefined ? {} : { verdict: input.verdict }), diff --git a/src/schemas/policy-trace.ts b/src/schemas/policy-trace.ts index deae3d2..3d57f5e 100644 --- a/src/schemas/policy-trace.ts +++ b/src/schemas/policy-trace.ts @@ -456,6 +456,7 @@ const PolicyStageEvaluationObjectSchema = z stage_id: PolicyStageIdSchema, order: z.number().int().positive(), reason_code: PolicyReasonCodeSchema, + member_count: z.number().int().positive().optional(), input_signatures: UniqueSignaturesSchema, output_signature: z.string().min(1).optional(), verdict: StageVerdictSchema.optional(), @@ -473,6 +474,15 @@ export const PolicyStageEvaluationSchema = PolicyStageEvaluationObjectSchema.sup } if (evaluation.stage_id === "aggregation.cluster") { + if (evaluation.member_count === undefined) { + addIssue(ctx, ["member_count"], "a cluster stage requires member_count"); + } else if (evaluation.member_count < evaluation.input_signatures.length) { + addIssue( + ctx, + ["member_count"], + "member_count cannot be smaller than the unique input signature count", + ); + } if (evaluation.input_signatures.length === 0) { addIssue(ctx, ["input_signatures"], "a cluster stage requires input signatures"); } @@ -484,15 +494,21 @@ export const PolicyStageEvaluationSchema = PolicyStageEvaluationObjectSchema.sup if (evaluation.verdict !== undefined) { addIssue(ctx, ["verdict"], "a cluster stage cannot carry a verdict"); } - if (evaluation.reason_code === "singleton" && evaluation.input_signatures.length !== 1) { - addIssue(ctx, ["reason_code"], "singleton requires exactly one input signature"); + if (evaluation.reason_code === "singleton" && evaluation.member_count !== 1) { + addIssue(ctx, ["reason_code"], "singleton requires member_count 1"); } - if (evaluation.reason_code === "clustered" && evaluation.input_signatures.length < 2) { - addIssue(ctx, ["reason_code"], "clustered requires at least two input signatures"); + if ( + evaluation.reason_code === "clustered" && + (evaluation.member_count === undefined || evaluation.member_count < 2) + ) { + addIssue(ctx, ["reason_code"], "clustered requires member_count at least 2"); } return; } + if (evaluation.member_count !== undefined) { + addIssue(ctx, ["member_count"], "the verdict stage cannot carry member_count"); + } if (evaluation.output_signature !== undefined) { addIssue(ctx, ["output_signature"], "the verdict stage cannot carry an output signature"); } diff --git a/tests/unit/policy-aggregator-first-half.test.ts b/tests/unit/policy-aggregator-first-half.test.ts index ac0c923..e918949 100644 --- a/tests/unit/policy-aggregator-first-half.test.ts +++ b/tests/unit/policy-aggregator-first-half.test.ts @@ -338,6 +338,53 @@ describe("aggregator policy numeric contracts, orders 60-100", () => { }); }); + it("protects a G0-clamped WARN from critic likely_fp in active and ablated runs", () => { + const clamped = finding({ + signature: "sig-critic-critical-floor", + severity: "WARN", + demoted_from_critical: true, + }); + const input = { + findings: [clamped], + reviewersTotal: 1, + critic: new Map([[clamped.signature, { verdict: "likely_fp" as const }]]), + }; + const active = run("critic-critical-floor-active", input); + const ablated = run("critic-critical-floor-ablated", input, ["judgment.critic"]); + + expect(active.recorder.telemetryError).toBe(false); + expect(ablated.recorder.telemetryError).toBe(false); + expect(numericSummary(active.recorder, "judgment.critic")).toEqual( + PROTECTED_BLOCKING_PRESERVED, + ); + expect(numericSummary(ablated.recorder, "judgment.critic")).toEqual( + PROTECTED_BLOCKING_PRESERVED, + ); + expectSingleEffect(active.result.dedupedFindings[0], { + pass_id: "judgment.critic", + order: 70, + action: "protected", + before: "WARN", + after: "WARN", + reason_code: "critic-likely-fp", + protected_by: "critical-floor", + }); + expectSingleEffect(ablated.result.dedupedFindings[0], { + pass_id: "judgment.critic", + order: 70, + action: "protected", + before: "WARN", + after: "WARN", + reason_code: "critic-likely-fp", + protected_by: "critical-floor", + }); + for (const output of [active, ablated]) { + expect(output.result.dedupedFindings[0]?.critic_verdict).toBeUndefined(); + expect(output.result.dedupedFindings[0]?.critic_reason).toBeUndefined(); + expect(output.result.dedupedFindings[0]?.protected_high_precision).toBeUndefined(); + } + }); + it("records diff-scope no-opportunity, miss, active, ablated, and protected tuples", () => { const ranges = new Map([["src/a.ts", [[10, 14]] as Array<[number, number]>]]); const noLine = run("diff-no-line", { @@ -612,6 +659,7 @@ describe("aggregation cluster lineage", () => { stage_id: "aggregation.cluster", order: 65, reason_code: "clustered", + member_count: 2, input_signatures: [artifact.signature, representative.signature], output_signature: representative.signature, }, @@ -660,8 +708,49 @@ describe("aggregation cluster lineage", () => { stage_id: "aggregation.cluster", order: 65, reason_code: "singleton", + member_count: 1, input_signatures: [singleton.signature], output_signature: singleton.signature, }); }); + + it("records duplicate-signature reviewer contributions as a two-member cluster", () => { + const sharedSignature = "sig-shared-contribution"; + const first = finding({ + signature: sharedSignature, + reviewer: { provider: "codex", model: "m", persona: "quality" }, + }); + const second = finding({ + signature: sharedSignature, + reviewer: { provider: "gemini", model: "m", persona: "quality" }, + }); + const { recorder, result } = run("cluster-shared-signature", { + findings: [first, second], + reviewersTotal: 2, + }); + const finalFinding = result.dedupedFindings[0]; + if (finalFinding === undefined) throw new Error("expected a shared-signature cluster"); + recorder.recordStage({ + stageId: "verdict.compute", + reasonCode: "corroborated-warn", + inputSignatures: [finalFinding.signature], + verdict: result.verdict, + }); + + const trace = recorder.finalize({ + rawResponseSha256: [], + verdict: result.verdict, + finalFindings: result.dedupedFindings, + }); + + expect(finalFinding.members).toHaveLength(2); + expect(trace?.stages[0]).toEqual({ + stage_id: "aggregation.cluster", + order: 65, + reason_code: "clustered", + member_count: 2, + input_signatures: [sharedSignature], + output_signature: sharedSignature, + }); + }); }); diff --git a/tests/unit/policy-catalog.test.ts b/tests/unit/policy-catalog.test.ts index d3fe687..0c791e3 100644 --- a/tests/unit/policy-catalog.test.ts +++ b/tests/unit/policy-catalog.test.ts @@ -90,6 +90,7 @@ describe("policy catalog", () => { protections: [ "claimed-fixed-pin", "self-refutation-visibility", + "critical-floor", "security-correctness-floor", "corroborated-majority", "corroborated-unanimous", diff --git a/tests/unit/policy-trace-recorder.test.ts b/tests/unit/policy-trace-recorder.test.ts index 91e3204..8d704c9 100644 --- a/tests/unit/policy-trace-recorder.test.ts +++ b/tests/unit/policy-trace-recorder.test.ts @@ -473,6 +473,42 @@ describe("policy effect merging", () => { }); describe("PolicyTraceRecorder finalization", () => { + it("records contributor cardinality separately from unique cluster lineage", () => { + const runtime = PolicyTraceRecorder.start({ runId: "run-member-count", iter: 1, ablated: [] }); + const finalWarn = { ...warnFinding, signature: "sig-shared" } satisfies Finding; + + runtime.recordStage({ + stageId: "aggregation.cluster", + reasonCode: "clustered", + memberCount: 2, + inputSignatures: ["sig-shared"], + outputSignature: "sig-shared", + }); + runtime.linkFinal(["sig-shared"], "sig-shared"); + runtime.recordStage({ + stageId: "verdict.compute", + reasonCode: "blocking-present", + inputSignatures: ["sig-shared"], + verdict: "SOFT-PASS", + }); + + const trace = runtime.finalize({ + rawResponseSha256: [], + verdict: "SOFT-PASS", + finalFindings: [finalWarn], + }); + + expect(runtime.telemetryError).toBe(false); + expect(trace?.stages[0]).toEqual({ + stage_id: "aggregation.cluster", + order: 65, + reason_code: "clustered", + member_count: 2, + input_signatures: ["sig-shared"], + output_signature: "sig-shared", + }); + }); + it("links cluster lineage and derives ordered final severity evidence", () => { const runtime = PolicyTraceRecorder.start({ runId: "run-final", iter: 2, ablated: [] }); const finalWarn = { @@ -502,12 +538,14 @@ describe("PolicyTraceRecorder finalization", () => { runtime.recordStage({ stageId: "aggregation.cluster", reasonCode: "clustered", + memberCount: 2, inputSignatures: ["sig-z-final", "sig-member"], outputSignature: "sig-z-final", }); runtime.recordStage({ stageId: "aggregation.cluster", reasonCode: "singleton", + memberCount: 1, inputSignatures: ["sig-a-final"], outputSignature: "sig-a-final", }); diff --git a/tests/unit/policy-trace-schema.test.ts b/tests/unit/policy-trace-schema.test.ts index 0deaeb1..cdbb6e9 100644 --- a/tests/unit/policy-trace-schema.test.ts +++ b/tests/unit/policy-trace-schema.test.ts @@ -138,6 +138,7 @@ function traceWithSingleFinal(severity: "CRITICAL" | "WARN" | "INFO") { stage_id: "aggregation.cluster" as const, order: 65, reason_code: "singleton" as const, + member_count: 1, input_signatures: ["sig-a"], output_signature: "sig-a", }, @@ -193,6 +194,7 @@ function traceWithWarnAndInfoFinals() { stage_id: "aggregation.cluster" as const, order: 65, reason_code: "singleton" as const, + member_count: 1, input_signatures: ["sig-info"], output_signature: "sig-info", }, @@ -200,6 +202,7 @@ function traceWithWarnAndInfoFinals() { stage_id: "aggregation.cluster" as const, order: 65, reason_code: "singleton" as const, + member_count: 1, input_signatures: ["sig-warn"], output_signature: "sig-warn", }, @@ -640,12 +643,51 @@ describe("PolicyPassSummarySchema", () => { }); describe("PolicyStageEvaluationSchema", () => { + it("requires cluster member_count and derives singleton versus clustered from it", () => { + const singleton = { + stage_id: "aggregation.cluster", + order: 65, + reason_code: "singleton", + member_count: 1, + input_signatures: ["sig-a"], + output_signature: "sig-a", + }; + expect(PolicyStageEvaluationSchema.safeParse(singleton).success).toBe(true); + expect( + PolicyStageEvaluationSchema.safeParse({ ...singleton, member_count: undefined }).success, + ).toBe(false); + expect( + PolicyStageEvaluationSchema.safeParse({ + ...singleton, + reason_code: "clustered", + member_count: 2, + }).success, + ).toBe(true); + expect( + PolicyStageEvaluationSchema.safeParse({ ...singleton, reason_code: "clustered" }).success, + ).toBe(false); + expect(PolicyStageEvaluationSchema.safeParse({ ...singleton, member_count: 2 }).success).toBe( + false, + ); + expect( + PolicyStageEvaluationSchema.safeParse({ + stage_id: "verdict.compute", + order: 190, + reason_code: "blocking-present", + member_count: 1, + input_signatures: ["sig-a"], + verdict: "SOFT-PASS", + }).success, + ).toBe(false); + }); + it("accepts closed cluster and verdict stage rows", () => { expect( PolicyStageEvaluationSchema.safeParse({ stage_id: "aggregation.cluster", order: 65, reason_code: "clustered", + member_count: 2, input_signatures: ["sig-a", "sig-b"], output_signature: "sig-a", }).success, @@ -675,7 +717,8 @@ describe("PolicyStageEvaluationSchema", () => { PolicyStageEvaluationSchema.safeParse({ stage_id: "aggregation.cluster", order: 65, - reason_code: "singleton", + reason_code: "clustered", + member_count: 2, input_signatures: ["sig-a", "sig-b"], output_signature: "sig-c", }).success, @@ -889,6 +932,7 @@ describe("PolicyTraceSchema", () => { stage_id: "aggregation.cluster", order: 65, reason_code: "singleton", + member_count: 1, input_signatures: ["sig-a"], output_signature: "sig-a", }, @@ -999,6 +1043,7 @@ describe("PolicyTraceSchema", () => { stage_id: "aggregation.cluster" as const, order: 65, reason_code: "singleton" as const, + member_count: 1, input_signatures: ["sig-a"], output_signature: "sig-a", }, @@ -1048,6 +1093,7 @@ describe("PolicyTraceSchema", () => { stage_id: "aggregation.cluster", order: 65, reason_code: "singleton", + member_count: 1, input_signatures: ["sig-a"], output_signature: "sig-a", }, @@ -1184,6 +1230,7 @@ describe("PolicyTraceSchema", () => { stage_id: "aggregation.cluster" as const, order: 65, reason_code: "singleton" as const, + member_count: 1, input_signatures: ["sig-a"], output_signature: "sig-a", }, @@ -1191,6 +1238,7 @@ describe("PolicyTraceSchema", () => { stage_id: "aggregation.cluster" as const, order: 65, reason_code: "singleton" as const, + member_count: 1, input_signatures: ["sig-b"], output_signature: "sig-b", }, From c545eceb8688efe170dc35cd424d60437168be36 Mon Sep 17 00:00:00 2001 From: Codevena Date: Mon, 10 Aug 2026 06:38:00 +0200 Subject: [PATCH 30/55] feat(policy): trace history and judgment decisions --- src/core/aggregator.ts | 635 ++++++++++---- .../unit/policy-aggregator-first-half.test.ts | 20 - .../policy-aggregator-second-half.test.ts | 806 ++++++++++++++++++ 3 files changed, 1260 insertions(+), 201 deletions(-) create mode 100644 tests/unit/policy-aggregator-second-half.test.ts diff --git a/src/core/aggregator.ts b/src/core/aggregator.ts index 1141a88..c5fd0de 100644 --- a/src/core/aggregator.ts +++ b/src/core/aggregator.ts @@ -956,49 +956,93 @@ export function aggregate(input: AggregateInput): AggregateResult { // is demoted to INFO + tagged. Never dropped — stays visible in the advisory // section, and the decisions-gate already ignores INFO. const fpActive = input.fpActive; - const fpScoped: Finding[] = fpActive - ? foreignScoped.map((f) => { - // Representative first, then members; dedup so a member equal to the - // representative is not double-counted. - const sigs = [...new Set([f.signature, ...(f.members?.map((m) => m.signature) ?? [])])]; - const matched = sigs.filter((s) => fpActive.has(s)); - if (matched.length === 0) return f; - // pattern_id = the first matching signature's entry (deterministic order). - const hit = fpActive.get(matched[0] as string); - const base = f.severity === "INFO" ? f : { ...f, severity: "INFO" as const }; - return { - ...base, - fp_ledger_match: { - pattern_id: (hit as { id: string }).id, - matched_count: matched.length, - suppressed: true, - }, - }; - }) - : foreignScoped; + const fpSignatureEnabled = fpActive !== undefined && fpActive.size > 0; + const fpScoped: Finding[] = + fpSignatureEnabled || input.policyRuntime + ? foreignScoped.map((f) => { + // Representative first, then members; dedup so a member equal to the + // representative is not double-counted. + const sigs = [...new Set([f.signature, ...(f.members?.map((m) => m.signature) ?? [])])]; + const matchingSignatures = fpActive ? sigs.filter((s) => fpActive.has(s)) : []; + const opportunity = fpSignatureEnabled && f.severity !== "INFO"; + const matched = opportunity && matchingSignatures.length > 0; + // pattern_id = the first matching signature's entry (deterministic order). + const hit = matchingSignatures[0] ? fpActive?.get(matchingSignatures[0]) : undefined; + const transitioned = + transitionFinding({ + ...(input.policyRuntime === undefined ? {} : { runtime: input.policyRuntime }), + passId: "history.fp-signature", + finding: f, + opportunity, + matched, + reasonCode: "active-fp-signature", + action: "suppressed", + sourceSignatures: sourceSignatures(f), + proposed: () => ({ + ...f, + severity: "INFO" as const, + fp_ledger_match: { + pattern_id: (hit as { id: string }).id, + matched_count: matchingSignatures.length, + suppressed: true, + }, + }), + }) ?? f; + // INFO-only matches keep their legacy attribution without entering the + // blocking opportunity denominator. + if (f.severity === "INFO" && hit !== undefined) { + return { + ...transitioned, + fp_ledger_match: { + pattern_id: hit.id, + matched_count: matchingSignatures.length, + suppressed: true, + }, + }; + } + return transitioned; + }) + : foreignScoped; // Per-cycle suppression: a finding whose representative OR any member signature // the agent already rejected (reviewer_was_wrong) earlier this cycle is demoted // to INFO (advisory). Breaks the re-flag→re-reject→fp-streak loop: the agent // dispositions a finding once and never sees it as blocking again this cycle. const cycleRejected = input.cycleRejected; + const cycleRejectedEnabled = cycleRejected !== undefined && cycleRejected.size > 0; const cycleScoped: Finding[] = - cycleRejected && cycleRejected.size > 0 + cycleRejectedEnabled || input.policyRuntime ? fpScoped.map((f) => { const sigs = [f.signature, ...(f.members?.map((m) => m.signature) ?? [])]; - if (!sigs.some((s) => cycleRejected.has(s))) return f; + const opportunity = cycleRejectedEnabled && f.severity !== "INFO"; + const matched = opportunity && sigs.some((s) => cycleRejected?.has(s)); // G0b ceiling (codex DoD 2026-06-21): NEVER auto-hide a CRITICAL or any // security/correctness finding via cycleRejected. One false reviewer_was_wrong // rejection must not silence a later REAL CRITICAL of the same signature this cycle // (a fail-open); it re-surfaces for an explicit per-iteration decision instead. - if (f.severity === "CRITICAL" || touchesSecurityOrCorrectness(f)) return f; - return f.severity === "INFO" - ? f - : { + let protectedBy: PolicyProtectionCode | undefined; + if (matched && f.severity === "CRITICAL") protectedBy = "critical-floor"; + else if (matched && touchesSecurityOrCorrectness(f)) { + protectedBy = "security-correctness-floor"; + } + return ( + transitionFinding({ + ...(input.policyRuntime === undefined ? {} : { runtime: input.policyRuntime }), + passId: "history.cycle-rejected", + finding: f, + opportunity, + matched, + reasonCode: "cycle-signature-rejected", + action: "suppressed", + ...(protectedBy === undefined ? {} : { protectedBy }), + sourceSignatures: sourceSignatures(f), + proposed: () => ({ ...f, severity: "INFO" as const, details: `${f.details.slice(0, 1900)}\n\n↓ already rejected earlier this cycle — advisory only.`, - }; + }), + }) ?? f + ); }) : fpScoped; @@ -1009,27 +1053,53 @@ export function aggregate(input: AggregateInput): AggregateResult { // produces the same output: re-running on already-cluster-tagged input // re-applies the identical tag + INFO severity. No explicit short-circuit. const fpClusters = input.fpActiveClusters; - const fpClusterScoped: Finding[] = fpClusters - ? cycleScoped.map((f) => { - // Check the representative AND every merged member rule_id (clustering is - // category/rule-id-independent, so a known-FP rule can ride as a member - // under a different representative). Same file for all cluster members. - const ruleIds = [f.rule_id, ...(f.members?.map((m) => m.rule_id) ?? [])]; - const keys = [...new Set(ruleIds.map((rid) => `${ruleIdToken0(rid)}@${f.file}`))]; - const matchKey = keys.find((k) => fpClusters.has(k)); - const hit = matchKey ? fpClusters.get(matchKey) : undefined; - if (!hit) return f; - const base = f.severity === "INFO" ? f : { ...f, severity: "INFO" as const }; - return { - ...base, - fp_cluster_match: { - cluster_key: hit.key, - member_ids: hit.member_ids, - suppressed: true, - }, - }; - }) - : cycleScoped; + const fpClusterEnabled = fpClusters !== undefined && fpClusters.size > 0; + const fpClusterScoped: Finding[] = + fpClusterEnabled || input.policyRuntime + ? cycleScoped.map((f) => { + // Check the representative AND every merged member rule_id (clustering is + // category/rule-id-independent, so a known-FP rule can ride as a member + // under a different representative). Same file for all cluster members. + const ruleIds = [f.rule_id, ...(f.members?.map((m) => m.rule_id) ?? [])]; + const keys = [...new Set(ruleIds.map((rid) => `${ruleIdToken0(rid)}@${f.file}`))]; + const matchKey = keys.find((key) => fpClusters?.has(key)); + const hit = matchKey ? fpClusters?.get(matchKey) : undefined; + const opportunity = fpClusterEnabled && f.severity !== "INFO"; + const matched = opportunity && hit !== undefined; + const transitioned = + transitionFinding({ + ...(input.policyRuntime === undefined ? {} : { runtime: input.policyRuntime }), + passId: "history.fp-cluster", + finding: f, + opportunity, + matched, + reasonCode: "active-fp-cluster", + action: "suppressed", + sourceSignatures: sourceSignatures(f), + proposed: () => ({ + ...f, + severity: "INFO" as const, + fp_cluster_match: { + cluster_key: (hit as { key: string }).key, + member_ids: (hit as { member_ids: string[] }).member_ids, + suppressed: true, + }, + }), + }) ?? f; + // INFO-only matches retain the legacy explanatory marker. + if (f.severity === "INFO" && hit !== undefined) { + return { + ...transitioned, + fp_cluster_match: { + cluster_key: hit.key, + member_ids: hit.member_ids, + suppressed: true, + }, + }; + } + return transitioned; + }) + : cycleScoped; // Phase 4 #7 — confidence demote: an uncorroborated finding below the floor is // advisory only. Exempt: corroborated findings (majority/unanimous — multiple @@ -1040,9 +1110,8 @@ export function aggregate(input: AggregateInput): AggregateResult { // representative — demoting the cluster would hide it and could flip FAIL→PASS). const floor = input.confidenceFloor ?? 0; const confScoped: Finding[] = - floor > 0 + floor > 0 || input.policyRuntime ? fpClusterScoped.map((f) => { - if (pinned.has(f.signature)) return f; // §4.3: pinned recurrence stays blocking // Cluster confidence = MAX over the representative and all merged members, // so a co-located high-confidence member is never masked by a // low-confidence representative. (memberOf records each member's @@ -1051,37 +1120,94 @@ export function aggregate(input: AggregateInput): AggregateResult { .map((m) => m.confidence) .filter((c): c is number => typeof c === "number"); const maxConfidence = Math.max(f.confidence, ...memberConfs); - if (maxConfidence >= floor) return f; - if (f.consensus === "unanimous" || f.consensus === "majority") return f; - if (f.severity === "CRITICAL" && touchesSecurityOrCorrectness(f)) return f; + const corroborated = f.consensus === "unanimous" || f.consensus === "majority"; + // A from-CRITICAL WARN is already at the G0 floor. The current catalog + // deliberately treats that marker-only repeat as ineligible rather than + // inventing a second WARN→WARN material transition. + const atCriticalFloor = f.severity === "WARN" && f.demoted_from_critical === true; + const opportunity = + floor > 0 && f.severity !== "INFO" && !corroborated && !atCriticalFloor; + const matched = opportunity && maxConfidence < floor; + let protectedBy: PolicyProtectionCode | undefined; + if (matched && pinned.has(f.signature)) protectedBy = "claimed-fixed-pin"; + else if (matched && f.severity === "CRITICAL" && touchesSecurityOrCorrectness(f)) { + protectedBy = "security-correctness-floor"; + } else if (matched && isProtected(f)) { + protectedBy = "high-precision-reviewer"; + } + + const action = f.severity === "CRITICAL" ? "capped" : "demoted"; + const transitioned = + transitionFinding({ + ...(input.policyRuntime === undefined ? {} : { runtime: input.policyRuntime }), + passId: "judgment.confidence", + finding: f, + opportunity, + matched, + reasonCode: "below-confidence-floor", + action, + ...(protectedBy === undefined ? {} : { protectedBy }), + sourceSignatures: sourceSignatures(f), + proposed: () => { + if (f.severity === "CRITICAL") { + const note = `\n\n↓ low reviewer confidence (${maxConfidence.toFixed(2)} < ${floor}) — demoted CRITICAL→WARN; kept blocking pending your decision.`; + return { + ...f, + severity: "WARN" as const, + low_confidence: true, + demoted_from_critical: true, + details: `${f.details.slice(0, 2000 - note.length)}${note}`, + }; + } + const note = `\n\n↓ low reviewer confidence (${maxConfidence.toFixed(2)} < ${floor}) — advisory only.`; + return { + ...f, + severity: "INFO" as const, + low_confidence: true, + details: `${f.details.slice(0, 2000 - note.length)}${note}`, + }; + }, + }) ?? f; + // #4: a high-precision reviewer's blocking finding is not demoted for low // self-reported confidence — its track record outweighs one low confidence call. - if (isProtected(f)) return { ...f, protected_high_precision: true }; - if (f.severity === "INFO") return { ...f, low_confidence: true }; + if (matched && protectedBy === "high-precision-reviewer") { + return { ...transitioned, protected_high_precision: true }; + } + if ( + f.severity === "INFO" && + floor > 0 && + maxConfidence < floor && + !corroborated && + !pinned.has(f.signature) + ) { + return { ...transitioned, low_confidence: true }; + } // G0: this pass sends a non-security/correctness low-confidence CRITICAL DIRECTLY to // INFO (not via DEMOTE) — that would flip a sole demoted-from-CRITICAL finding to a // non-blocking INFO/PASS and auto-hide a possibly-real CRITICAL. CLAMP a from-CRITICAL // at WARN (kept blocking + decision-required) and stamp provenance; a genuine // never-CRITICAL WARN still demotes to INFO as before. - if (f.demoted_from_critical === true || f.severity === "CRITICAL") { + if ( + atCriticalFloor && + floor > 0 && + maxConfidence < floor && + !corroborated && + !pinned.has(f.signature) + ) { + if (isProtected(f)) { + return { ...transitioned, protected_high_precision: true }; + } const note = `\n\n↓ low reviewer confidence (${maxConfidence.toFixed(2)} < ${floor}) — demoted CRITICAL→WARN; kept blocking pending your decision.`; return { - ...f, + ...transitioned, severity: "WARN" as const, low_confidence: true, demoted_from_critical: true, - details: `${f.details.slice(0, 2000 - note.length)}${note}`, + details: `${transitioned.details.slice(0, 2000 - note.length)}${note}`, }; } - const note = `\n\n↓ low reviewer confidence (${maxConfidence.toFixed(2)} < ${floor}) — advisory only.`; - // Truncate the ORIGINAL (not the note) so the explanation is never lost - // — mirrors scopeFindings' demote() and stays within the 2000-char cap. - return { - ...f, - severity: "INFO" as const, - low_confidence: true, - details: `${f.details.slice(0, 2000 - note.length)}${note}`, - }; + return transitioned; }) : fpClusterScoped; @@ -1091,75 +1217,108 @@ export function aggregate(input: AggregateInput): AggregateResult { // security is never demoted; correctness demotes to INFO when demoteCorrectness is on; // INFO is untouched. const repUnreliable = input.repUnreliable; + const reputationEnabled = repUnreliable !== undefined && repUnreliable.size > 0; const repScoped: Finding[] = - repUnreliable && repUnreliable.size > 0 + reputationEnabled || input.policyRuntime ? confScoped.map((f) => { - if (pinned.has(f.signature)) return f; // §4.3: pinned recurrence stays blocking - if (f.severity === "INFO") return f; - if (f.consensus === "unanimous" || f.consensus === "majority") return f; - // security is NEVER softened — hard veto preserved. - if (touchesSecurity(f)) return f; + const corroborated = f.consensus === "unanimous" || f.consensus === "majority"; const isCorrectness = touchesCorrectness(f); - // correctness is exempt UNLESS the demoteCorrectness flag is on. - if (isCorrectness && input.demoteCorrectness !== true) return f; const keys = f.confirmed_by && f.confirmed_by.length > 0 ? f.confirmed_by : [`${f.reviewer.provider}:${f.reviewer.persona}`]; - if (!keys.every((k) => repUnreliable.has(k))) return f; - if (isCorrectness) { - // R5 (field report 2026-07-03): a CRITICAL correctness finding is never sent - // to a no-decision INFO — but the old UNCONDITIONAL exemption predates G0 and - // let a chronically-wrong lone reviewer manufacture unconditional hard FAILs - // (the field's ~38%-precision reviewer blocked turns with hallucinated - // CRITICALs). With corroborateCritical on and >= 2 reviewers, clamp it to a - // decision-required WARN instead: G0 keeps it SOFT-PASS-blocking and forces an - // explicit per-finding decision, so a real data-corruption bug still cannot - // ship silently — the singleton-CRITICAL-must-FAIL invariant (PR#22) is - // untouched because the clamp never fires at reviewersTotal <= 1, and - // security remains an unconditional hard FAIL (returned above). - if (f.severity === "CRITICAL") { - if (input.corroborateCritical !== true || input.reviewersTotal <= 1) return f; - const note = - "\n\n↓ low reviewer reputation — uncorroborated CRITICAL correctness from a chronically-unreliable reviewer; demoted CRITICAL→WARN pending corroboration. Kept blocking + decision-required: verify the claim in the cited code, then fix it or reject it with evidence."; - return { - ...f, - severity: "WARN" as const, - reputation_demoted: true, - demoted_from_critical: true, - reputation_corroboration_required: true, - details: `${f.details.slice(0, 2000 - note.length)}${note}`, - }; - } - // G0: a from-CRITICAL finding that wording-merged into a correctness cluster - // (touchesCorrectness) must not be pushed below WARN by this value-judgment demote — - // keep it a blocking WARN (decision-required), do not soften to advisory INFO. - if (f.demoted_from_critical === true) return f; - // Advisory tier: a chronically-wrong lone reviewer's WARN correctness - // finding goes to INFO (non-blocking). Mirrors the FP-ledger advisory demote. + const opportunity = reputationEnabled && f.severity !== "INFO" && !corroborated; + const matched = opportunity && keys.every((key) => repUnreliable?.has(key)); + const canClampCorrectnessCritical = + input.demoteCorrectness === true && + input.corroborateCritical === true && + input.reviewersTotal > 1; + let protectedBy: PolicyProtectionCode | undefined; + if (matched && pinned.has(f.signature)) protectedBy = "claimed-fixed-pin"; + else if (matched && touchesSecurity(f)) protectedBy = "security-floor"; + else if (matched && isCorrectness && input.demoteCorrectness !== true) { + protectedBy = "correctness-demote-disabled"; + } else if ( + matched && + ((f.severity === "WARN" && f.demoted_from_critical === true) || + (isCorrectness && f.severity === "CRITICAL" && !canClampCorrectnessCritical)) + ) { + protectedBy = "critical-floor"; + } + + const action = isCorrectness && f.severity === "CRITICAL" ? "capped" : "demoted"; + const transitioned = + transitionFinding({ + ...(input.policyRuntime === undefined ? {} : { runtime: input.policyRuntime }), + passId: "judgment.reputation", + finding: f, + opportunity, + matched, + reasonCode: "unreliable-reviewer", + action, + ...(protectedBy === undefined ? {} : { protectedBy }), + sourceSignatures: sourceSignatures(f), + proposed: () => { + if (isCorrectness) { + // R5: an eligible CRITICAL correctness claim is clamped to a + // decision-required WARN pending corroboration. + if (f.severity === "CRITICAL") { + const note = + "\n\n↓ low reviewer reputation — uncorroborated CRITICAL correctness from a chronically-unreliable reviewer; demoted CRITICAL→WARN pending corroboration. Kept blocking + decision-required: verify the claim in the cited code, then fix it or reject it with evidence."; + return { + ...f, + severity: "WARN" as const, + reputation_demoted: true, + demoted_from_critical: true, + reputation_corroboration_required: true, + details: `${f.details.slice(0, 2000 - note.length)}${note}`, + }; + } + const note = + "\n\n↓ low reviewer reputation — correctness finding from an unreliable lone reviewer; advisory only."; + return { + ...f, + severity: "INFO" as const, + reputation_demoted: true, + details: `${f.details.slice(0, 2000 - note.length)}${note}`, + }; + } + + // Pure quality/style: existing one-step demote (CRITICAL→WARN, + // WARN→INFO), with the G0 floor handled above as a protection. + const demoted = demoteOneStep(f); + if (demoted.severity === "drop") return f; + const note = demoted.demoted_from_critical + ? "\n\n↓ low reviewer reputation — demoted CRITICAL→WARN; kept blocking pending your decision." + : "\n\n↓ low reviewer reputation — advisory only."; + return { + ...f, + severity: demoted.severity, + reputation_demoted: true, + ...(demoted.demoted_from_critical ? { demoted_from_critical: true } : {}), + details: `${f.details.slice(0, 2000 - note.length)}${note}`, + }; + }, + }) ?? f; + // Preserve the pre-trace marker/details behavior for a pure-quality + // from-CRITICAL WARN already held at the G0 floor. The trace records + // the attempted second demotion as protected. + if ( + matched && + protectedBy === "critical-floor" && + !isCorrectness && + f.severity === "WARN" + ) { const note = - "\n\n↓ low reviewer reputation — correctness finding from an unreliable lone reviewer; advisory only."; + "\n\n↓ low reviewer reputation — demoted CRITICAL→WARN; kept blocking pending your decision."; return { - ...f, - severity: "INFO" as const, + ...transitioned, reputation_demoted: true, - details: `${f.details.slice(0, 2000 - note.length)}${note}`, + demoted_from_critical: true, + details: `${transitioned.details.slice(0, 2000 - note.length)}${note}`, }; } - // Pure quality/style: existing one-step demote (CRITICAL→WARN, WARN→INFO), - // clamped by G0 so a from-CRITICAL stays ≥WARN (decision-required) instead of INFO. - const demoted = demoteOneStep(f); - if (demoted.severity === "drop") return f; - const note = demoted.demoted_from_critical - ? "\n\n↓ low reviewer reputation — demoted CRITICAL→WARN; kept blocking pending your decision." - : "\n\n↓ low reviewer reputation — advisory only."; - return { - ...f, - severity: demoted.severity, - reputation_demoted: true, - ...(demoted.demoted_from_critical ? { demoted_from_critical: true } : {}), - details: `${f.details.slice(0, 2000 - note.length)}${note}`, - }; + return transitioned; }) : confScoped; @@ -1181,17 +1340,16 @@ export function aggregate(input: AggregateInput): AggregateResult { // finding stays blocking. // Fail-safe: findings without line data and unparseable regions are untouched. const rejectedRegions = input.rejectedRegions; + const regionRejectedEnabled = rejectedRegions !== undefined && rejectedRegions.length > 0; let regionSuppressedCount = 0; const regionScoped: Finding[] = - rejectedRegions && rejectedRegions.length > 0 + regionRejectedEnabled || input.policyRuntime ? repScoped.map((f) => { - if (f.severity === "INFO") return f; - // Same no-usable-line convention as scopeFindings (0/absent → conservative keep). - if (!f.line_start) return f; - if (f.claimed_fixed_recurred) return f; + const opportunity = + regionRejectedEnabled && f.severity !== "INFO" && Boolean(f.line_start); const file = normalizeRepoPath(f.file); const lineEnd = typeof f.line_end === "number" ? f.line_end : f.line_start; - const match = rejectedRegions.find( + const match = rejectedRegions?.find( (r) => typeof r.start_line === "number" && typeof r.end_line === "number" && @@ -1199,31 +1357,73 @@ export function aggregate(input: AggregateInput): AggregateResult { f.line_start <= r.end_line + REGION_WINDOW && lineEnd >= r.start_line - REGION_WINDOW, ); - if (!match) return f; + const matched = opportunity && match !== undefined; const memberCats = [f.category, ...(f.members?.map((m) => m.category) ?? [])]; - const categoryCompatible = memberCats.every((c) => match.categories.includes(c)); - const severityDominated = SEVERITY_RANK[f.severity] <= SEVERITY_RANK[match.severity]; - const suppress = - match.distinct_count >= 2 && - categoryCompatible && - severityDominated && - f.severity !== "CRITICAL" && - !touchesSecurity(f) && - f.demoted_from_critical !== true; - const tag = { - distinct_count: match.distinct_count, - prior_reason: match.reason.slice(0, 200), - suppressed: suppress, - }; - if (!suppress) return { ...f, region_rejected_match: tag }; - regionSuppressedCount++; - const note = `\n\n↓ overlaps a region you already rejected ${match.distinct_count}× this cycle ("${match.reason.slice(0, 120)}") — advisory only.`; - return { - ...f, - severity: "INFO" as const, - region_rejected_match: tag, - details: `${f.details.slice(0, 2000 - note.length)}${note}`, - }; + const categoryCompatible = + match !== undefined && + memberCats.every((category) => match.categories.includes(category)); + const severityDominated = + match !== undefined && SEVERITY_RANK[f.severity] <= SEVERITY_RANK[match.severity]; + let protectedBy: PolicyProtectionCode | undefined; + if (matched && f.claimed_fixed_recurred) protectedBy = "claimed-fixed-pin"; + else if (matched && (match?.distinct_count ?? 0) < 2) { + protectedBy = "insufficient-distinct-rejections"; + } else if (matched && !categoryCompatible) protectedBy = "category-change"; + else if (matched && !severityDominated) protectedBy = "severity-increase"; + else if (matched && (f.severity === "CRITICAL" || f.demoted_from_critical === true)) { + protectedBy = "critical-floor"; + } else if (matched && touchesSecurity(f)) { + protectedBy = "security-correctness-floor"; + } + + const transitioned = + transitionFinding({ + ...(input.policyRuntime === undefined ? {} : { runtime: input.policyRuntime }), + passId: "history.region-rejected", + finding: f, + opportunity, + matched, + reasonCode: "rejected-region-overlap", + action: "suppressed", + ...(protectedBy === undefined ? {} : { protectedBy }), + sourceSignatures: sourceSignatures(f), + proposed: () => { + const hit = match as NonNullable; + const tag = { + distinct_count: hit.distinct_count, + prior_reason: hit.reason.slice(0, 200), + suppressed: true, + }; + const note = `\n\n↓ overlaps a region you already rejected ${hit.distinct_count}× this cycle ("${hit.reason.slice(0, 120)}") — advisory only.`; + return { + ...f, + severity: "INFO" as const, + region_rejected_match: tag, + details: `${f.details.slice(0, 2000 - note.length)}${note}`, + }; + }, + }) ?? f; + + if (matched && protectedBy !== undefined && protectedBy !== "claimed-fixed-pin") { + const hit = match as NonNullable; + return { + ...transitioned, + region_rejected_match: { + distinct_count: hit.distinct_count, + prior_reason: hit.reason.slice(0, 200), + suppressed: false, + }, + }; + } + if ( + matched && + protectedBy === undefined && + transitioned.severity === "INFO" && + transitioned.region_rejected_match?.suppressed === true + ) { + regionSuppressedCount++; + } + return transitioned; }) : repScoped; @@ -1238,19 +1438,49 @@ export function aggregate(input: AggregateInput): AggregateResult { // when EVERY clustered member is also security: a single non-security member keeps the whole // cluster blocking. (members[] includes the representative's own entry; absent → lone finding.) const testScoped: Finding[] = - input.demoteTestSecurity === true + input.demoteTestSecurity === true || input.policyRuntime ? regionScoped.map((f) => { - if (f.category !== "security" || classify(f.file) !== "tests") return f; - if ((f.members ?? []).some((m) => m.category !== "security")) return f; - if (f.severity === "INFO") return { ...f, test_severity_demoted: true }; - const note = - "\n\n↓ security finding on a test/fixture file — not production code; advisory only."; - return { - ...f, - severity: "INFO" as const, - test_severity_demoted: true, - details: `${f.details.slice(0, 2000 - note.length)}${note}`, - }; + const testFile = classify(f.file) === "tests"; + const opportunity = + input.demoteTestSecurity === true && f.severity !== "INFO" && testFile; + const matched = opportunity && f.category === "security"; + const protectedBy = + matched && (f.members ?? []).some((member) => member.category !== "security") + ? "mixed-category-cluster" + : undefined; + const transitioned = + transitionFinding({ + ...(input.policyRuntime === undefined ? {} : { runtime: input.policyRuntime }), + passId: "judgment.test-security", + finding: f, + opportunity, + matched, + reasonCode: "test-only-security", + action: "demoted", + ...(protectedBy === undefined ? {} : { protectedBy }), + sourceSignatures: sourceSignatures(f), + proposed: () => { + const note = + "\n\n↓ security finding on a test/fixture file — not production code; advisory only."; + return { + ...f, + severity: "INFO" as const, + test_severity_demoted: true, + details: `${f.details.slice(0, 2000 - note.length)}${note}`, + }; + }, + }) ?? f; + // Preserve the legacy marker for already-advisory security test findings. + if ( + input.demoteTestSecurity === true && + f.severity === "INFO" && + testFile && + f.category === "security" && + !(f.members ?? []).some((member) => member.category !== "security") + ) { + return { ...transitioned, test_severity_demoted: true }; + } + return transitioned; }) : regionScoped; @@ -1263,22 +1493,38 @@ export function aggregate(input: AggregateInput): AggregateResult { // Fires BEFORE the verdict loop so a capped docs finding no longer trips the singleton // reviewersTotal<=1 hard-FAIL; the sec/corr exemption preserves that path for dangerous docs. const docsScoped: Finding[] = - input.capDocsSeverity === true + input.capDocsSeverity === true || input.policyRuntime ? testScoped.map((f) => { - if (f.severity !== "CRITICAL") return f; - if (classify(f.file) !== "docs") return f; - if (touchesSecurityOrCorrectness(f)) return f; - const demoted = demoteOneStep(f); // CRITICAL → WARN (+ demoted_from_critical) - if (demoted.severity !== "WARN") return f; // defensive: only ever a one-step cap - const note = - "\n\n↓ docs/markdown file — capped CRITICAL→WARN (a stale doc is not a security/data-loss bug); kept blocking pending your decision."; - return { - ...f, - severity: "WARN" as const, - docs_severity_capped: true, - ...(demoted.demoted_from_critical ? { demoted_from_critical: true } : {}), - details: `${f.details.slice(0, 2000 - note.length)}${note}`, - }; + const opportunity = input.capDocsSeverity === true && f.severity === "CRITICAL"; + const matched = opportunity && classify(f.file) === "docs"; + const protectedBy = + matched && touchesSecurityOrCorrectness(f) ? "security-correctness-floor" : undefined; + return ( + transitionFinding({ + ...(input.policyRuntime === undefined ? {} : { runtime: input.policyRuntime }), + passId: "judgment.docs-cap", + finding: f, + opportunity, + matched, + reasonCode: "docs-critical-cap", + action: "capped", + ...(protectedBy === undefined ? {} : { protectedBy }), + sourceSignatures: sourceSignatures(f), + proposed: () => { + const demoted = demoteOneStep(f); // CRITICAL → WARN (+ demoted_from_critical) + if (demoted.severity !== "WARN") return f; + const note = + "\n\n↓ docs/markdown file — capped CRITICAL→WARN (a stale doc is not a security/data-loss bug); kept blocking pending your decision."; + return { + ...f, + severity: "WARN" as const, + docs_severity_capped: true, + ...(demoted.demoted_from_critical ? { demoted_from_critical: true } : {}), + details: `${f.details.slice(0, 2000 - note.length)}${note}`, + }; + }, + }) ?? f + ); }) : testScoped; @@ -1297,6 +1543,9 @@ export function aggregate(input: AggregateInput): AggregateResult { let info = 0; let fail = false; let warnFail = false; + let hardCritical = false; + let corroboratedWarn = false; + let claimedFixedRecurrence = false; for (const f of loneTagged) { if (f.severity === "CRITICAL") { critical++; @@ -1305,8 +1554,10 @@ export function aggregate(input: AggregateInput): AggregateResult { // categories, so a security/correctness concern clustered under a // different representative category is never silently non-blocking. fail = true; + hardCritical = true; } else if (f.consensus === "unanimous" || f.consensus === "majority") { fail = true; + hardCritical = true; } else if (input.reviewersTotal <= 1) { // Single-reviewer panel (e.g. the only non-capped reviewer after a quota // failover): `singleton` is the STRONGEST consensus achievable — there is @@ -1315,20 +1566,24 @@ export function aggregate(input: AggregateInput): AggregateResult { // ≥2 reviewers the consensus gate above still guards against one reviewer's // lone over-call.) fail = true; + hardCritical = true; } else if (f.claimed_fixed_recurred) { // §4.3: a pinned claimed-fixed recurrence still CRITICAL here is a hard FAIL — // the agent claimed to fix it and it is still present; the gate must not open. fail = true; + claimedFixedRecurrence = true; } } else if (f.severity === "WARN") { warn++; if (f.consensus === "unanimous" || f.consensus === "majority") { warnFail = true; + corroboratedWarn = true; } else if (f.claimed_fixed_recurred) { // §4.3: a pinned WARN recurrence forces FAIL even as a singleton — otherwise a // lone-reviewer claimed-fixed recurrence would only SOFT-PASS and the gate would // open, breaking the "still-blocking" guarantee. warnFail = true; + claimedFixedRecurrence = true; } } else { info++; @@ -1344,6 +1599,24 @@ export function aggregate(input: AggregateInput): AggregateResult { else if (warn > 0 || critical > 0) verdict = "SOFT-PASS"; else verdict = "PASS"; + const verdictReason: PolicyReasonCode = hardCritical + ? "hard-critical" + : corroboratedWarn + ? "corroborated-warn" + : claimedFixedRecurrence + ? "claimed-fixed-recurrence" + : warn > 0 || critical > 0 + ? "blocking-present" + : "no-blocking-findings"; + input.policyRuntime?.recordStage({ + stageId: "verdict.compute", + reasonCode: verdictReason, + inputSignatures: loneTagged + .filter((finding) => finding.severity !== "INFO") + .map((finding) => finding.signature), + verdict, + }); + // Reassign unique sequential ids across the merged panel. Each reviewer // numbers its own findings from F-001, so without this two distinct findings // could share an id — and the decisions-gate keys on finding_id, so a single diff --git a/tests/unit/policy-aggregator-first-half.test.ts b/tests/unit/policy-aggregator-first-half.test.ts index e918949..97e9dfb 100644 --- a/tests/unit/policy-aggregator-first-half.test.ts +++ b/tests/unit/policy-aggregator-first-half.test.ts @@ -641,12 +641,6 @@ describe("aggregation cluster lineage", () => { }, ]); - recorder.recordStage({ - stageId: "verdict.compute", - reasonCode: "corroborated-warn", - inputSignatures: [finalFinding.signature], - verdict: result.verdict, - }); const trace = recorder.finalize({ rawResponseSha256: [], verdict: result.verdict, @@ -692,13 +686,6 @@ describe("aggregation cluster lineage", () => { }); const finalFinding = result.dedupedFindings[0]; if (finalFinding === undefined) throw new Error("expected singleton output"); - recorder.recordStage({ - stageId: "verdict.compute", - reasonCode: "blocking-present", - inputSignatures: [finalFinding.signature], - verdict: result.verdict, - }); - const trace = recorder.finalize({ rawResponseSha256: [], verdict: result.verdict, @@ -730,13 +717,6 @@ describe("aggregation cluster lineage", () => { }); const finalFinding = result.dedupedFindings[0]; if (finalFinding === undefined) throw new Error("expected a shared-signature cluster"); - recorder.recordStage({ - stageId: "verdict.compute", - reasonCode: "corroborated-warn", - inputSignatures: [finalFinding.signature], - verdict: result.verdict, - }); - const trace = recorder.finalize({ rawResponseSha256: [], verdict: result.verdict, diff --git a/tests/unit/policy-aggregator-second-half.test.ts b/tests/unit/policy-aggregator-second-half.test.ts new file mode 100644 index 0000000..ee58ad7 --- /dev/null +++ b/tests/unit/policy-aggregator-second-half.test.ts @@ -0,0 +1,806 @@ +import { describe, expect, it } from "bun:test"; +import { type AggregateInput, aggregate } from "../../src/core/aggregator.ts"; +import type { + PolicyPassId, + PolicyProtectionCode, + PolicyReasonCode, +} from "../../src/core/policy/catalog.ts"; +import { PolicyTraceRecorder } from "../../src/core/policy/trace.ts"; +import type { Finding } from "../../src/schemas/finding.ts"; +import type { PolicyTrace } from "../../src/schemas/policy-trace.ts"; + +type NumericSummary = readonly [number, number, number, number, number, number, number, number]; + +const NO_OPPORTUNITY = [1, 0, 0, 0, 0, 0, 0, 0] as const; +const PREDICATE_MISS = [1, 1, 0, 0, 0, 0, 0, 0] as const; +const ACTIVE_BLOCKING_REMOVAL = [1, 1, 1, 1, 0, 1, 0, 0] as const; +const ABLATED_BLOCKING_PRESERVED = [1, 1, 1, 0, 0, 0, 1, 0] as const; +const PROTECTED_BLOCKING_PRESERVED = [1, 1, 1, 0, 1, 0, 1, 0] as const; +const ACTIVE_BLOCKING_PRESERVED = [1, 1, 1, 1, 0, 0, 1, 0] as const; + +function finding(overrides: Partial = {}): Finding { + return { + id: "F-001", + signature: "sig-policy", + severity: "WARN", + category: "quality", + rule_id: "policy-contract", + file: "src/a.ts", + line_start: 10, + line_end: 10, + message: "A concrete policy finding", + details: "The implementation has a concrete defect.", + reviewer: { provider: "codex", model: "m", persona: "quality" }, + confidence: 0.9, + consensus: "singleton", + ...overrides, + }; +} + +function runtime(runId: string, ablated: readonly PolicyPassId[] = []): PolicyTraceRecorder { + return PolicyTraceRecorder.start({ runId, iter: 1, ablated }); +} + +function run( + runId: string, + input: AggregateInput, + ablated: readonly PolicyPassId[] = [], +): { recorder: PolicyTraceRecorder; result: ReturnType } { + const recorder = runtime(runId, ablated); + return { recorder, result: aggregate({ ...input, policyRuntime: recorder }) }; +} + +function numericSummary(recorder: PolicyTraceRecorder, passId: PolicyPassId): NumericSummary { + const summary = recorder.summary(passId); + expect(summary.status).toBe("ran"); + if (summary.status !== "ran") throw new Error(`${passId} did not run`); + return [ + summary.considered, + summary.opportunities, + summary.would_apply, + summary.applied, + summary.protected, + summary.blocking_removed, + summary.blocking_preserved, + summary.dropped, + ]; +} + +function expectEffect( + value: Finding | undefined, + expected: { + pass_id: PolicyPassId; + order: number; + action: "demoted" | "capped" | "protected" | "suppressed"; + before: Finding["severity"]; + after: Finding["severity"]; + reason_code: PolicyReasonCode; + protected_by?: PolicyProtectionCode; + source_signatures?: string[]; + }, +): void { + if (value === undefined) throw new Error("expected a visible finding"); + const { source_signatures = [value.signature], ...effect } = expected; + expect(value.policy_effects).toContainEqual({ + ...effect, + source_signatures, + }); +} + +function finalized(output: ReturnType): PolicyTrace { + const trace = output.recorder.finalize({ + rawResponseSha256: [], + verdict: output.result.verdict, + finalFindings: output.result.dedupedFindings, + }); + expect(output.recorder.telemetryError).toBe(false); + if (trace === null) throw new Error("expected a complete policy trace"); + return trace; +} + +function activeCluster() { + return new Map([["policy@src/a.ts", { key: "policy@src/a.ts", member_ids: ["FP-001"] }]]); +} + +function rejectedRegion( + overrides: Partial[number]> = {}, +) { + return { + file: "src/a.ts", + start_line: 8, + end_line: 12, + severity: "WARN" as const, + categories: ["quality" as const], + reason: "this exact region was already disproven twice", + distinct_count: 2, + ...overrides, + }; +} + +describe("aggregator policy numeric contracts, orders 110-180", () => { + it("records FP-signature no-opportunity, miss, active, and ablated tuples", () => { + const info = run("fp-signature-info", { + findings: [finding({ severity: "INFO" })], + reviewersTotal: 1, + fpActive: new Map([["sig-policy", { id: "FP-001" }]]), + }); + const miss = run("fp-signature-miss", { + findings: [finding()], + reviewersTotal: 1, + fpActive: new Map([["other", { id: "FP-001" }]]), + }); + const input = { + findings: [finding()], + reviewersTotal: 1, + fpActive: new Map([["sig-policy", { id: "FP-001" }]]), + }; + const active = run("fp-signature-active", input); + const ablated = run("fp-signature-ablated", input, ["history.fp-signature"]); + + expect(numericSummary(info.recorder, "history.fp-signature")).toEqual(NO_OPPORTUNITY); + expect(numericSummary(miss.recorder, "history.fp-signature")).toEqual(PREDICATE_MISS); + expect(numericSummary(active.recorder, "history.fp-signature")).toEqual( + ACTIVE_BLOCKING_REMOVAL, + ); + expect(numericSummary(ablated.recorder, "history.fp-signature")).toEqual( + ABLATED_BLOCKING_PRESERVED, + ); + expect(active.result.dedupedFindings[0]).toMatchObject({ + severity: "INFO", + fp_ledger_match: { pattern_id: "FP-001", matched_count: 1, suppressed: true }, + }); + expect(ablated.result.dedupedFindings[0]).not.toHaveProperty("fp_ledger_match"); + expectEffect(active.result.dedupedFindings[0], { + pass_id: "history.fp-signature", + order: 110, + action: "suppressed", + before: "WARN", + after: "INFO", + reason_code: "active-fp-signature", + }); + }); + + it("records cycle-rejection no-opportunity, miss, active, ablated, and protected tuples", () => { + const info = run("cycle-info", { + findings: [finding({ severity: "INFO" })], + reviewersTotal: 1, + cycleRejected: new Set(["sig-policy"]), + }); + const miss = run("cycle-miss", { + findings: [finding()], + reviewersTotal: 1, + cycleRejected: new Set(["other"]), + }); + const input = { + findings: [finding()], + reviewersTotal: 1, + cycleRejected: new Set(["sig-policy"]), + }; + const active = run("cycle-active", input); + const ablated = run("cycle-ablated", input, ["history.cycle-rejected"]); + const protectedResult = run("cycle-protected", { + findings: [finding({ category: "correctness" })], + reviewersTotal: 1, + cycleRejected: new Set(["sig-policy"]), + }); + const criticalFloor = run("cycle-critical-floor", { + findings: [finding({ severity: "CRITICAL" })], + reviewersTotal: 2, + cycleRejected: new Set(["sig-policy"]), + }); + + expect(numericSummary(info.recorder, "history.cycle-rejected")).toEqual(NO_OPPORTUNITY); + expect(numericSummary(miss.recorder, "history.cycle-rejected")).toEqual(PREDICATE_MISS); + expect(numericSummary(active.recorder, "history.cycle-rejected")).toEqual( + ACTIVE_BLOCKING_REMOVAL, + ); + expect(numericSummary(ablated.recorder, "history.cycle-rejected")).toEqual( + ABLATED_BLOCKING_PRESERVED, + ); + expect(numericSummary(protectedResult.recorder, "history.cycle-rejected")).toEqual( + PROTECTED_BLOCKING_PRESERVED, + ); + expect(numericSummary(criticalFloor.recorder, "history.cycle-rejected")).toEqual( + PROTECTED_BLOCKING_PRESERVED, + ); + expect(ablated.result.dedupedFindings[0]?.severity).toBe("WARN"); + expectEffect(protectedResult.result.dedupedFindings[0], { + pass_id: "history.cycle-rejected", + order: 120, + action: "protected", + before: "WARN", + after: "WARN", + reason_code: "cycle-signature-rejected", + protected_by: "security-correctness-floor", + }); + expectEffect(criticalFloor.result.dedupedFindings[0], { + pass_id: "history.cycle-rejected", + order: 120, + action: "protected", + before: "CRITICAL", + after: "CRITICAL", + reason_code: "cycle-signature-rejected", + protected_by: "critical-floor", + }); + }); + + it("records FP-cluster no-opportunity, miss, active, and ablated tuples", () => { + const info = run("fp-cluster-info", { + findings: [finding({ severity: "INFO" })], + reviewersTotal: 1, + fpActiveClusters: activeCluster(), + }); + const miss = run("fp-cluster-miss", { + findings: [finding({ rule_id: "other-contract" })], + reviewersTotal: 1, + fpActiveClusters: activeCluster(), + }); + const input = { findings: [finding()], reviewersTotal: 1, fpActiveClusters: activeCluster() }; + const active = run("fp-cluster-active", input); + const ablated = run("fp-cluster-ablated", input, ["history.fp-cluster"]); + + expect(numericSummary(info.recorder, "history.fp-cluster")).toEqual(NO_OPPORTUNITY); + expect(numericSummary(miss.recorder, "history.fp-cluster")).toEqual(PREDICATE_MISS); + expect(numericSummary(active.recorder, "history.fp-cluster")).toEqual(ACTIVE_BLOCKING_REMOVAL); + expect(numericSummary(ablated.recorder, "history.fp-cluster")).toEqual( + ABLATED_BLOCKING_PRESERVED, + ); + expect(active.result.dedupedFindings[0]?.fp_cluster_match).toEqual({ + cluster_key: "policy@src/a.ts", + member_ids: ["FP-001"], + suppressed: true, + }); + expect(ablated.result.dedupedFindings[0]).not.toHaveProperty("fp_cluster_match"); + expectEffect(active.result.dedupedFindings[0], { + pass_id: "history.fp-cluster", + order: 130, + action: "suppressed", + before: "WARN", + after: "INFO", + reason_code: "active-fp-cluster", + }); + }); + + it("records confidence no-opportunity, miss, active, ablated, and high-precision protection", () => { + const majorityA = finding({ + signature: "sig-majority-a", + confidence: 0.2, + reviewer: { provider: "codex", model: "m", persona: "quality" }, + }); + const majorityB = finding({ + signature: "sig-majority-b", + confidence: 0.2, + reviewer: { provider: "gemini", model: "m", persona: "quality" }, + }); + const noOpportunity = run("confidence-majority", { + findings: [majorityA, majorityB], + reviewersTotal: 2, + confidenceFloor: 0.5, + }); + const miss = run("confidence-miss", { + findings: [finding({ confidence: 0.5 })], + reviewersTotal: 1, + confidenceFloor: 0.5, + }); + const low = finding({ confidence: 0.2 }); + const input = { findings: [low], reviewersTotal: 1, confidenceFloor: 0.5 }; + const active = run("confidence-active", input); + const ablated = run("confidence-ablated", input, ["judgment.confidence"]); + const protectedResult = run("confidence-protected", { + ...input, + protectedReviewers: new Set(["codex"]), + }); + + expect(numericSummary(noOpportunity.recorder, "judgment.confidence")).toEqual(NO_OPPORTUNITY); + expect(numericSummary(miss.recorder, "judgment.confidence")).toEqual(PREDICATE_MISS); + expect(numericSummary(active.recorder, "judgment.confidence")).toEqual(ACTIVE_BLOCKING_REMOVAL); + expect(numericSummary(ablated.recorder, "judgment.confidence")).toEqual( + ABLATED_BLOCKING_PRESERVED, + ); + expect(numericSummary(protectedResult.recorder, "judgment.confidence")).toEqual( + PROTECTED_BLOCKING_PRESERVED, + ); + expect(protectedResult.result.dedupedFindings[0]).toMatchObject({ + severity: "WARN", + protected_high_precision: true, + }); + expectEffect(protectedResult.result.dedupedFindings[0], { + pass_id: "judgment.confidence", + order: 140, + action: "protected", + before: "WARN", + after: "WARN", + reason_code: "below-confidence-floor", + protected_by: "high-precision-reviewer", + }); + }); + + it("records confidence security/pin protections and the CRITICAL G0 clamp", () => { + const critical = run("confidence-critical", { + findings: [finding({ severity: "CRITICAL", confidence: 0.1 })], + reviewersTotal: 2, + confidenceFloor: 0.5, + }); + const security = run("confidence-security", { + findings: [finding({ severity: "CRITICAL", category: "security", confidence: 0.1 })], + reviewersTotal: 2, + confidenceFloor: 0.5, + }); + const pinned = run("confidence-pinned", { + findings: [finding({ confidence: 0.1 })], + reviewersTotal: 1, + confidenceFloor: 0.5, + claimedFixed: new Map([["sig-policy", 1]]), + }); + + expect(numericSummary(critical.recorder, "judgment.confidence")).toEqual( + ACTIVE_BLOCKING_PRESERVED, + ); + expect(critical.result.dedupedFindings[0]).toMatchObject({ + severity: "WARN", + low_confidence: true, + demoted_from_critical: true, + }); + expectEffect(critical.result.dedupedFindings[0], { + pass_id: "judgment.confidence", + order: 140, + action: "capped", + before: "CRITICAL", + after: "WARN", + reason_code: "below-confidence-floor", + }); + expect(numericSummary(security.recorder, "judgment.confidence")).toEqual( + PROTECTED_BLOCKING_PRESERVED, + ); + expectEffect(security.result.dedupedFindings[0], { + pass_id: "judgment.confidence", + order: 140, + action: "protected", + before: "CRITICAL", + after: "CRITICAL", + reason_code: "below-confidence-floor", + protected_by: "security-correctness-floor", + }); + expect(numericSummary(pinned.recorder, "judgment.confidence")).toEqual( + PROTECTED_BLOCKING_PRESERVED, + ); + expectEffect(pinned.result.dedupedFindings[0], { + pass_id: "judgment.confidence", + order: 140, + action: "protected", + before: "WARN", + after: "WARN", + reason_code: "below-confidence-floor", + protected_by: "claimed-fixed-pin", + }); + }); + + it("records reputation no-opportunity, miss, active, ablated, and security protection", () => { + const majorityA = finding({ + signature: "sig-reputation-a", + reviewer: { provider: "codex", model: "m", persona: "quality" }, + }); + const majorityB = finding({ + signature: "sig-reputation-b", + reviewer: { provider: "gemini", model: "m", persona: "quality" }, + }); + const noOpportunity = run("reputation-majority", { + findings: [majorityA, majorityB], + reviewersTotal: 2, + repUnreliable: new Set(["codex:quality", "gemini:quality"]), + }); + const miss = run("reputation-miss", { + findings: [finding()], + reviewersTotal: 1, + repUnreliable: new Set(["gemini:quality"]), + }); + const input = { + findings: [finding()], + reviewersTotal: 1, + repUnreliable: new Set(["codex:quality"]), + }; + const active = run("reputation-active", input); + const ablated = run("reputation-ablated", input, ["judgment.reputation"]); + const protectedResult = run("reputation-security", { + findings: [finding({ category: "security" })], + reviewersTotal: 1, + repUnreliable: new Set(["codex:quality"]), + }); + + expect(numericSummary(noOpportunity.recorder, "judgment.reputation")).toEqual(NO_OPPORTUNITY); + expect(numericSummary(miss.recorder, "judgment.reputation")).toEqual(PREDICATE_MISS); + expect(numericSummary(active.recorder, "judgment.reputation")).toEqual(ACTIVE_BLOCKING_REMOVAL); + expect(numericSummary(ablated.recorder, "judgment.reputation")).toEqual( + ABLATED_BLOCKING_PRESERVED, + ); + expect(numericSummary(protectedResult.recorder, "judgment.reputation")).toEqual( + PROTECTED_BLOCKING_PRESERVED, + ); + expectEffect(protectedResult.result.dedupedFindings[0], { + pass_id: "judgment.reputation", + order: 150, + action: "protected", + before: "WARN", + after: "WARN", + reason_code: "unreliable-reviewer", + protected_by: "security-floor", + }); + }); + + it("records reputation CRITICAL quality/correctness clamps and remaining guards", () => { + const quality = run("reputation-quality-critical", { + findings: [finding({ severity: "CRITICAL" })], + reviewersTotal: 2, + repUnreliable: new Set(["codex:quality"]), + }); + const correctness = run("reputation-correctness-critical", { + findings: [finding({ severity: "CRITICAL", category: "correctness" })], + reviewersTotal: 2, + repUnreliable: new Set(["codex:quality"]), + demoteCorrectness: true, + corroborateCritical: true, + }); + const correctnessDisabled = run("reputation-correctness-disabled", { + findings: [finding({ category: "correctness" })], + reviewersTotal: 2, + repUnreliable: new Set(["codex:quality"]), + }); + const criticalFloor = run("reputation-critical-floor", { + findings: [finding({ demoted_from_critical: true })], + reviewersTotal: 2, + repUnreliable: new Set(["codex:quality"]), + }); + const pinned = run("reputation-pinned", { + findings: [finding()], + reviewersTotal: 2, + repUnreliable: new Set(["codex:quality"]), + claimedFixed: new Map([["sig-policy", 1]]), + }); + + expect(numericSummary(quality.recorder, "judgment.reputation")).toEqual( + ACTIVE_BLOCKING_PRESERVED, + ); + expectEffect(quality.result.dedupedFindings[0], { + pass_id: "judgment.reputation", + order: 150, + action: "demoted", + before: "CRITICAL", + after: "WARN", + reason_code: "unreliable-reviewer", + }); + expect(numericSummary(correctness.recorder, "judgment.reputation")).toEqual( + ACTIVE_BLOCKING_PRESERVED, + ); + expect(correctness.result.dedupedFindings[0]).toMatchObject({ + severity: "WARN", + reputation_corroboration_required: true, + }); + expectEffect(correctness.result.dedupedFindings[0], { + pass_id: "judgment.reputation", + order: 150, + action: "capped", + before: "CRITICAL", + after: "WARN", + reason_code: "unreliable-reviewer", + }); + for (const [output, protectedBy] of [ + [correctnessDisabled, "correctness-demote-disabled"], + [criticalFloor, "critical-floor"], + [pinned, "claimed-fixed-pin"], + ] as const) { + expect(numericSummary(output.recorder, "judgment.reputation")).toEqual( + PROTECTED_BLOCKING_PRESERVED, + ); + expectEffect(output.result.dedupedFindings[0], { + pass_id: "judgment.reputation", + order: 150, + action: "protected", + before: "WARN", + after: "WARN", + reason_code: "unreliable-reviewer", + protected_by: protectedBy, + }); + } + }); + + it("records region no-opportunity, miss, active, ablated, and insufficient-history protection", () => { + const noOpportunity = run("region-no-line", { + findings: [finding({ line_start: 0, line_end: 0 })], + reviewersTotal: 1, + rejectedRegions: [rejectedRegion()], + }); + const miss = run("region-miss", { + findings: [finding({ line_start: 40, line_end: 40 })], + reviewersTotal: 1, + rejectedRegions: [rejectedRegion()], + }); + const input = { + findings: [finding()], + reviewersTotal: 1, + rejectedRegions: [rejectedRegion()], + }; + const active = run("region-active", input); + const ablated = run("region-ablated", input, ["history.region-rejected"]); + const protectedResult = run("region-insufficient", { + ...input, + rejectedRegions: [rejectedRegion({ distinct_count: 1 })], + }); + + expect(numericSummary(noOpportunity.recorder, "history.region-rejected")).toEqual( + NO_OPPORTUNITY, + ); + expect(numericSummary(miss.recorder, "history.region-rejected")).toEqual(PREDICATE_MISS); + expect(numericSummary(active.recorder, "history.region-rejected")).toEqual( + ACTIVE_BLOCKING_REMOVAL, + ); + expect(numericSummary(ablated.recorder, "history.region-rejected")).toEqual( + ABLATED_BLOCKING_PRESERVED, + ); + expect(numericSummary(protectedResult.recorder, "history.region-rejected")).toEqual( + PROTECTED_BLOCKING_PRESERVED, + ); + expect(active.result.regionSuppressedCount).toBe(1); + expect(ablated.result.regionSuppressedCount).toBe(0); + expect(ablated.result.dedupedFindings[0]).not.toHaveProperty("region_rejected_match"); + expect(protectedResult.result.dedupedFindings[0]?.region_rejected_match?.suppressed).toBe( + false, + ); + expectEffect(protectedResult.result.dedupedFindings[0], { + pass_id: "history.region-rejected", + order: 160, + action: "protected", + before: "WARN", + after: "WARN", + reason_code: "rejected-region-overlap", + protected_by: "insufficient-distinct-rejections", + }); + }); + + it("records every region-rejection guard on the attempted overlap", () => { + const cases: Array<{ + name: string; + input: AggregateInput; + protectedBy: PolicyProtectionCode; + before?: Finding["severity"]; + }> = [ + { + name: "claimed-fixed", + input: { + findings: [finding()], + reviewersTotal: 2, + rejectedRegions: [rejectedRegion()], + claimedFixed: new Map([["sig-policy", 1]]), + }, + protectedBy: "claimed-fixed-pin", + }, + { + name: "category-change", + input: { + findings: [finding({ category: "performance" })], + reviewersTotal: 2, + rejectedRegions: [rejectedRegion()], + }, + protectedBy: "category-change", + }, + { + name: "severity-increase", + input: { + findings: [finding({ severity: "CRITICAL" })], + reviewersTotal: 2, + rejectedRegions: [rejectedRegion()], + }, + protectedBy: "severity-increase", + before: "CRITICAL", + }, + { + name: "critical-floor", + input: { + findings: [finding({ demoted_from_critical: true })], + reviewersTotal: 2, + rejectedRegions: [rejectedRegion()], + }, + protectedBy: "critical-floor", + }, + { + name: "security-floor", + input: { + findings: [finding({ category: "security" })], + reviewersTotal: 2, + rejectedRegions: [rejectedRegion({ categories: ["security"] })], + }, + protectedBy: "security-correctness-floor", + }, + ]; + + for (const testCase of cases) { + const output = run(`region-${testCase.name}`, testCase.input); + expect(numericSummary(output.recorder, "history.region-rejected")).toEqual( + PROTECTED_BLOCKING_PRESERVED, + ); + expectEffect(output.result.dedupedFindings[0], { + pass_id: "history.region-rejected", + order: 160, + action: "protected", + before: testCase.before ?? "WARN", + after: testCase.before ?? "WARN", + reason_code: "rejected-region-overlap", + protected_by: testCase.protectedBy, + }); + } + }); + + it("records test-security no-opportunity, miss, active, ablated, and mixed-cluster protection", () => { + const info = run("test-security-info", { + findings: [finding({ severity: "INFO", category: "security", file: "src/a.test.ts" })], + reviewersTotal: 1, + demoteTestSecurity: true, + }); + const miss = run("test-security-miss", { + findings: [finding({ file: "src/a.test.ts" })], + reviewersTotal: 1, + demoteTestSecurity: true, + }); + const security = finding({ category: "security", file: "src/a.test.ts" }); + const input = { findings: [security], reviewersTotal: 1, demoteTestSecurity: true }; + const active = run("test-security-active", input); + const ablated = run("test-security-ablated", input, ["judgment.test-security"]); + const mixedSecurity = finding({ + signature: "sig-test-security", + category: "security", + file: "src/a.test.ts", + message: "same test issue reported here", + }); + const mixedCorrectness = finding({ + signature: "sig-test-correctness", + category: "correctness", + file: "src/a.test.ts", + message: "same test issue reported here", + }); + const protectedResult = run("test-security-protected", { + findings: [mixedSecurity, mixedCorrectness], + reviewersTotal: 1, + demoteTestSecurity: true, + }); + + expect(numericSummary(info.recorder, "judgment.test-security")).toEqual(NO_OPPORTUNITY); + expect(numericSummary(miss.recorder, "judgment.test-security")).toEqual(PREDICATE_MISS); + expect(numericSummary(active.recorder, "judgment.test-security")).toEqual( + ACTIVE_BLOCKING_REMOVAL, + ); + expect(numericSummary(ablated.recorder, "judgment.test-security")).toEqual( + ABLATED_BLOCKING_PRESERVED, + ); + expect(numericSummary(protectedResult.recorder, "judgment.test-security")).toEqual( + PROTECTED_BLOCKING_PRESERVED, + ); + expect(ablated.result.dedupedFindings[0]).not.toHaveProperty("test_severity_demoted"); + const protectedFinding = protectedResult.result.dedupedFindings[0]; + expect(protectedFinding?.members).toHaveLength(2); + expectEffect(protectedFinding, { + pass_id: "judgment.test-security", + order: 170, + action: "protected", + before: "WARN", + after: "WARN", + reason_code: "test-only-security", + protected_by: "mixed-category-cluster", + source_signatures: ["sig-test-correctness", "sig-test-security"], + }); + }); + + it("records docs-cap no-opportunity, miss, active, ablated, and protected tuples", () => { + const noOpportunity = run("docs-warn", { + findings: [finding({ file: "README.md" })], + reviewersTotal: 1, + capDocsSeverity: true, + }); + const miss = run("docs-source", { + findings: [finding({ severity: "CRITICAL" })], + reviewersTotal: 2, + capDocsSeverity: true, + }); + const docs = finding({ severity: "CRITICAL", file: "README.md" }); + const input = { findings: [docs], reviewersTotal: 1, capDocsSeverity: true }; + const active = run("docs-active", input); + const ablated = run("docs-ablated", input, ["judgment.docs-cap"]); + const protectedResult = run("docs-protected", { + findings: [finding({ severity: "CRITICAL", category: "correctness", file: "README.md" })], + reviewersTotal: 1, + capDocsSeverity: true, + }); + + expect(numericSummary(noOpportunity.recorder, "judgment.docs-cap")).toEqual(NO_OPPORTUNITY); + expect(numericSummary(miss.recorder, "judgment.docs-cap")).toEqual(PREDICATE_MISS); + expect(numericSummary(active.recorder, "judgment.docs-cap")).toEqual(ACTIVE_BLOCKING_PRESERVED); + expect(numericSummary(ablated.recorder, "judgment.docs-cap")).toEqual( + ABLATED_BLOCKING_PRESERVED, + ); + expect(numericSummary(protectedResult.recorder, "judgment.docs-cap")).toEqual( + PROTECTED_BLOCKING_PRESERVED, + ); + expect(active.result.dedupedFindings[0]).toMatchObject({ + severity: "WARN", + docs_severity_capped: true, + demoted_from_critical: true, + }); + expect(ablated.result.dedupedFindings[0]?.severity).toBe("CRITICAL"); + expect(ablated.result.dedupedFindings[0]).not.toHaveProperty("docs_severity_capped"); + expectEffect(active.result.dedupedFindings[0], { + pass_id: "judgment.docs-cap", + order: 180, + action: "capped", + before: "CRITICAL", + after: "WARN", + reason_code: "docs-critical-cap", + }); + expectEffect(protectedResult.result.dedupedFindings[0], { + pass_id: "judgment.docs-cap", + order: 180, + action: "protected", + before: "CRITICAL", + after: "CRITICAL", + reason_code: "docs-critical-cap", + protected_by: "security-correctness-floor", + }); + }); +}); + +describe("verdict.compute trace stage", () => { + it("records exactly one closed verdict reason for every judgment branch", () => { + const hardCritical = run("verdict-hard-critical", { + findings: [finding({ category: "security", severity: "CRITICAL" })], + reviewersTotal: 2, + }); + const corroboratedA = finding({ + signature: "sig-corroborated-a", + reviewer: { provider: "codex", model: "m", persona: "quality" }, + }); + const corroboratedB = finding({ + signature: "sig-corroborated-b", + reviewer: { provider: "gemini", model: "m", persona: "quality" }, + }); + const corroboratedWarn = run("verdict-corroborated-warn", { + findings: [corroboratedA, corroboratedB], + reviewersTotal: 2, + }); + const claimedFixed = run("verdict-claimed-fixed", { + findings: [finding()], + reviewersTotal: 2, + claimedFixed: new Map([["sig-policy", 1]]), + }); + const blocking = run("verdict-blocking", { findings: [finding()], reviewersTotal: 2 }); + const noBlocking = run("verdict-no-blocking", { + findings: [finding({ severity: "INFO" })], + reviewersTotal: 2, + }); + + const cases = [ + [hardCritical, "hard-critical", "FAIL"], + [corroboratedWarn, "corroborated-warn", "FAIL"], + [claimedFixed, "claimed-fixed-recurrence", "FAIL"], + [blocking, "blocking-present", "SOFT-PASS"], + [noBlocking, "no-blocking-findings", "PASS"], + ] as const; + for (const [output, reason, verdict] of cases) { + const trace = finalized(output); + const verdictRows = trace.stages.filter((stage) => stage.stage_id === "verdict.compute"); + expect(verdictRows).toHaveLength(1); + expect(verdictRows[0]).toMatchObject({ reason_code: reason, verdict }); + } + }); + + it("uses the deterministic final-finding order for verdict blocking signatures", () => { + const output = run("verdict-signature-order", { + findings: [ + finding({ signature: "sig-a", file: "src/b.ts", line_start: 20, line_end: 20 }), + finding({ signature: "sig-z", file: "src/a.ts", line_start: 10, line_end: 10 }), + ], + reviewersTotal: 2, + }); + const trace = finalized(output); + const stage = trace.stages.find((row) => row.stage_id === "verdict.compute"); + + expect(output.result.dedupedFindings.map((item) => item.signature)).toEqual(["sig-z", "sig-a"]); + expect(stage?.input_signatures).toEqual(["sig-z", "sig-a"]); + }); +}); From b68a893f2e01dda2a0db50c912d2c22732b630d4 Mon Sep 17 00:00:00 2001 From: Codevena Date: Mon, 10 Aug 2026 07:04:02 +0200 Subject: [PATCH 31/55] fix(policy): preserve inactive and ablated trace semantics --- src/core/aggregator.ts | 959 +++++++++--------- src/core/policy/trace.ts | 69 +- .../unit/policy-aggregator-first-half.test.ts | 16 +- .../policy-aggregator-second-half.test.ts | 209 ++++ tests/unit/policy-trace-recorder.test.ts | 149 +++ 5 files changed, 912 insertions(+), 490 deletions(-) diff --git a/src/core/aggregator.ts b/src/core/aggregator.ts index c5fd0de..33bb055 100644 --- a/src/core/aggregator.ts +++ b/src/core/aggregator.ts @@ -957,52 +957,55 @@ export function aggregate(input: AggregateInput): AggregateResult { // section, and the decisions-gate already ignores INFO. const fpActive = input.fpActive; const fpSignatureEnabled = fpActive !== undefined && fpActive.size > 0; - const fpScoped: Finding[] = - fpSignatureEnabled || input.policyRuntime - ? foreignScoped.map((f) => { - // Representative first, then members; dedup so a member equal to the - // representative is not double-counted. - const sigs = [...new Set([f.signature, ...(f.members?.map((m) => m.signature) ?? [])])]; - const matchingSignatures = fpActive ? sigs.filter((s) => fpActive.has(s)) : []; - const opportunity = fpSignatureEnabled && f.severity !== "INFO"; - const matched = opportunity && matchingSignatures.length > 0; - // pattern_id = the first matching signature's entry (deterministic order). - const hit = matchingSignatures[0] ? fpActive?.get(matchingSignatures[0]) : undefined; - const transitioned = - transitionFinding({ - ...(input.policyRuntime === undefined ? {} : { runtime: input.policyRuntime }), - passId: "history.fp-signature", - finding: f, - opportunity, - matched, - reasonCode: "active-fp-signature", - action: "suppressed", - sourceSignatures: sourceSignatures(f), - proposed: () => ({ - ...f, - severity: "INFO" as const, - fp_ledger_match: { - pattern_id: (hit as { id: string }).id, - matched_count: matchingSignatures.length, - suppressed: true, - }, - }), - }) ?? f; - // INFO-only matches keep their legacy attribution without entering the - // blocking opportunity denominator. - if (f.severity === "INFO" && hit !== undefined) { - return { - ...transitioned, + if (!fpSignatureEnabled) { + input.policyRuntime?.markInactive("history.fp-signature", "stage-precondition-miss"); + } + const fpSignatureAblated = input.policyRuntime?.isAblated("history.fp-signature") ?? false; + const fpScoped: Finding[] = fpSignatureEnabled + ? foreignScoped.map((f) => { + // Representative first, then members; dedup so a member equal to the + // representative is not double-counted. + const sigs = [...new Set([f.signature, ...(f.members?.map((m) => m.signature) ?? [])])]; + const matchingSignatures = fpActive ? sigs.filter((s) => fpActive.has(s)) : []; + const opportunity = fpSignatureEnabled && f.severity !== "INFO"; + const matched = opportunity && matchingSignatures.length > 0; + // pattern_id = the first matching signature's entry (deterministic order). + const hit = matchingSignatures[0] ? fpActive?.get(matchingSignatures[0]) : undefined; + const transitioned = + transitionFinding({ + ...(input.policyRuntime === undefined ? {} : { runtime: input.policyRuntime }), + passId: "history.fp-signature", + finding: f, + opportunity, + matched, + reasonCode: "active-fp-signature", + action: "suppressed", + sourceSignatures: sourceSignatures(f), + proposed: () => ({ + ...f, + severity: "INFO" as const, fp_ledger_match: { - pattern_id: hit.id, + pattern_id: (hit as { id: string }).id, matched_count: matchingSignatures.length, suppressed: true, }, - }; - } - return transitioned; - }) - : foreignScoped; + }), + }) ?? f; + // INFO-only matches keep their legacy attribution without entering the + // blocking opportunity denominator. + if (!fpSignatureAblated && f.severity === "INFO" && hit !== undefined) { + return { + ...transitioned, + fp_ledger_match: { + pattern_id: hit.id, + matched_count: matchingSignatures.length, + suppressed: true, + }, + }; + } + return transitioned; + }) + : foreignScoped; // Per-cycle suppression: a finding whose representative OR any member signature // the agent already rejected (reviewer_was_wrong) earlier this cycle is demoted @@ -1010,41 +1013,43 @@ export function aggregate(input: AggregateInput): AggregateResult { // dispositions a finding once and never sees it as blocking again this cycle. const cycleRejected = input.cycleRejected; const cycleRejectedEnabled = cycleRejected !== undefined && cycleRejected.size > 0; - const cycleScoped: Finding[] = - cycleRejectedEnabled || input.policyRuntime - ? fpScoped.map((f) => { - const sigs = [f.signature, ...(f.members?.map((m) => m.signature) ?? [])]; - const opportunity = cycleRejectedEnabled && f.severity !== "INFO"; - const matched = opportunity && sigs.some((s) => cycleRejected?.has(s)); - // G0b ceiling (codex DoD 2026-06-21): NEVER auto-hide a CRITICAL or any - // security/correctness finding via cycleRejected. One false reviewer_was_wrong - // rejection must not silence a later REAL CRITICAL of the same signature this cycle - // (a fail-open); it re-surfaces for an explicit per-iteration decision instead. - let protectedBy: PolicyProtectionCode | undefined; - if (matched && f.severity === "CRITICAL") protectedBy = "critical-floor"; - else if (matched && touchesSecurityOrCorrectness(f)) { - protectedBy = "security-correctness-floor"; - } - return ( - transitionFinding({ - ...(input.policyRuntime === undefined ? {} : { runtime: input.policyRuntime }), - passId: "history.cycle-rejected", - finding: f, - opportunity, - matched, - reasonCode: "cycle-signature-rejected", - action: "suppressed", - ...(protectedBy === undefined ? {} : { protectedBy }), - sourceSignatures: sourceSignatures(f), - proposed: () => ({ - ...f, - severity: "INFO" as const, - details: `${f.details.slice(0, 1900)}\n\n↓ already rejected earlier this cycle — advisory only.`, - }), - }) ?? f - ); - }) - : fpScoped; + if (!cycleRejectedEnabled) { + input.policyRuntime?.markInactive("history.cycle-rejected", "stage-precondition-miss"); + } + const cycleScoped: Finding[] = cycleRejectedEnabled + ? fpScoped.map((f) => { + const sigs = [f.signature, ...(f.members?.map((m) => m.signature) ?? [])]; + const opportunity = cycleRejectedEnabled && f.severity !== "INFO"; + const matched = opportunity && sigs.some((s) => cycleRejected?.has(s)); + // G0b ceiling (codex DoD 2026-06-21): NEVER auto-hide a CRITICAL or any + // security/correctness finding via cycleRejected. One false reviewer_was_wrong + // rejection must not silence a later REAL CRITICAL of the same signature this cycle + // (a fail-open); it re-surfaces for an explicit per-iteration decision instead. + let protectedBy: PolicyProtectionCode | undefined; + if (matched && f.severity === "CRITICAL") protectedBy = "critical-floor"; + else if (matched && touchesSecurityOrCorrectness(f)) { + protectedBy = "security-correctness-floor"; + } + return ( + transitionFinding({ + ...(input.policyRuntime === undefined ? {} : { runtime: input.policyRuntime }), + passId: "history.cycle-rejected", + finding: f, + opportunity, + matched, + reasonCode: "cycle-signature-rejected", + action: "suppressed", + ...(protectedBy === undefined ? {} : { protectedBy }), + sourceSignatures: sourceSignatures(f), + proposed: () => ({ + ...f, + severity: "INFO" as const, + details: `${f.details.slice(0, 1900)}\n\n↓ already rejected earlier this cycle — advisory only.`, + }), + }) ?? f + ); + }) + : fpScoped; // F3 Phase 2 — DERIVED FP-cluster demote. Applies AFTER the signature-keyed // pass so a finding already tagged via fp_ledger_match keeps both tags @@ -1054,52 +1059,55 @@ export function aggregate(input: AggregateInput): AggregateResult { // re-applies the identical tag + INFO severity. No explicit short-circuit. const fpClusters = input.fpActiveClusters; const fpClusterEnabled = fpClusters !== undefined && fpClusters.size > 0; - const fpClusterScoped: Finding[] = - fpClusterEnabled || input.policyRuntime - ? cycleScoped.map((f) => { - // Check the representative AND every merged member rule_id (clustering is - // category/rule-id-independent, so a known-FP rule can ride as a member - // under a different representative). Same file for all cluster members. - const ruleIds = [f.rule_id, ...(f.members?.map((m) => m.rule_id) ?? [])]; - const keys = [...new Set(ruleIds.map((rid) => `${ruleIdToken0(rid)}@${f.file}`))]; - const matchKey = keys.find((key) => fpClusters?.has(key)); - const hit = matchKey ? fpClusters?.get(matchKey) : undefined; - const opportunity = fpClusterEnabled && f.severity !== "INFO"; - const matched = opportunity && hit !== undefined; - const transitioned = - transitionFinding({ - ...(input.policyRuntime === undefined ? {} : { runtime: input.policyRuntime }), - passId: "history.fp-cluster", - finding: f, - opportunity, - matched, - reasonCode: "active-fp-cluster", - action: "suppressed", - sourceSignatures: sourceSignatures(f), - proposed: () => ({ - ...f, - severity: "INFO" as const, - fp_cluster_match: { - cluster_key: (hit as { key: string }).key, - member_ids: (hit as { member_ids: string[] }).member_ids, - suppressed: true, - }, - }), - }) ?? f; - // INFO-only matches retain the legacy explanatory marker. - if (f.severity === "INFO" && hit !== undefined) { - return { - ...transitioned, + if (!fpClusterEnabled) { + input.policyRuntime?.markInactive("history.fp-cluster", "stage-precondition-miss"); + } + const fpClusterAblated = input.policyRuntime?.isAblated("history.fp-cluster") ?? false; + const fpClusterScoped: Finding[] = fpClusterEnabled + ? cycleScoped.map((f) => { + // Check the representative AND every merged member rule_id (clustering is + // category/rule-id-independent, so a known-FP rule can ride as a member + // under a different representative). Same file for all cluster members. + const ruleIds = [f.rule_id, ...(f.members?.map((m) => m.rule_id) ?? [])]; + const keys = [...new Set(ruleIds.map((rid) => `${ruleIdToken0(rid)}@${f.file}`))]; + const matchKey = keys.find((key) => fpClusters?.has(key)); + const hit = matchKey ? fpClusters?.get(matchKey) : undefined; + const opportunity = fpClusterEnabled && f.severity !== "INFO"; + const matched = opportunity && hit !== undefined; + const transitioned = + transitionFinding({ + ...(input.policyRuntime === undefined ? {} : { runtime: input.policyRuntime }), + passId: "history.fp-cluster", + finding: f, + opportunity, + matched, + reasonCode: "active-fp-cluster", + action: "suppressed", + sourceSignatures: sourceSignatures(f), + proposed: () => ({ + ...f, + severity: "INFO" as const, fp_cluster_match: { - cluster_key: hit.key, - member_ids: hit.member_ids, + cluster_key: (hit as { key: string }).key, + member_ids: (hit as { member_ids: string[] }).member_ids, suppressed: true, }, - }; - } - return transitioned; - }) - : cycleScoped; + }), + }) ?? f; + // INFO-only matches retain the legacy explanatory marker. + if (!fpClusterAblated && f.severity === "INFO" && hit !== undefined) { + return { + ...transitioned, + fp_cluster_match: { + cluster_key: hit.key, + member_ids: hit.member_ids, + suppressed: true, + }, + }; + } + return transitioned; + }) + : cycleScoped; // Phase 4 #7 — confidence demote: an uncorroborated finding below the floor is // advisory only. Exempt: corroborated findings (majority/unanimous — multiple @@ -1109,107 +1117,112 @@ export function aggregate(input: AggregateInput): AggregateResult { // security/correctness concern can ride as a member under, e.g., a quality // representative — demoting the cluster would hide it and could flip FAIL→PASS). const floor = input.confidenceFloor ?? 0; - const confScoped: Finding[] = - floor > 0 || input.policyRuntime - ? fpClusterScoped.map((f) => { - // Cluster confidence = MAX over the representative and all merged members, - // so a co-located high-confidence member is never masked by a - // low-confidence representative. (memberOf records each member's - // confidence; older/persisted members may omit it → ignored in the max.) - const memberConfs = (f.members ?? []) - .map((m) => m.confidence) - .filter((c): c is number => typeof c === "number"); - const maxConfidence = Math.max(f.confidence, ...memberConfs); - const corroborated = f.consensus === "unanimous" || f.consensus === "majority"; - // A from-CRITICAL WARN is already at the G0 floor. The current catalog - // deliberately treats that marker-only repeat as ineligible rather than - // inventing a second WARN→WARN material transition. - const atCriticalFloor = f.severity === "WARN" && f.demoted_from_critical === true; - const opportunity = - floor > 0 && f.severity !== "INFO" && !corroborated && !atCriticalFloor; - const matched = opportunity && maxConfidence < floor; - let protectedBy: PolicyProtectionCode | undefined; - if (matched && pinned.has(f.signature)) protectedBy = "claimed-fixed-pin"; - else if (matched && f.severity === "CRITICAL" && touchesSecurityOrCorrectness(f)) { - protectedBy = "security-correctness-floor"; - } else if (matched && isProtected(f)) { - protectedBy = "high-precision-reviewer"; - } + const confidenceEnabled = floor > 0; + if (!confidenceEnabled) { + input.policyRuntime?.markInactive("judgment.confidence", "configured-off"); + } + const confidenceAblated = input.policyRuntime?.isAblated("judgment.confidence") ?? false; + const confScoped: Finding[] = confidenceEnabled + ? fpClusterScoped.map((f) => { + // Cluster confidence = MAX over the representative and all merged members, + // so a co-located high-confidence member is never masked by a + // low-confidence representative. (memberOf records each member's + // confidence; older/persisted members may omit it → ignored in the max.) + const memberConfs = (f.members ?? []) + .map((m) => m.confidence) + .filter((c): c is number => typeof c === "number"); + const maxConfidence = Math.max(f.confidence, ...memberConfs); + const corroborated = f.consensus === "unanimous" || f.consensus === "majority"; + // A from-CRITICAL WARN is already at the G0 floor. The current catalog + // deliberately treats that marker-only repeat as ineligible rather than + // inventing a second WARN→WARN material transition. + const atCriticalFloor = f.severity === "WARN" && f.demoted_from_critical === true; + const opportunity = floor > 0 && f.severity !== "INFO" && !corroborated && !atCriticalFloor; + const matched = opportunity && maxConfidence < floor; + let protectedBy: PolicyProtectionCode | undefined; + if (matched && pinned.has(f.signature)) protectedBy = "claimed-fixed-pin"; + else if (matched && f.severity === "CRITICAL" && touchesSecurityOrCorrectness(f)) { + protectedBy = "security-correctness-floor"; + } else if (matched && isProtected(f)) { + protectedBy = "high-precision-reviewer"; + } - const action = f.severity === "CRITICAL" ? "capped" : "demoted"; - const transitioned = - transitionFinding({ - ...(input.policyRuntime === undefined ? {} : { runtime: input.policyRuntime }), - passId: "judgment.confidence", - finding: f, - opportunity, - matched, - reasonCode: "below-confidence-floor", - action, - ...(protectedBy === undefined ? {} : { protectedBy }), - sourceSignatures: sourceSignatures(f), - proposed: () => { - if (f.severity === "CRITICAL") { - const note = `\n\n↓ low reviewer confidence (${maxConfidence.toFixed(2)} < ${floor}) — demoted CRITICAL→WARN; kept blocking pending your decision.`; - return { - ...f, - severity: "WARN" as const, - low_confidence: true, - demoted_from_critical: true, - details: `${f.details.slice(0, 2000 - note.length)}${note}`, - }; - } - const note = `\n\n↓ low reviewer confidence (${maxConfidence.toFixed(2)} < ${floor}) — advisory only.`; + const action = f.severity === "CRITICAL" ? "capped" : "demoted"; + const transitioned = + transitionFinding({ + ...(input.policyRuntime === undefined ? {} : { runtime: input.policyRuntime }), + passId: "judgment.confidence", + finding: f, + opportunity, + matched, + reasonCode: "below-confidence-floor", + action, + ...(protectedBy === undefined ? {} : { protectedBy }), + sourceSignatures: sourceSignatures(f), + proposed: () => { + if (f.severity === "CRITICAL") { + const note = `\n\n↓ low reviewer confidence (${maxConfidence.toFixed(2)} < ${floor}) — demoted CRITICAL→WARN; kept blocking pending your decision.`; return { ...f, - severity: "INFO" as const, + severity: "WARN" as const, low_confidence: true, + demoted_from_critical: true, details: `${f.details.slice(0, 2000 - note.length)}${note}`, }; - }, - }) ?? f; + } + const note = `\n\n↓ low reviewer confidence (${maxConfidence.toFixed(2)} < ${floor}) — advisory only.`; + return { + ...f, + severity: "INFO" as const, + low_confidence: true, + details: `${f.details.slice(0, 2000 - note.length)}${note}`, + }; + }, + }) ?? f; - // #4: a high-precision reviewer's blocking finding is not demoted for low - // self-reported confidence — its track record outweighs one low confidence call. - if (matched && protectedBy === "high-precision-reviewer") { + // #4: a high-precision reviewer's blocking finding is not demoted for low + // self-reported confidence — its track record outweighs one low confidence call. + if (!confidenceAblated && matched && protectedBy === "high-precision-reviewer") { + return { ...transitioned, protected_high_precision: true }; + } + if ( + !confidenceAblated && + f.severity === "INFO" && + floor > 0 && + maxConfidence < floor && + !corroborated && + !pinned.has(f.signature) + ) { + return { ...transitioned, low_confidence: true }; + } + // G0: this pass sends a non-security/correctness low-confidence CRITICAL DIRECTLY to + // INFO (not via DEMOTE) — that would flip a sole demoted-from-CRITICAL finding to a + // non-blocking INFO/PASS and auto-hide a possibly-real CRITICAL. CLAMP a from-CRITICAL + // at WARN (kept blocking + decision-required) and stamp provenance; a genuine + // never-CRITICAL WARN still demotes to INFO as before. + if ( + !confidenceAblated && + atCriticalFloor && + floor > 0 && + maxConfidence < floor && + !corroborated && + !pinned.has(f.signature) + ) { + if (isProtected(f)) { return { ...transitioned, protected_high_precision: true }; } - if ( - f.severity === "INFO" && - floor > 0 && - maxConfidence < floor && - !corroborated && - !pinned.has(f.signature) - ) { - return { ...transitioned, low_confidence: true }; - } - // G0: this pass sends a non-security/correctness low-confidence CRITICAL DIRECTLY to - // INFO (not via DEMOTE) — that would flip a sole demoted-from-CRITICAL finding to a - // non-blocking INFO/PASS and auto-hide a possibly-real CRITICAL. CLAMP a from-CRITICAL - // at WARN (kept blocking + decision-required) and stamp provenance; a genuine - // never-CRITICAL WARN still demotes to INFO as before. - if ( - atCriticalFloor && - floor > 0 && - maxConfidence < floor && - !corroborated && - !pinned.has(f.signature) - ) { - if (isProtected(f)) { - return { ...transitioned, protected_high_precision: true }; - } - const note = `\n\n↓ low reviewer confidence (${maxConfidence.toFixed(2)} < ${floor}) — demoted CRITICAL→WARN; kept blocking pending your decision.`; - return { - ...transitioned, - severity: "WARN" as const, - low_confidence: true, - demoted_from_critical: true, - details: `${transitioned.details.slice(0, 2000 - note.length)}${note}`, - }; - } - return transitioned; - }) - : fpClusterScoped; + const note = `\n\n↓ low reviewer confidence (${maxConfidence.toFixed(2)} < ${floor}) — demoted CRITICAL→WARN; kept blocking pending your decision.`; + return { + ...transitioned, + severity: "WARN" as const, + low_confidence: true, + demoted_from_critical: true, + details: `${transitioned.details.slice(0, 2000 - note.length)}${note}`, + }; + } + return transitioned; + }) + : fpClusterScoped; // Reviewer-reputation demote (Slice B: provider:persona keys): an un-corroborated finding whose every // contributing reviewer key is currently unreliable is demoted one step. Mirrors the @@ -1218,109 +1231,113 @@ export function aggregate(input: AggregateInput): AggregateResult { // INFO is untouched. const repUnreliable = input.repUnreliable; const reputationEnabled = repUnreliable !== undefined && repUnreliable.size > 0; - const repScoped: Finding[] = - reputationEnabled || input.policyRuntime - ? confScoped.map((f) => { - const corroborated = f.consensus === "unanimous" || f.consensus === "majority"; - const isCorrectness = touchesCorrectness(f); - const keys = - f.confirmed_by && f.confirmed_by.length > 0 - ? f.confirmed_by - : [`${f.reviewer.provider}:${f.reviewer.persona}`]; - const opportunity = reputationEnabled && f.severity !== "INFO" && !corroborated; - const matched = opportunity && keys.every((key) => repUnreliable?.has(key)); - const canClampCorrectnessCritical = - input.demoteCorrectness === true && - input.corroborateCritical === true && - input.reviewersTotal > 1; - let protectedBy: PolicyProtectionCode | undefined; - if (matched && pinned.has(f.signature)) protectedBy = "claimed-fixed-pin"; - else if (matched && touchesSecurity(f)) protectedBy = "security-floor"; - else if (matched && isCorrectness && input.demoteCorrectness !== true) { - protectedBy = "correctness-demote-disabled"; - } else if ( - matched && - ((f.severity === "WARN" && f.demoted_from_critical === true) || - (isCorrectness && f.severity === "CRITICAL" && !canClampCorrectnessCritical)) - ) { - protectedBy = "critical-floor"; - } + if (!reputationEnabled) { + input.policyRuntime?.markInactive("judgment.reputation", "stage-precondition-miss"); + } + const reputationAblated = input.policyRuntime?.isAblated("judgment.reputation") ?? false; + const repScoped: Finding[] = reputationEnabled + ? confScoped.map((f) => { + const corroborated = f.consensus === "unanimous" || f.consensus === "majority"; + const isCorrectness = touchesCorrectness(f); + const keys = + f.confirmed_by && f.confirmed_by.length > 0 + ? f.confirmed_by + : [`${f.reviewer.provider}:${f.reviewer.persona}`]; + const opportunity = reputationEnabled && f.severity !== "INFO" && !corroborated; + const matched = opportunity && keys.every((key) => repUnreliable?.has(key)); + const canClampCorrectnessCritical = + input.demoteCorrectness === true && + input.corroborateCritical === true && + input.reviewersTotal > 1; + let protectedBy: PolicyProtectionCode | undefined; + if (matched && pinned.has(f.signature)) protectedBy = "claimed-fixed-pin"; + else if (matched && touchesSecurity(f)) protectedBy = "security-floor"; + else if (matched && isCorrectness && input.demoteCorrectness !== true) { + protectedBy = "correctness-demote-disabled"; + } else if ( + matched && + ((f.severity === "WARN" && f.demoted_from_critical === true) || + (isCorrectness && f.severity === "CRITICAL" && !canClampCorrectnessCritical)) + ) { + protectedBy = "critical-floor"; + } - const action = isCorrectness && f.severity === "CRITICAL" ? "capped" : "demoted"; - const transitioned = - transitionFinding({ - ...(input.policyRuntime === undefined ? {} : { runtime: input.policyRuntime }), - passId: "judgment.reputation", - finding: f, - opportunity, - matched, - reasonCode: "unreliable-reviewer", - action, - ...(protectedBy === undefined ? {} : { protectedBy }), - sourceSignatures: sourceSignatures(f), - proposed: () => { - if (isCorrectness) { - // R5: an eligible CRITICAL correctness claim is clamped to a - // decision-required WARN pending corroboration. - if (f.severity === "CRITICAL") { - const note = - "\n\n↓ low reviewer reputation — uncorroborated CRITICAL correctness from a chronically-unreliable reviewer; demoted CRITICAL→WARN pending corroboration. Kept blocking + decision-required: verify the claim in the cited code, then fix it or reject it with evidence."; - return { - ...f, - severity: "WARN" as const, - reputation_demoted: true, - demoted_from_critical: true, - reputation_corroboration_required: true, - details: `${f.details.slice(0, 2000 - note.length)}${note}`, - }; - } + const action = isCorrectness && f.severity === "CRITICAL" ? "capped" : "demoted"; + const transitioned = + transitionFinding({ + ...(input.policyRuntime === undefined ? {} : { runtime: input.policyRuntime }), + passId: "judgment.reputation", + finding: f, + opportunity, + matched, + reasonCode: "unreliable-reviewer", + action, + ...(protectedBy === undefined ? {} : { protectedBy }), + sourceSignatures: sourceSignatures(f), + proposed: () => { + if (isCorrectness) { + // R5: an eligible CRITICAL correctness claim is clamped to a + // decision-required WARN pending corroboration. + if (f.severity === "CRITICAL") { const note = - "\n\n↓ low reviewer reputation — correctness finding from an unreliable lone reviewer; advisory only."; + "\n\n↓ low reviewer reputation — uncorroborated CRITICAL correctness from a chronically-unreliable reviewer; demoted CRITICAL→WARN pending corroboration. Kept blocking + decision-required: verify the claim in the cited code, then fix it or reject it with evidence."; return { ...f, - severity: "INFO" as const, + severity: "WARN" as const, reputation_demoted: true, + demoted_from_critical: true, + reputation_corroboration_required: true, details: `${f.details.slice(0, 2000 - note.length)}${note}`, }; } - - // Pure quality/style: existing one-step demote (CRITICAL→WARN, - // WARN→INFO), with the G0 floor handled above as a protection. - const demoted = demoteOneStep(f); - if (demoted.severity === "drop") return f; - const note = demoted.demoted_from_critical - ? "\n\n↓ low reviewer reputation — demoted CRITICAL→WARN; kept blocking pending your decision." - : "\n\n↓ low reviewer reputation — advisory only."; + const note = + "\n\n↓ low reviewer reputation — correctness finding from an unreliable lone reviewer; advisory only."; return { ...f, - severity: demoted.severity, + severity: "INFO" as const, reputation_demoted: true, - ...(demoted.demoted_from_critical ? { demoted_from_critical: true } : {}), details: `${f.details.slice(0, 2000 - note.length)}${note}`, }; - }, - }) ?? f; - // Preserve the pre-trace marker/details behavior for a pure-quality - // from-CRITICAL WARN already held at the G0 floor. The trace records - // the attempted second demotion as protected. - if ( - matched && - protectedBy === "critical-floor" && - !isCorrectness && - f.severity === "WARN" - ) { - const note = - "\n\n↓ low reviewer reputation — demoted CRITICAL→WARN; kept blocking pending your decision."; - return { - ...transitioned, - reputation_demoted: true, - demoted_from_critical: true, - details: `${transitioned.details.slice(0, 2000 - note.length)}${note}`, - }; - } - return transitioned; - }) - : confScoped; + } + + // Pure quality/style: existing one-step demote (CRITICAL→WARN, + // WARN→INFO), with the G0 floor handled above as a protection. + const demoted = demoteOneStep(f); + if (demoted.severity === "drop") return f; + const note = demoted.demoted_from_critical + ? "\n\n↓ low reviewer reputation — demoted CRITICAL→WARN; kept blocking pending your decision." + : "\n\n↓ low reviewer reputation — advisory only."; + return { + ...f, + severity: demoted.severity, + reputation_demoted: true, + ...(demoted.demoted_from_critical ? { demoted_from_critical: true } : {}), + details: `${f.details.slice(0, 2000 - note.length)}${note}`, + }; + }, + }) ?? f; + // Preserve the pre-trace marker/details behavior for a pure-quality + // from-CRITICAL WARN already held at the G0 floor. The trace records + // the attempted second demotion as protected. + if ( + !reputationAblated && + matched && + protectedBy === "critical-floor" && + !isCorrectness && + f.severity === "WARN" + ) { + const note = + "\n\n↓ low reviewer reputation — demoted CRITICAL→WARN; kept blocking pending your decision."; + return { + ...transitioned, + reputation_demoted: true, + demoted_from_critical: true, + details: `${transitioned.details.slice(0, 2000 - note.length)}${note}`, + }; + } + return transitioned; + }) + : confScoped; // T3/R4 (field report 2026-07-03) — region-rejection pass. Placed AFTER the // reputation clamp so it evaluates POST-clamp severity (a clamped from-CRITICAL @@ -1341,91 +1358,98 @@ export function aggregate(input: AggregateInput): AggregateResult { // Fail-safe: findings without line data and unparseable regions are untouched. const rejectedRegions = input.rejectedRegions; const regionRejectedEnabled = rejectedRegions !== undefined && rejectedRegions.length > 0; + if (!regionRejectedEnabled) { + input.policyRuntime?.markInactive("history.region-rejected", "stage-precondition-miss"); + } + const regionRejectedAblated = input.policyRuntime?.isAblated("history.region-rejected") ?? false; let regionSuppressedCount = 0; - const regionScoped: Finding[] = - regionRejectedEnabled || input.policyRuntime - ? repScoped.map((f) => { - const opportunity = - regionRejectedEnabled && f.severity !== "INFO" && Boolean(f.line_start); - const file = normalizeRepoPath(f.file); - const lineEnd = typeof f.line_end === "number" ? f.line_end : f.line_start; - const match = rejectedRegions?.find( - (r) => - typeof r.start_line === "number" && - typeof r.end_line === "number" && - normalizeRepoPath(r.file) === file && - f.line_start <= r.end_line + REGION_WINDOW && - lineEnd >= r.start_line - REGION_WINDOW, - ); - const matched = opportunity && match !== undefined; - const memberCats = [f.category, ...(f.members?.map((m) => m.category) ?? [])]; - const categoryCompatible = - match !== undefined && - memberCats.every((category) => match.categories.includes(category)); - const severityDominated = - match !== undefined && SEVERITY_RANK[f.severity] <= SEVERITY_RANK[match.severity]; - let protectedBy: PolicyProtectionCode | undefined; - if (matched && f.claimed_fixed_recurred) protectedBy = "claimed-fixed-pin"; - else if (matched && (match?.distinct_count ?? 0) < 2) { - protectedBy = "insufficient-distinct-rejections"; - } else if (matched && !categoryCompatible) protectedBy = "category-change"; - else if (matched && !severityDominated) protectedBy = "severity-increase"; - else if (matched && (f.severity === "CRITICAL" || f.demoted_from_critical === true)) { - protectedBy = "critical-floor"; - } else if (matched && touchesSecurity(f)) { - protectedBy = "security-correctness-floor"; - } - - const transitioned = - transitionFinding({ - ...(input.policyRuntime === undefined ? {} : { runtime: input.policyRuntime }), - passId: "history.region-rejected", - finding: f, - opportunity, - matched, - reasonCode: "rejected-region-overlap", - action: "suppressed", - ...(protectedBy === undefined ? {} : { protectedBy }), - sourceSignatures: sourceSignatures(f), - proposed: () => { - const hit = match as NonNullable; - const tag = { - distinct_count: hit.distinct_count, - prior_reason: hit.reason.slice(0, 200), - suppressed: true, - }; - const note = `\n\n↓ overlaps a region you already rejected ${hit.distinct_count}× this cycle ("${hit.reason.slice(0, 120)}") — advisory only.`; - return { - ...f, - severity: "INFO" as const, - region_rejected_match: tag, - details: `${f.details.slice(0, 2000 - note.length)}${note}`, - }; - }, - }) ?? f; + const regionScoped: Finding[] = regionRejectedEnabled + ? repScoped.map((f) => { + const opportunity = regionRejectedEnabled && f.severity !== "INFO" && Boolean(f.line_start); + const file = normalizeRepoPath(f.file); + const lineEnd = typeof f.line_end === "number" ? f.line_end : f.line_start; + const match = rejectedRegions?.find( + (r) => + typeof r.start_line === "number" && + typeof r.end_line === "number" && + normalizeRepoPath(r.file) === file && + f.line_start <= r.end_line + REGION_WINDOW && + lineEnd >= r.start_line - REGION_WINDOW, + ); + const matched = opportunity && match !== undefined; + const memberCats = [f.category, ...(f.members?.map((m) => m.category) ?? [])]; + const categoryCompatible = + match !== undefined && + memberCats.every((category) => match.categories.includes(category)); + const severityDominated = + match !== undefined && SEVERITY_RANK[f.severity] <= SEVERITY_RANK[match.severity]; + let protectedBy: PolicyProtectionCode | undefined; + if (matched && f.claimed_fixed_recurred) protectedBy = "claimed-fixed-pin"; + else if (matched && (match?.distinct_count ?? 0) < 2) { + protectedBy = "insufficient-distinct-rejections"; + } else if (matched && !categoryCompatible) protectedBy = "category-change"; + else if (matched && !severityDominated) protectedBy = "severity-increase"; + else if (matched && (f.severity === "CRITICAL" || f.demoted_from_critical === true)) { + protectedBy = "critical-floor"; + } else if (matched && touchesSecurity(f)) { + protectedBy = "security-correctness-floor"; + } - if (matched && protectedBy !== undefined && protectedBy !== "claimed-fixed-pin") { - const hit = match as NonNullable; - return { - ...transitioned, - region_rejected_match: { + const transitioned = + transitionFinding({ + ...(input.policyRuntime === undefined ? {} : { runtime: input.policyRuntime }), + passId: "history.region-rejected", + finding: f, + opportunity, + matched, + reasonCode: "rejected-region-overlap", + action: "suppressed", + ...(protectedBy === undefined ? {} : { protectedBy }), + sourceSignatures: sourceSignatures(f), + proposed: () => { + const hit = match as NonNullable; + const tag = { distinct_count: hit.distinct_count, prior_reason: hit.reason.slice(0, 200), - suppressed: false, - }, - }; - } - if ( - matched && - protectedBy === undefined && - transitioned.severity === "INFO" && - transitioned.region_rejected_match?.suppressed === true - ) { - regionSuppressedCount++; - } - return transitioned; - }) - : repScoped; + suppressed: true, + }; + const note = `\n\n↓ overlaps a region you already rejected ${hit.distinct_count}× this cycle ("${hit.reason.slice(0, 120)}") — advisory only.`; + return { + ...f, + severity: "INFO" as const, + region_rejected_match: tag, + details: `${f.details.slice(0, 2000 - note.length)}${note}`, + }; + }, + }) ?? f; + + if ( + !regionRejectedAblated && + matched && + protectedBy !== undefined && + protectedBy !== "claimed-fixed-pin" + ) { + const hit = match as NonNullable; + return { + ...transitioned, + region_rejected_match: { + distinct_count: hit.distinct_count, + prior_reason: hit.reason.slice(0, 200), + suppressed: false, + }, + }; + } + if ( + matched && + protectedBy === undefined && + transitioned.severity === "INFO" && + transitioned.region_rejected_match?.suppressed === true + ) { + regionSuppressedCount++; + } + return transitioned; + }) + : repScoped; // Slice 2 (field report #9): demote a SECURITY finding on a test/fixture file to INFO // (advisory). Only category "security"; correctness/other test-file findings stay blocking @@ -1437,52 +1461,56 @@ export function aggregate(input: AggregateInput): AggregateResult { // "correctness stays blocking" rule — flagged by the dogfood gate iter 3). So we demote only // when EVERY clustered member is also security: a single non-security member keeps the whole // cluster blocking. (members[] includes the representative's own entry; absent → lone finding.) - const testScoped: Finding[] = - input.demoteTestSecurity === true || input.policyRuntime - ? regionScoped.map((f) => { - const testFile = classify(f.file) === "tests"; - const opportunity = - input.demoteTestSecurity === true && f.severity !== "INFO" && testFile; - const matched = opportunity && f.category === "security"; - const protectedBy = - matched && (f.members ?? []).some((member) => member.category !== "security") - ? "mixed-category-cluster" - : undefined; - const transitioned = - transitionFinding({ - ...(input.policyRuntime === undefined ? {} : { runtime: input.policyRuntime }), - passId: "judgment.test-security", - finding: f, - opportunity, - matched, - reasonCode: "test-only-security", - action: "demoted", - ...(protectedBy === undefined ? {} : { protectedBy }), - sourceSignatures: sourceSignatures(f), - proposed: () => { - const note = - "\n\n↓ security finding on a test/fixture file — not production code; advisory only."; - return { - ...f, - severity: "INFO" as const, - test_severity_demoted: true, - details: `${f.details.slice(0, 2000 - note.length)}${note}`, - }; - }, - }) ?? f; - // Preserve the legacy marker for already-advisory security test findings. - if ( - input.demoteTestSecurity === true && - f.severity === "INFO" && - testFile && - f.category === "security" && - !(f.members ?? []).some((member) => member.category !== "security") - ) { - return { ...transitioned, test_severity_demoted: true }; - } - return transitioned; - }) - : regionScoped; + const testSecurityEnabled = input.demoteTestSecurity === true; + if (!testSecurityEnabled) { + input.policyRuntime?.markInactive("judgment.test-security", "configured-off"); + } + const testSecurityAblated = input.policyRuntime?.isAblated("judgment.test-security") ?? false; + const testScoped: Finding[] = testSecurityEnabled + ? regionScoped.map((f) => { + const testFile = classify(f.file) === "tests"; + const opportunity = input.demoteTestSecurity === true && f.severity !== "INFO" && testFile; + const matched = opportunity && f.category === "security"; + const protectedBy = + matched && (f.members ?? []).some((member) => member.category !== "security") + ? "mixed-category-cluster" + : undefined; + const transitioned = + transitionFinding({ + ...(input.policyRuntime === undefined ? {} : { runtime: input.policyRuntime }), + passId: "judgment.test-security", + finding: f, + opportunity, + matched, + reasonCode: "test-only-security", + action: "demoted", + ...(protectedBy === undefined ? {} : { protectedBy }), + sourceSignatures: sourceSignatures(f), + proposed: () => { + const note = + "\n\n↓ security finding on a test/fixture file — not production code; advisory only."; + return { + ...f, + severity: "INFO" as const, + test_severity_demoted: true, + details: `${f.details.slice(0, 2000 - note.length)}${note}`, + }; + }, + }) ?? f; + // Preserve the legacy marker for already-advisory security test findings. + if ( + !testSecurityAblated && + input.demoteTestSecurity === true && + f.severity === "INFO" && + testFile && + f.category === "security" && + !(f.members ?? []).some((member) => member.category !== "security") + ) { + return { ...transitioned, test_severity_demoted: true }; + } + return transitioned; + }) + : regionScoped; // Slice D (P5) — docs severity cap. A CRITICAL whose FILE classifies as "docs" is // over-severity (a stale doc is not a security/data-loss bug). Cap to WARN via demoteOneStep @@ -1492,41 +1520,44 @@ export function aggregate(input: AggregateInput): AggregateResult { // tests BEFORE docs, so a *.md fixture under tests/ is "tests", not "docs", and is untouched. // Fires BEFORE the verdict loop so a capped docs finding no longer trips the singleton // reviewersTotal<=1 hard-FAIL; the sec/corr exemption preserves that path for dangerous docs. - const docsScoped: Finding[] = - input.capDocsSeverity === true || input.policyRuntime - ? testScoped.map((f) => { - const opportunity = input.capDocsSeverity === true && f.severity === "CRITICAL"; - const matched = opportunity && classify(f.file) === "docs"; - const protectedBy = - matched && touchesSecurityOrCorrectness(f) ? "security-correctness-floor" : undefined; - return ( - transitionFinding({ - ...(input.policyRuntime === undefined ? {} : { runtime: input.policyRuntime }), - passId: "judgment.docs-cap", - finding: f, - opportunity, - matched, - reasonCode: "docs-critical-cap", - action: "capped", - ...(protectedBy === undefined ? {} : { protectedBy }), - sourceSignatures: sourceSignatures(f), - proposed: () => { - const demoted = demoteOneStep(f); // CRITICAL → WARN (+ demoted_from_critical) - if (demoted.severity !== "WARN") return f; - const note = - "\n\n↓ docs/markdown file — capped CRITICAL→WARN (a stale doc is not a security/data-loss bug); kept blocking pending your decision."; - return { - ...f, - severity: "WARN" as const, - docs_severity_capped: true, - ...(demoted.demoted_from_critical ? { demoted_from_critical: true } : {}), - details: `${f.details.slice(0, 2000 - note.length)}${note}`, - }; - }, - }) ?? f - ); - }) - : testScoped; + const docsCapEnabled = input.capDocsSeverity === true; + if (!docsCapEnabled) { + input.policyRuntime?.markInactive("judgment.docs-cap", "configured-off"); + } + const docsScoped: Finding[] = docsCapEnabled + ? testScoped.map((f) => { + const opportunity = input.capDocsSeverity === true && f.severity === "CRITICAL"; + const matched = opportunity && classify(f.file) === "docs"; + const protectedBy = + matched && touchesSecurityOrCorrectness(f) ? "security-correctness-floor" : undefined; + return ( + transitionFinding({ + ...(input.policyRuntime === undefined ? {} : { runtime: input.policyRuntime }), + passId: "judgment.docs-cap", + finding: f, + opportunity, + matched, + reasonCode: "docs-critical-cap", + action: "capped", + ...(protectedBy === undefined ? {} : { protectedBy }), + sourceSignatures: sourceSignatures(f), + proposed: () => { + const demoted = demoteOneStep(f); // CRITICAL → WARN (+ demoted_from_critical) + if (demoted.severity !== "WARN") return f; + const note = + "\n\n↓ docs/markdown file — capped CRITICAL→WARN (a stale doc is not a security/data-loss bug); kept blocking pending your decision."; + return { + ...f, + severity: "WARN" as const, + docs_severity_capped: true, + ...(demoted.demoted_from_critical ? { demoted_from_critical: true } : {}), + details: `${f.details.slice(0, 2000 - note.length)}${note}`, + }; + }, + }) ?? f + ); + }) + : testScoped; // Slice C (P4) — render-only honest framing for a lone uncorroborated CRITICAL. It STILL // hard-FAILs in the verdict loop below (PR#22 unchanged); the badge just tells the agent to diff --git a/src/core/policy/trace.ts b/src/core/policy/trace.ts index 7bea02e..b34ef06 100644 --- a/src/core/policy/trace.ts +++ b/src/core/policy/trace.ts @@ -23,6 +23,10 @@ import { } from "./catalog.ts"; type RanPolicyPassSummary = Extract; +type InactivePolicyReasonCode = Extract< + PolicyReasonCode, + "configured-off" | "stage-precondition-miss" +>; type TraceVerdict = PolicyTrace["final"]["verdict"]; const PASS_BY_ID = new Map(POLICY_PASSES.map((pass) => [pass.id, pass])); @@ -67,6 +71,8 @@ export interface FinalizePolicyTraceInput { export interface PolicyRuntime { readonly telemetryError: boolean; + isAblated(passId: PolicyPassId): boolean; + markInactive(passId: PolicyPassId, reasonCode: InactivePolicyReasonCode): void; transition(input: TransitionInput): Finding | null; summary(passId: PolicyPassId): PolicyPassSummary; evaluations(): PolicyEvaluation[]; @@ -163,7 +169,7 @@ export class PolicyTraceRecorder implements PolicyRuntime { readonly #runId: string; readonly #iter: number; readonly #ablated: ReadonlySet; - readonly #summaries = new Map(); + readonly #summaries = new Map(); readonly #evaluations: PolicyEvaluation[] = []; readonly #stages: PolicyStageEvaluation[] = []; readonly #finalBySource = new Map(); @@ -184,6 +190,37 @@ export class PolicyTraceRecorder implements PolicyRuntime { return this.#telemetryError; } + isAblated(passId: PolicyPassId): boolean { + return this.#ablated.has(passId); + } + + markInactive(passId: PolicyPassId, reasonCode: InactivePolicyReasonCode): void { + try { + const current = this.#summaries.get(passId); + if (current === undefined) throw new Error(`unknown policy pass: ${passId}`); + if (current.status === "not-run") { + if (current.reason_code !== reasonCode) { + throw new Error(`${passId} already has a different inactivity reason`); + } + return; + } + if (current.status !== "ran" || current.considered > 0) { + throw new Error(`${passId} cannot become inactive after evaluation`); + } + const inactive = PolicyPassSummarySchema.parse({ + pass_id: passId, + status: "not-run", + reason_code: reasonCode, + }); + if (inactive.status !== "not-run") { + throw new Error("inactive lifecycle unexpectedly produced a ran summary"); + } + this.#summaries.set(passId, inactive); + } catch { + this.#telemetryError = true; + } + } + transition(input: TransitionInput): Finding | null { // These are production inputs. Read them and calculate the production result // before entering the fail-open telemetry boundary so their errors propagate. @@ -221,6 +258,21 @@ export class PolicyTraceRecorder implements PolicyRuntime { } const sourceSignatures = this.#sourceSignatures(input, finding); + // An internal ablation removes this pass's mutation and every material + // marker/effect, including a protected effect. The positive predicate is + // still observable as would-apply and later passes receive the original. + if (ablated) { + this.#recordEvaluation({ + pass_id: input.passId, + result: "would-apply", + before: finding.severity, + after: finding.severity, + reason_code: input.reasonCode, + source_signatures: sourceSignatures, + }); + return finding; + } + if (protectedBy !== undefined) { const effect = PolicyEffectSchema.parse({ pass_id: input.passId, @@ -250,18 +302,6 @@ export class PolicyTraceRecorder implements PolicyRuntime { }; } - if (ablated) { - this.#recordEvaluation({ - pass_id: input.passId, - result: "would-apply", - before: finding.severity, - after: finding.severity, - reason_code: input.reasonCode, - source_signatures: sourceSignatures, - }); - return finding; - } - const after = productionResult?.severity ?? null; const effect = PolicyEffectSchema.parse({ pass_id: input.passId, @@ -419,6 +459,9 @@ export class PolicyTraceRecorder implements PolicyRuntime { }); const current = this.#summaries.get(evaluation.pass_id); if (current === undefined) throw new Error(`unknown policy pass: ${evaluation.pass_id}`); + if (current.status !== "ran") { + throw new Error(`inactive policy pass cannot record evaluations: ${evaluation.pass_id}`); + } const next: RanPolicyPassSummary = { ...current, considered: current.considered + 1 }; if (evaluation.result !== "no-opportunity") next.opportunities += 1; diff --git a/tests/unit/policy-aggregator-first-half.test.ts b/tests/unit/policy-aggregator-first-half.test.ts index 97e9dfb..a5f4db5 100644 --- a/tests/unit/policy-aggregator-first-half.test.ts +++ b/tests/unit/policy-aggregator-first-half.test.ts @@ -338,7 +338,7 @@ describe("aggregator policy numeric contracts, orders 60-100", () => { }); }); - it("protects a G0-clamped WARN from critic likely_fp in active and ablated runs", () => { + it("records active G0 protection but removes its material effect when critic is ablated", () => { const clamped = finding({ signature: "sig-critic-critical-floor", severity: "WARN", @@ -357,9 +357,7 @@ describe("aggregator policy numeric contracts, orders 60-100", () => { expect(numericSummary(active.recorder, "judgment.critic")).toEqual( PROTECTED_BLOCKING_PRESERVED, ); - expect(numericSummary(ablated.recorder, "judgment.critic")).toEqual( - PROTECTED_BLOCKING_PRESERVED, - ); + expect(numericSummary(ablated.recorder, "judgment.critic")).toEqual(ABLATED_BLOCKING_PRESERVED); expectSingleEffect(active.result.dedupedFindings[0], { pass_id: "judgment.critic", order: 70, @@ -369,15 +367,7 @@ describe("aggregator policy numeric contracts, orders 60-100", () => { reason_code: "critic-likely-fp", protected_by: "critical-floor", }); - expectSingleEffect(ablated.result.dedupedFindings[0], { - pass_id: "judgment.critic", - order: 70, - action: "protected", - before: "WARN", - after: "WARN", - reason_code: "critic-likely-fp", - protected_by: "critical-floor", - }); + expect(ablated.result.dedupedFindings[0]?.policy_effects).toBeUndefined(); for (const output of [active, ablated]) { expect(output.result.dedupedFindings[0]?.critic_verdict).toBeUndefined(); expect(output.result.dedupedFindings[0]?.critic_reason).toBeUndefined(); diff --git a/tests/unit/policy-aggregator-second-half.test.ts b/tests/unit/policy-aggregator-second-half.test.ts index ee58ad7..0696273 100644 --- a/tests/unit/policy-aggregator-second-half.test.ts +++ b/tests/unit/policy-aggregator-second-half.test.ts @@ -1,5 +1,6 @@ import { describe, expect, it } from "bun:test"; import { type AggregateInput, aggregate } from "../../src/core/aggregator.ts"; +import { deriveImplicitOutcomes } from "../../src/core/learnings/implicit-outcomes.ts"; import type { PolicyPassId, PolicyProtectionCode, @@ -98,6 +99,20 @@ function finalized(output: ReturnType): PolicyTrace { return trace; } +function withoutPolicyEffects(value: Finding | undefined): Finding | undefined { + if (value === undefined) return undefined; + const { policy_effects: _policyEffects, ...legacyFinding } = value; + return legacyFinding as Finding; +} + +function implicitOutcomes(value: Finding | undefined) { + return deriveImplicitOutcomes(value === undefined ? [] : [value], [], { + runId: "run-ablation-marker", + iter: 1, + nowIso: "2026-08-10T00:00:00Z", + }); +} + function activeCluster() { return new Map([["policy@src/a.ts", { key: "policy@src/a.ts", member_ids: ["FP-001"] }]]); } @@ -118,6 +133,43 @@ function rejectedRegion( } describe("aggregator policy numeric contracts, orders 110-180", () => { + it("marks every configured-inactive second-half pass not-run and finalizes without evaluations", () => { + const output = run("second-half-inactive", { + findings: [finding()], + reviewersTotal: 2, + }); + const expected = [ + ["history.fp-signature", "stage-precondition-miss"], + ["history.cycle-rejected", "stage-precondition-miss"], + ["history.fp-cluster", "stage-precondition-miss"], + ["judgment.confidence", "configured-off"], + ["judgment.reputation", "stage-precondition-miss"], + ["history.region-rejected", "stage-precondition-miss"], + ["judgment.test-security", "configured-off"], + ["judgment.docs-cap", "configured-off"], + ] as const; + + for (const [passId, reasonCode] of expected) { + expect(output.recorder.summary(passId)).toEqual({ + pass_id: passId, + status: "not-run", + reason_code: reasonCode, + }); + expect(output.recorder.evaluations().some((row) => row.pass_id === passId)).toBe(false); + } + + const trace = finalized(output); + expect( + trace.passes + .filter((pass) => expected.some(([passId]) => pass.pass_id === passId)) + .map((pass) => [ + pass.pass_id, + pass.status, + "reason_code" in pass ? pass.reason_code : null, + ]), + ).toEqual(expected.map(([passId, reasonCode]) => [passId, "not-run", reasonCode])); + }); + it("records FP-signature no-opportunity, miss, active, and ablated tuples", () => { const info = run("fp-signature-info", { findings: [finding({ severity: "INFO" })], @@ -745,6 +797,163 @@ describe("aggregator policy numeric contracts, orders 110-180", () => { }); }); +describe("second-half ablation marker isolation", () => { + it("preserves legacy FP-signature INFO attribution only when the pass is not ablated", () => { + const original = finding({ severity: "INFO", details: "original FP details" }); + const input = { + findings: [original], + reviewersTotal: 1, + fpActive: new Map([["sig-policy", { id: "FP-001" }]]), + }; + const legacy = aggregate(input); + const control = aggregate({ findings: [original], reviewersTotal: 1 }); + const active = run("fp-info-marker-active", input); + const ablated = run("fp-info-marker-ablated", input, ["history.fp-signature"]); + + expect(withoutPolicyEffects(active.result.dedupedFindings[0])).toEqual( + legacy.dedupedFindings[0], + ); + expect(active.result.dedupedFindings[0]?.fp_ledger_match).toEqual({ + pattern_id: "FP-001", + matched_count: 1, + suppressed: true, + }); + expect(ablated.result.dedupedFindings[0]).toEqual(control.dedupedFindings[0]); + expect(implicitOutcomes(ablated.result.dedupedFindings[0])).toEqual([]); + }); + + it("does not leak the FP-cluster INFO attribution branch through ablation", () => { + const original = finding({ severity: "INFO", details: "original cluster details" }); + const input = { + findings: [original], + reviewersTotal: 1, + fpActiveClusters: activeCluster(), + }; + const legacy = aggregate(input); + const control = aggregate({ findings: [original], reviewersTotal: 1 }); + const active = run("fp-cluster-info-marker-active", input); + const ablated = run("fp-cluster-info-marker-ablated", input, ["history.fp-cluster"]); + + expect(withoutPolicyEffects(active.result.dedupedFindings[0])).toEqual( + legacy.dedupedFindings[0], + ); + expect(active.result.dedupedFindings[0]?.fp_cluster_match?.suppressed).toBe(true); + expect(ablated.result.dedupedFindings[0]).toEqual(control.dedupedFindings[0]); + }); + + it("removes confidence G0 and high-precision markers, details, effects, and implicit outcomes", () => { + const g0 = finding({ + confidence: 0.1, + demoted_from_critical: true, + details: "original confidence details", + }); + const g0Input = { findings: [g0], reviewersTotal: 1, confidenceFloor: 0.5 }; + const g0Legacy = aggregate(g0Input); + const g0Control = aggregate({ findings: [g0], reviewersTotal: 1 }); + const g0Active = run("confidence-g0-marker-active", g0Input); + const g0Ablated = run("confidence-g0-marker-ablated", g0Input, ["judgment.confidence"]); + + expect(withoutPolicyEffects(g0Active.result.dedupedFindings[0])).toEqual( + g0Legacy.dedupedFindings[0], + ); + expect(g0Active.result.dedupedFindings[0]).toMatchObject({ + low_confidence: true, + demoted_from_critical: true, + }); + expect(g0Ablated.result.dedupedFindings[0]).toEqual(g0Control.dedupedFindings[0]); + expect(implicitOutcomes(g0Ablated.result.dedupedFindings[0])).toEqual([]); + + const protectedFinding = finding({ + confidence: 0.1, + details: "original protected details", + }); + const protectedInput = { + findings: [protectedFinding], + reviewersTotal: 1, + confidenceFloor: 0.5, + protectedReviewers: new Set(["codex"]), + }; + const protectedLegacy = aggregate(protectedInput); + const protectedControl = aggregate({ findings: [protectedFinding], reviewersTotal: 1 }); + const protectedActive = run("confidence-protected-marker-active", protectedInput); + const protectedAblated = run("confidence-protected-marker-ablated", protectedInput, [ + "judgment.confidence", + ]); + + expect(withoutPolicyEffects(protectedActive.result.dedupedFindings[0])).toEqual( + protectedLegacy.dedupedFindings[0], + ); + expect(protectedActive.result.dedupedFindings[0]?.protected_high_precision).toBe(true); + expect(protectedAblated.result.dedupedFindings[0]).toEqual(protectedControl.dedupedFindings[0]); + }); + + it("removes reputation G0 markers, details, effects, and implicit outcomes", () => { + const original = finding({ + demoted_from_critical: true, + details: "original reputation details", + }); + const input = { + findings: [original], + reviewersTotal: 1, + repUnreliable: new Set(["codex:quality"]), + }; + const legacy = aggregate(input); + const control = aggregate({ findings: [original], reviewersTotal: 1 }); + const active = run("reputation-g0-marker-active", input); + const ablated = run("reputation-g0-marker-ablated", input, ["judgment.reputation"]); + + expect(withoutPolicyEffects(active.result.dedupedFindings[0])).toEqual( + legacy.dedupedFindings[0], + ); + expect(active.result.dedupedFindings[0]?.reputation_demoted).toBe(true); + expect(implicitOutcomes(ablated.result.dedupedFindings[0])).toEqual([]); + expect(ablated.result.dedupedFindings[0]).toEqual(control.dedupedFindings[0]); + }); + + it("removes protected region badges and INFO test-security markers under ablation", () => { + const regionFinding = finding({ details: "original region details" }); + const regionInput = { + findings: [regionFinding], + reviewersTotal: 1, + rejectedRegions: [rejectedRegion({ distinct_count: 1 })], + }; + const regionLegacy = aggregate(regionInput); + const regionControl = aggregate({ findings: [regionFinding], reviewersTotal: 1 }); + const regionActive = run("region-marker-active", regionInput); + const regionAblated = run("region-marker-ablated", regionInput, ["history.region-rejected"]); + + expect(withoutPolicyEffects(regionActive.result.dedupedFindings[0])).toEqual( + regionLegacy.dedupedFindings[0], + ); + expect(regionActive.result.dedupedFindings[0]?.region_rejected_match?.suppressed).toBe(false); + expect(regionAblated.result.dedupedFindings[0]).toEqual(regionControl.dedupedFindings[0]); + + const testFinding = finding({ + severity: "INFO", + category: "security", + file: "tests/a.test.ts", + details: "original test details", + }); + const testInput = { + findings: [testFinding], + reviewersTotal: 1, + demoteTestSecurity: true, + }; + const testLegacy = aggregate(testInput); + const testControl = aggregate({ findings: [testFinding], reviewersTotal: 1 }); + const testActive = run("test-security-info-marker-active", testInput); + const testAblated = run("test-security-info-marker-ablated", testInput, [ + "judgment.test-security", + ]); + + expect(withoutPolicyEffects(testActive.result.dedupedFindings[0])).toEqual( + testLegacy.dedupedFindings[0], + ); + expect(testActive.result.dedupedFindings[0]?.test_severity_demoted).toBe(true); + expect(testAblated.result.dedupedFindings[0]).toEqual(testControl.dedupedFindings[0]); + }); +}); + describe("verdict.compute trace stage", () => { it("records exactly one closed verdict reason for every judgment branch", () => { const hardCritical = run("verdict-hard-critical", { diff --git a/tests/unit/policy-trace-recorder.test.ts b/tests/unit/policy-trace-recorder.test.ts index 8d704c9..016b527 100644 --- a/tests/unit/policy-trace-recorder.test.ts +++ b/tests/unit/policy-trace-recorder.test.ts @@ -447,6 +447,155 @@ describe("PolicyTraceRecorder fail-open telemetry", () => { }); }); +describe("PolicyTraceRecorder pass lifecycle", () => { + it("records an inactive pass without counters or evaluations and finalizes a valid mixed trace", () => { + const runtime = PolicyTraceRecorder.start({ runId: "run-inactive", iter: 1, ablated: [] }); + runtime.markInactive("judgment.confidence", "configured-off"); + runtime.recordStage({ + stageId: "aggregation.cluster", + reasonCode: "singleton", + memberCount: 1, + inputSignatures: [warnFinding.signature], + outputSignature: warnFinding.signature, + }); + runtime.linkFinal([warnFinding.signature], warnFinding.signature); + runtime.recordStage({ + stageId: "verdict.compute", + reasonCode: "blocking-present", + inputSignatures: [warnFinding.signature], + verdict: "SOFT-PASS", + }); + + const trace = runtime.finalize({ + rawResponseSha256: [], + verdict: "SOFT-PASS", + finalFindings: [warnFinding], + }); + + expect(runtime.telemetryError).toBe(false); + expect(runtime.summary("judgment.confidence")).toEqual({ + pass_id: "judgment.confidence", + status: "not-run", + reason_code: "configured-off", + }); + expect(runtime.evaluations()).toEqual([]); + expect(trace?.passes.find((pass) => pass.pass_id === "judgment.confidence")).toEqual({ + pass_id: "judgment.confidence", + status: "not-run", + reason_code: "configured-off", + }); + }); + + it("isolates ran-versus-inactive conflicts without changing production transition semantics", () => { + const evaluated = PolicyTraceRecorder.start({ + runId: "run-inactive-after-evaluation", + iter: 1, + ablated: [], + }); + transitionFinding({ + runtime: evaluated, + passId: "judgment.confidence", + finding: warnFinding, + opportunity: true, + matched: false, + reasonCode: "below-confidence-floor", + action: "demoted", + proposed: () => { + throw new Error("predicate miss must stay lazy"); + }, + }); + evaluated.markInactive("judgment.confidence", "configured-off"); + + expect(evaluated.telemetryError).toBe(true); + expect(evaluated.summary("judgment.confidence")).toMatchObject({ + status: "ran", + considered: 1, + }); + expect( + evaluated.finalize({ rawResponseSha256: [], verdict: "PASS", finalFindings: [] }), + ).toBeNull(); + + const inactive = PolicyTraceRecorder.start({ + runId: "run-evaluation-after-inactive", + iter: 1, + ablated: [], + }); + inactive.markInactive("judgment.confidence", "configured-off"); + const proposed = { ...warnFinding, severity: "INFO" as const, low_confidence: true }; + const after = transitionFinding({ + runtime: inactive, + passId: "judgment.confidence", + finding: warnFinding, + opportunity: true, + matched: true, + reasonCode: "below-confidence-floor", + action: "demoted", + proposed: () => proposed, + }); + + expect(after).toBe(proposed); + expect(inactive.telemetryError).toBe(true); + expect(inactive.summary("judgment.confidence")).toEqual({ + pass_id: "judgment.confidence", + status: "not-run", + reason_code: "configured-off", + }); + expect(inactive.evaluations()).toEqual([]); + }); + + it("exposes only internal ablation membership to pass-owned marker branches", () => { + const runtime = PolicyTraceRecorder.start({ + runId: "run-ablation-membership", + iter: 1, + ablated: ["judgment.confidence"], + }); + + expect(runtime.isAblated("judgment.confidence")).toBe(true); + }); + + it("lets ablation outrank protection without attaching a render-visible effect", () => { + const runtime = PolicyTraceRecorder.start({ + runId: "run-ablated-protection", + iter: 1, + ablated: ["judgment.confidence"], + }); + + const after = transitionFinding({ + runtime, + passId: "judgment.confidence", + finding: warnFinding, + opportunity: true, + matched: true, + reasonCode: "below-confidence-floor", + action: "demoted", + protectedBy: "high-precision-reviewer", + proposed: () => { + throw new Error("an ablated protected proposal must stay lazy"); + }, + }); + + expect(after).toBe(warnFinding); + expect(after?.policy_effects).toBeUndefined(); + expect(runtime.evaluations()).toEqual([ + { + pass_id: "judgment.confidence", + order: 140, + result: "would-apply", + before: "WARN", + after: "WARN", + reason_code: "below-confidence-floor", + source_signatures: ["sig-confidence"], + }, + ]); + expect(runtime.summary("judgment.confidence")).toMatchObject({ + would_apply: 1, + applied: 0, + protected: 0, + blocking_preserved: 1, + }); + }); +}); + describe("policy effect merging", () => { it("deduplicates identical effects and restores ascending catalog order", () => { const earlier = { From 00d804bc711881ba7f9b9086c9843bf81853f3b9 Mon Sep 17 00:00:00 2001 From: Codevena Date: Mon, 10 Aug 2026 07:23:06 +0200 Subject: [PATCH 32/55] fix(policy): isolate protected ablations --- src/core/aggregator.ts | 12 +- .../unit/policy-aggregator-first-half.test.ts | 179 ++++++++++++++++++ 2 files changed, 188 insertions(+), 3 deletions(-) diff --git a/src/core/aggregator.ts b/src/core/aggregator.ts index 33bb055..c8fdd2d 100644 --- a/src/core/aggregator.ts +++ b/src/core/aggregator.ts @@ -689,6 +689,7 @@ export function aggregate(input: AggregateInput): AggregateResult { : deduped; const critic = input.critic; + const criticAblated = input.policyRuntime?.isAblated("judgment.critic") ?? false; // #4: a BLOCKING finding whose every contributing base provider is high-precision is // protected from the SOFT demoters. Never protects a self_refuted (T1) or INFO finding — // only a real, blocking finding from a trusted reviewer. Anti-suppression by construction. @@ -789,11 +790,11 @@ export function aggregate(input: AggregateInput): AggregateResult { criticDropped.push(f); // INFO likely_fp dropped entirely — keep it attributable continue; } - if (matched && protectedBy === "high-precision-reviewer") { + if (!criticAblated && matched && protectedBy === "high-precision-reviewer") { survivors.push({ ...transitioned, protected_high_precision: true }); continue; } - if (matched && (isSecurityProtected || isCorroborated)) { + if (!criticAblated && matched && (isSecurityProtected || isCorroborated)) { survivors.push({ ...transitioned, critic_verdict: "keep" }); continue; } @@ -892,6 +893,7 @@ export function aggregate(input: AggregateInput): AggregateResult { // foreign (still tagged, so the agent can dispose it via an out-of-scope decision). Done as // an INDEPENDENT pass (not inside scopeFindings, which early-returns when scopeToDiff is off). const foreignFiles = input.foreignFiles; + const sessionAblated = input.policyRuntime?.isAblated("scope.session") ?? false; const foreignScoped: Finding[] = foreignFiles && foreignFiles.size > 0 ? deltaScoped.map((f) => { @@ -929,7 +931,11 @@ export function aggregate(input: AggregateInput): AggregateResult { // The legacy INFO marker is explanatory rather than a blocking policy // transition. A protected blocking finding is likewise tagged so the // out-of-scope disposition remains available. - if (isForeign && (f.severity === "INFO" || protectedBy !== undefined)) { + if ( + !sessionAblated && + isForeign && + (f.severity === "INFO" || protectedBy !== undefined) + ) { return { ...transitioned, foreign_to_session: true }; } return transitioned; diff --git a/tests/unit/policy-aggregator-first-half.test.ts b/tests/unit/policy-aggregator-first-half.test.ts index a5f4db5..bbb763a 100644 --- a/tests/unit/policy-aggregator-first-half.test.ts +++ b/tests/unit/policy-aggregator-first-half.test.ts @@ -375,6 +375,143 @@ describe("aggregator policy numeric contracts, orders 60-100", () => { } }); + it("removes the high-precision Critic marker when its protected match is ablated", () => { + const protectedFinding = finding({ signature: "sig-critic-high-precision-ablated" }); + const critic = new Map([ + [protectedFinding.signature, { verdict: "likely_fp" as const, reason: "not actionable" }], + ]); + const input = { + findings: [protectedFinding], + reviewersTotal: 1, + protectedReviewers: new Set(["codex"]), + critic, + }; + const legacy = aggregate(input); + const active = run("critic-high-precision-active", input); + const ablated = run("critic-high-precision-ablated", input, ["judgment.critic"]); + const control = aggregate({ + findings: [protectedFinding], + reviewersTotal: 1, + protectedReviewers: input.protectedReviewers, + }); + + expect(legacy.dedupedFindings[0]).toMatchObject({ protected_high_precision: true }); + expect(active.result.dedupedFindings[0]).toMatchObject({ protected_high_precision: true }); + expect(numericSummary(active.recorder, "judgment.critic")).toEqual( + PROTECTED_BLOCKING_PRESERVED, + ); + expect(numericSummary(ablated.recorder, "judgment.critic")).toEqual(ABLATED_BLOCKING_PRESERVED); + expect(ablated.result.dedupedFindings[0]).toEqual(control.dedupedFindings[0]); + expect(ablated.result.dedupedFindings[0]).toMatchObject({ + details: protectedFinding.details, + }); + expect(ablated.result.dedupedFindings[0]?.protected_high_precision).toBeUndefined(); + expect(ablated.result.dedupedFindings[0]?.critic_verdict).toBeUndefined(); + expect(ablated.result.dedupedFindings[0]?.critic_reason).toBeUndefined(); + expect(ablated.result.dedupedFindings[0]?.policy_effects).toBeUndefined(); + expect( + ablated.recorder + .evaluations() + .filter((evaluation) => evaluation.pass_id === "judgment.critic"), + ).toEqual([ + { + pass_id: "judgment.critic", + order: 70, + result: "would-apply", + before: "WARN", + after: "WARN", + reason_code: "critic-likely-fp", + source_signatures: [protectedFinding.signature], + }, + ]); + }); + + it("removes security and corroboration Critic keep markers when protected matches are ablated", () => { + const security = finding({ + signature: "sig-critic-security-ablated", + severity: "CRITICAL", + category: "security", + }); + const corroboratedA = finding({ + signature: "sig-critic-corroborated-a", + reviewer: { provider: "codex", model: "m", persona: "quality" }, + }); + const corroboratedB = finding({ + signature: "sig-critic-corroborated-b", + reviewer: { provider: "gemini", model: "m", persona: "quality" }, + }); + const cases: Array<{ + name: string; + findings: Finding[]; + reviewersTotal: number; + criticSignature: string; + severity: Finding["severity"]; + }> = [ + { + name: "security", + findings: [security], + reviewersTotal: 1, + criticSignature: security.signature, + severity: "CRITICAL", + }, + { + name: "corroboration", + findings: [corroboratedA, corroboratedB], + reviewersTotal: 3, + criticSignature: corroboratedB.signature, + severity: "WARN", + }, + ]; + + for (const testCase of cases) { + const critic = new Map([[testCase.criticSignature, { verdict: "likely_fp" as const }]]); + const input = { + findings: testCase.findings, + reviewersTotal: testCase.reviewersTotal, + critic, + }; + const legacy = aggregate(input); + const active = run(`critic-${testCase.name}-active`, input); + const ablated = run(`critic-${testCase.name}-ablated`, input, ["judgment.critic"]); + const control = aggregate({ + findings: testCase.findings, + reviewersTotal: testCase.reviewersTotal, + }); + + expect(legacy.dedupedFindings[0]).toMatchObject({ critic_verdict: "keep" }); + expect(active.result.dedupedFindings[0]).toMatchObject({ critic_verdict: "keep" }); + expect(numericSummary(active.recorder, "judgment.critic")).toEqual( + PROTECTED_BLOCKING_PRESERVED, + ); + expect(numericSummary(ablated.recorder, "judgment.critic")).toEqual( + ABLATED_BLOCKING_PRESERVED, + ); + expect(ablated.result.dedupedFindings[0]).toEqual(control.dedupedFindings[0]); + expect(ablated.result.dedupedFindings[0]).toMatchObject({ + details: testCase.findings[0]?.details, + }); + expect(ablated.result.dedupedFindings[0]?.critic_verdict).toBeUndefined(); + expect(ablated.result.dedupedFindings[0]?.critic_reason).toBeUndefined(); + expect(ablated.result.dedupedFindings[0]?.protected_high_precision).toBeUndefined(); + expect(ablated.result.dedupedFindings[0]?.policy_effects).toBeUndefined(); + expect( + ablated.recorder + .evaluations() + .filter((evaluation) => evaluation.pass_id === "judgment.critic"), + ).toEqual([ + { + pass_id: "judgment.critic", + order: 70, + result: "would-apply", + before: testCase.severity, + after: testCase.severity, + reason_code: "critic-likely-fp", + source_signatures: testCase.findings.map((item) => item.signature), + }, + ]); + } + }); + it("records diff-scope no-opportunity, miss, active, ablated, and protected tuples", () => { const ranges = new Map([["src/a.ts", [[10, 14]] as Array<[number, number]>]]); const noLine = run("diff-no-line", { @@ -592,6 +729,48 @@ describe("aggregator policy numeric contracts, orders 60-100", () => { protected_by: "out-of-diff-blocking-hatch", }); }); + + it("removes the foreign-session hatch marker when its protected match is ablated", () => { + const protectedFinding = finding({ signature: "sig-session-hatch-ablated" }); + const input = { + findings: [protectedFinding], + reviewersTotal: 1, + foreignFiles: new Set([protectedFinding.file]), + outOfDiffBlocking: [protectedFinding.category], + }; + const legacy = aggregate(input); + const active = run("session-hatch-active", input); + const ablated = run("session-hatch-ablated", input, ["scope.session"]); + const control = aggregate({ + findings: [protectedFinding], + reviewersTotal: 1, + outOfDiffBlocking: input.outOfDiffBlocking, + }); + + expect(legacy.dedupedFindings[0]).toMatchObject({ foreign_to_session: true }); + expect(active.result.dedupedFindings[0]).toMatchObject({ foreign_to_session: true }); + expect(numericSummary(active.recorder, "scope.session")).toEqual(PROTECTED_BLOCKING_PRESERVED); + expect(numericSummary(ablated.recorder, "scope.session")).toEqual(ABLATED_BLOCKING_PRESERVED); + expect(ablated.result.dedupedFindings[0]).toEqual(control.dedupedFindings[0]); + expect(ablated.result.dedupedFindings[0]).toMatchObject({ + details: protectedFinding.details, + }); + expect(ablated.result.dedupedFindings[0]?.foreign_to_session).toBeUndefined(); + expect(ablated.result.dedupedFindings[0]?.policy_effects).toBeUndefined(); + expect( + ablated.recorder.evaluations().filter((evaluation) => evaluation.pass_id === "scope.session"), + ).toEqual([ + { + pass_id: "scope.session", + order: 100, + result: "would-apply", + before: "WARN", + after: "WARN", + reason_code: "foreign-to-session", + source_signatures: [protectedFinding.signature], + }, + ]); + }); }); describe("aggregation cluster lineage", () => { From 3938b6e3b53718ba63492de4dcd0d19dc211ba27 Mon Sep 17 00:00:00 2001 From: Codevena Date: Mon, 10 Aug 2026 07:49:54 +0200 Subject: [PATCH 33/55] feat(policy): wire trace lifecycle through orchestration --- src/core/orchestrator.ts | 73 ++- src/core/policy/replay.ts | 38 ++ src/core/run-summary.ts | 10 + .../policy-trace-equivalence.test.ts | 463 ++++++++++++++++++ .../run-summary-orchestrator.test.ts | 3 + tests/unit/orchestrator-policy-trace.test.ts | 59 +++ 6 files changed, 641 insertions(+), 5 deletions(-) create mode 100644 src/core/policy/replay.ts create mode 100644 tests/integration/policy-trace-equivalence.test.ts create mode 100644 tests/unit/orchestrator-policy-trace.test.ts diff --git a/src/core/orchestrator.ts b/src/core/orchestrator.ts index bef5992..a3ce012 100644 --- a/src/core/orchestrator.ts +++ b/src/core/orchestrator.ts @@ -52,6 +52,7 @@ import type { RunSummary } from "../schemas/audit-event.ts"; import { type MemoryProposal, VALID_EVIDENCE_KINDS } from "../schemas/brain.ts"; import type { Finding, FindingCategory } from "../schemas/finding.ts"; import { NO_PANEL_REVIEWER_ID } from "../schemas/pending-report.ts"; +import type { PolicySummary, PolicyTrace } from "../schemas/policy-trace.ts"; import type { PassLedger, ReviewedSnapshot } from "../schemas/state.ts"; import { triageFromFacts } from "../triage/matrix.ts"; import { refineTriage } from "../triage/triage-engine.ts"; @@ -97,6 +98,10 @@ import { orderForBudget, renderLoreBlock, selectForDiff } from "./lore/render.ts import { classifyEntry } from "./lore/staleness.ts"; import { type LoreEntryParsed, loadLore } from "./lore/store.ts"; import { PERSONA_REAFFIRM, reaffirmFor, resolvePersonas } from "./personas.ts"; +import type { PolicyExecutionOptions } from "./policy/replay.ts"; +import { resolvePolicyExecutionOptions } from "./policy/replay.ts"; +import { OrderedResponseHashes } from "./policy/response-hashes.ts"; +import { PolicyTraceRecorder } from "./policy/trace.ts"; import { HIGH_PRECISION_FLOOR, PROTECT_MIN_DECISIONS, @@ -233,6 +238,9 @@ export interface OrchestratorInput { // Benchmark-only reliability control. Runtime callers omit this and retain // exactly one critic completion attempt. criticMaxAttempts?: number; + // Internal-only policy instrumentation/ablation. Normal Gate construction omits + // this and resolves to persist with an AuditLogger; direct/tests resolve off. + policyExecution?: PolicyExecutionOptions; } // P1a (bench): a single reviewer's pre-aggregation output, captured per attempt. @@ -306,6 +314,10 @@ export interface IterationResult { // — a documented v1 boundary: the canon-guard finding fires on the NEXT gate // run instead, see docs/superpowers/specs/2026-07-09-lore-design.md). loreOutcomes?: { reminderEmittedId?: string; promotions: string[] }; + // Server-owned policy telemetry. Physical persistence and compact summary + // binding are added by Task 7; memory mode exposes only the complete trace. + policyTrace?: PolicyTrace; + policySummary?: PolicySummary; } // Structural contract the LoopDriver depends on — lets the driver race a run @@ -717,6 +729,10 @@ export class Orchestrator { }): Promise { const start = Date.now(); const repo = this.input.repoRoot; + const policyExecution = resolvePolicyExecutionOptions( + this.input.policyExecution, + this.input.audit !== undefined, + ); // S1: render prior adjudications ONCE — injected as trusted prompt context (before the // untrusted diff fence) AND hashed into the behavior cache key below. const adjudicationsText = renderAdjudications(opts.priorAdjudications ?? []); @@ -802,6 +818,7 @@ export class Orchestrator { criticCostUsd: 0, findings: [], runs: [], + policyTraceStatus: "not-run", }), }; } @@ -822,6 +839,7 @@ export class Orchestrator { criticCostUsd: 0, findings: [], runs: [], + policyTraceStatus: "not-run", }), }; } @@ -870,6 +888,7 @@ export class Orchestrator { criticCostUsd: 0, findings: [f], runs: [], + policyTraceStatus: "not-run", }), }; } @@ -1447,6 +1466,7 @@ export class Orchestrator { criticCostUsd: 0, findings: [], runs: [], + policyTraceStatus: "not-run", }), }; } @@ -1488,6 +1508,7 @@ export class Orchestrator { criticCostUsd: 0, findings: [], runs: [], + policyTraceStatus: "not-run", }), }; } @@ -2203,6 +2224,7 @@ export class Orchestrator { criticCostUsd: 0, findings: [], runs: reviewerOutcomes, + policyTraceStatus: "not-run", }), }; } @@ -2217,6 +2239,23 @@ export class Orchestrator { .flatMap((s) => s.res.findings) .filter((f) => !isExcludedFromReview(f.file)); const symbolFindings = await this.applySymbolSignatures(rawFindings); + const reviewerResponseHashes = new OrderedResponseHashes(); + if (policyExecution.trace !== "off") { + for (const [ordinal, run] of settled.entries()) { + reviewerResponseHashes.record(`reviewer:${run.provider}`, ordinal, run.res.rawText); + } + } + const rawResponseSha256 = reviewerResponseHashes.values(); + // One recorder owns the complete policy lifecycle. It is deliberately + // created at the last boundary before the first outcome-changing pass. + const policyRuntime = + policyExecution.trace === "off" + ? undefined + : PolicyTraceRecorder.start({ + runId: opts.runId, + iter: opts.iter, + ablated: policyExecution.policyAblations, + }); // Deterministic fact-check BEFORE grounding: a finding whose cited file:line // provably does not exist in the working tree (file empty / line out of range) is // a hallucination — demote it to advisory so a singleton reviewer can't hard-FAIL @@ -2227,21 +2266,32 @@ export class Orchestrator { symbolFindings, this.input.repoRoot, parseDeletedPaths(this.input.diff), + policyRuntime, ); // #1 (field report 2026-06-17): demote a finding whose OWN conclusion retracts it // ("…appears safe", "No issue", "No defect") to INFO before grounding/critic/aggregate, // so a self-contradicting WARN/CRITICAL never blocks the gate. First-party retraction // signal → category-independent; deterministic, demote-only, fail-safe. + const selfRefutationEnabled = this.input.config.phases.review.selfRefutationFilter !== false; + if (!selfRefutationEnabled) { + policyRuntime?.markInactive("evidence.self-refutation", "configured-off"); + } const selfScreenedFindings = demoteSelfRefuting( factCheckedFindings, - this.input.config.phases.review.selfRefutationFilter !== false, + selfRefutationEnabled, + policyRuntime, ); // non-convergence #2: demote a CRITICAL the reviewer's own text frames as currently-safe / // hypothetical / future fragility (no present defect) one step to WARN — pre-aggregate, like // self-refutation/grounding. One-step, security/correctness-exempt, fail-safe. + const hypotheticalEnabled = this.input.config.phases.review.hypotheticalSeverityGuard !== false; + if (!hypotheticalEnabled) { + policyRuntime?.markInactive("judgment.hypothetical", "configured-off"); + } const allFindings = demoteHypotheticalCriticals( selfScreenedFindings, - this.input.config.phases.review.hypotheticalSeverityGuard !== false, + hypotheticalEnabled, + policyRuntime, ); // S6 grounding corpus = diff + the WHOLE-FILE content of changed files (`fileContext`), // deliberately NOT the scoped `promptContext` the reviewer prompt now uses: grounding @@ -2253,7 +2303,7 @@ export class Orchestrator { const groundingCorpus = `${this.input.diff}\n${fileContext ?? ""}`; // Layer 1 (deterministic, no LLM): demote a CRITICAL citing a code-shaped token absent // from the corpus. - let groundedFindings = groundFindings(allFindings, groundingCorpus); + let groundedFindings = groundFindings(allFindings, groundingCorpus, policyRuntime); // Layer 2 (LLM judge, opt-in via phases.grounding): demote a CRITICAL whose claim is // SEMANTICALLY fabricated (e.g. an invented `outerHTML` XSS sink where the code only sets // a React aria-label). Only fires when there is a CRITICAL to judge; any error → no demote. @@ -2264,7 +2314,7 @@ export class Orchestrator { | ProviderConfig | undefined; if (gAdapter && gProviderCfg) { - const { map } = await judgeGrounding( + const { map, rawResponseSha256: groundingResponseSha256 } = await judgeGrounding( gAdapter, { model: groundingCfg.model ?? gProviderCfg.model, @@ -2279,8 +2329,13 @@ export class Orchestrator { groundedFindings, groundingCorpus, ); - groundedFindings = applyGroundingJudgeVerdicts(groundedFindings, map); + if (groundingResponseSha256 !== undefined) { + rawResponseSha256.push(groundingResponseSha256); + } + groundedFindings = applyGroundingJudgeVerdicts(groundedFindings, map, policyRuntime); } + } else if (groundingCfg === null || groundingCfg === undefined) { + policyRuntime?.markInactive("judgment.grounding-llm", "configured-off"); } // --- Optional critic phase (demote-only) --- @@ -2340,6 +2395,7 @@ export class Orchestrator { ); criticMap = r.map.size > 0 ? r.map : undefined; criticInfo = r.info; + if (r.rawResponseSha256 !== undefined) rawResponseSha256.push(r.rawResponseSha256); } else { criticInfo = { provider: criticCfg.provider, status: "misconfigured", verdicts: 0 }; } @@ -2480,6 +2536,7 @@ export class Orchestrator { // (one-shot plan reviews have no decision cycle to bind regions to). The same // value feeds regionsSegment in the cache key above. ...(activeRegions ? { rejectedRegions: activeRegions } : {}), + ...(policyRuntime === undefined ? {} : { policyRuntime }), }); // Include critic-DROPPED likely_fp findings (INFO → drop): they never reach @@ -2680,6 +2737,11 @@ export class Orchestrator { loreFindingsBuilt.length > 0 ? { ...agg.counts, info: agg.counts.info + loreFindingsBuilt.length } : agg.counts; + const policyTrace = policyRuntime?.finalize({ + rawResponseSha256, + verdict: agg.verdict, + finalFindings: reportFindings, + }); // Banner data for invalid/broad/zero-match entries + the render-budget drop // count — render-only (report-writer.ts), never affects the verdict. const loreBanner = @@ -2871,6 +2933,7 @@ export class Orchestrator { }, } : {}), + ...(policyTrace === undefined || policyTrace === null ? {} : { policyTrace }), summary: buildRunSummary({ verdict: agg.verdict, source: "panel", diff --git a/src/core/policy/replay.ts b/src/core/policy/replay.ts new file mode 100644 index 0000000..240dd65 --- /dev/null +++ b/src/core/policy/replay.ts @@ -0,0 +1,38 @@ +import type { PolicyPassId } from "./catalog.ts"; + +export type PolicyTraceMode = "off" | "memory" | "persist"; + +export interface PolicyIsolatedStateMetadata { + readonly startingStateSha256: string; + readonly scratchStateRoot: string; + readonly productionStateRoot: string; +} + +export interface PolicyExecutionOptions { + readonly trace: PolicyTraceMode; + readonly policyAblations: ReadonlySet; + readonly authoritative: boolean; + readonly isolatedState?: PolicyIsolatedStateMetadata; +} + +export const EMPTY_POLICY_ABLATIONS: ReadonlySet = new Set(); + +const DIRECT_DEFAULT: PolicyExecutionOptions = { + trace: "off", + policyAblations: EMPTY_POLICY_ABLATIONS, + authoritative: false, +}; + +const AUDITED_DEFAULT: PolicyExecutionOptions = { + trace: "persist", + policyAblations: EMPTY_POLICY_ABLATIONS, + authoritative: false, +}; + +export function resolvePolicyExecutionOptions( + options: PolicyExecutionOptions | undefined, + hasAuditLogger: boolean, +): PolicyExecutionOptions { + if (options !== undefined) return options; + return hasAuditLogger ? AUDITED_DEFAULT : DIRECT_DEFAULT; +} diff --git a/src/core/run-summary.ts b/src/core/run-summary.ts index b14ae2e..79663f3 100644 --- a/src/core/run-summary.ts +++ b/src/core/run-summary.ts @@ -23,6 +23,9 @@ export interface BuildRunSummaryInput { // #6 instrumentation: count of uncited project/house-rule findings this run. Omitted on the // non-panel paths (skip/cache/error) where no reviewer findings were produced. ruleUncited?: number; + policyTraceStatus?: RunSummary["policy_trace_status"]; + policyTraceRef?: string; + policyTraceSha256?: string; } function isDemoted(f: Finding): boolean { @@ -97,5 +100,12 @@ export function buildRunSummary(input: BuildRunSummaryInput): RunSummary { (f) => f.reputation_corroboration_required === true, ).length, ...(input.ruleUncited !== undefined ? { rule_uncited: input.ruleUncited } : {}), + ...(input.policyTraceStatus !== undefined + ? { policy_trace_status: input.policyTraceStatus } + : {}), + ...(input.policyTraceRef !== undefined ? { policy_trace_ref: input.policyTraceRef } : {}), + ...(input.policyTraceSha256 !== undefined + ? { policy_trace_sha256: input.policyTraceSha256 } + : {}), }; } diff --git a/tests/integration/policy-trace-equivalence.test.ts b/tests/integration/policy-trace-equivalence.test.ts new file mode 100644 index 0000000..4d5a008 --- /dev/null +++ b/tests/integration/policy-trace-equivalence.test.ts @@ -0,0 +1,463 @@ +import { afterAll, describe, expect, it, setSystemTime } from "bun:test"; +import { createHash } from "node:crypto"; +import { mkdtempSync, readFileSync, readdirSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { AuditLogger } from "../../src/audit/logger.ts"; +import { defaultConfig } from "../../src/config/defaults.ts"; +import type { ReviewgateConfig } from "../../src/config/define-config.ts"; +import { + type IterationResult, + Orchestrator, + type OrchestratorInput, +} from "../../src/core/orchestrator.ts"; +import type { ProviderAdapter, ReviewResult } from "../../src/providers/adapter-base.ts"; +import type { Finding } from "../../src/schemas/finding.ts"; + +const DIFF = [ + "diff --git a/a.ts b/a.ts", + "--- a/a.ts", + "+++ b/a.ts", + "@@ -1 +1 @@", + "-export const value = 0;", + "+export const value = 1;", + "", +].join("\n"); + +const RAW_BY_PROVIDER = { + codex: '{"verdict":"FAIL","findings":[{"source":"codex"}]}', + "claude-code": '{"verdict":"FAIL","findings":[{"source":"claude-code"}]}', +} as const; + +type TraceMode = NonNullable["trace"]; +type PolicyAwareInput = OrchestratorInput; +type PolicyAwareResult = IterationResult; + +function sha256(text: string): string { + return createHash("sha256").update(Buffer.from(text, "utf8")).digest("hex"); +} + +function finding(provider: "codex" | "claude-code"): Finding { + return { + id: "F-001", + signature: "same-finding", + severity: "INFO", + category: "quality", + rule_id: "same-rule", + file: "a.ts", + line_start: 1, + line_end: 1, + message: "The changed value needs review", + details: "The same deterministic observation from both configured reviewer slots.", + reviewer: { provider, model: "fixture-model", persona: "quality" }, + confidence: 0.9, + consensus: "singleton", + }; +} + +function adapter(provider: "codex" | "claude-code", calls: string[]): ProviderAdapter { + return { + id: provider, + async preflight() { + return { available: true, version: "fixture", authMode: "oauth", error: null }; + }, + async review(input) { + calls.push(provider); + return { + reviewerId: input.reviewerId, + verdict: "FAIL", + findings: [finding(provider)], + usage: { inputTokens: 1, outputTokens: 1, costUsd: 0, quotaUsedPct: null }, + durationMs: 1, + exitCode: 0, + rawEventsPath: "", + rawText: RAW_BY_PROVIDER[provider], + status: "ok", + } satisfies ReviewResult; + }, + }; +} + +function stripPolicyTelemetry(value: unknown): unknown { + if (Array.isArray(value)) return value.map(stripPolicyTelemetry); + if (value === null || typeof value !== "object") return value; + const out: Record = {}; + for (const [key, child] of Object.entries(value)) { + if ( + key === "policy_effects" || + key === "policy_summary" || + key === "policy_trace_status" || + key === "policy_trace_ref" || + key === "policy_trace_sha256" + ) { + continue; + } + out[key] = stripPolicyTelemetry(child); + } + return out; +} + +async function run( + mode?: TraceMode, + withAudit = false, + reviewOverrides: Partial = {}, + reportMode: OrchestratorInput["reportMode"] = "gate", +): Promise<{ + repo: string; + calls: string[]; + result: PolicyAwareResult; + report: unknown; + markdown: Buffer; + reviewCache: Record; +}> { + const repo = mkdtempSync(join(tmpdir(), `rg-policy-${mode ?? "default"}-`)); + writeFileSync(join(repo, "a.ts"), "export const value = 1;\n"); + const calls: string[] = []; + const config: ReviewgateConfig = { + ...defaultConfig, + cache: { enabled: true, reviewTtlDays: 7 }, + providers: { + ...defaultConfig.providers, + "claude-code": { ...defaultConfig.providers["claude-code"], enabled: true }, + }, + phases: { + ...defaultConfig.phases, + review: { + ...defaultConfig.phases.review, + ...reviewOverrides, + reviewers: [ + { provider: "codex" as const, persona: "quality" }, + { provider: "claude-code" as const, persona: "quality" }, + ], + providerPrecisionContext: false, + }, + brain: null, + critic: null, + fpLedger: null, + grounding: null, + implicitOutcomes: null, + lore: null, + triage: null, + }, + }; + const baseInput: OrchestratorInput = { + repoRoot: repo, + config, + ...(withAudit ? { audit: new AuditLogger(join(repo, ".reviewgate", "audit")) } : {}), + adapters: { + codex: adapter("codex", calls), + "claude-code": adapter("claude-code", calls), + }, + sandboxMode: "off", + hostTier: "opus", + agentHost: "codex", + diff: DIFF, + gitInfo: { sha: "a".repeat(40), branch: "fixture", dirtyFiles: ["a.ts"] }, + reasonOnFailEnabled: true, + disableLastResortFailover: true, + reportMode, + }; + const input: OrchestratorInput | PolicyAwareInput = + mode === undefined + ? baseInput + : { + ...baseInput, + policyExecution: { + trace: mode, + policyAblations: new Set(), + authoritative: false, + }, + }; + const result = (await new Orchestrator(input).runIteration({ + runId: "RUN-POLICY-EQUIVALENCE", + iter: 1, + })) as PolicyAwareResult; + const reviewCacheDir = join(repo, ".reviewgate", "cache", "reviews"); + const reviewCache = Object.fromEntries( + readdirSync(reviewCacheDir) + .sort() + .map((name) => [name, readFileSync(join(reviewCacheDir, name), "utf8")]), + ); + return { + repo, + calls, + result, + report: JSON.parse( + readFileSync( + join(repo, ".reviewgate", reportMode === "one-shot" ? "plan-review.json" : "pending.json"), + "utf8", + ), + ), + markdown: readFileSync( + join(repo, ".reviewgate", reportMode === "one-shot" ? "plan-review.md" : "pending.md"), + ), + reviewCache, + }; +} + +describe("policy trace lifecycle equivalence", () => { + setSystemTime(new Date("2026-08-10T12:00:00.000Z")); + afterAll(() => setSystemTime()); + + it("keeps findings, Markdown, and cache identity byte-neutral while hashing reviewer slots in order", async () => { + const traced = await run("memory"); + const legacy = await run("off"); + + expect(traced.calls).toEqual(["codex", "claude-code"]); + expect(legacy.calls).toEqual(traced.calls); + expect(traced.result.policyTrace?.raw_response_sha256).toEqual([ + sha256(RAW_BY_PROVIDER.codex), + sha256(RAW_BY_PROVIDER["claude-code"]), + ]); + expect(traced.result.verdict).toBe(legacy.result.verdict); + expect(traced.result.summary.counts).toEqual(legacy.result.summary.counts); + expect(traced.result.signaturesThisIter).toEqual(legacy.result.signaturesThisIter); + expect(stripPolicyTelemetry(traced.report)).toEqual(stripPolicyTelemetry(legacy.report)); + expect(traced.markdown.equals(legacy.markdown)).toBe(true); + expect(traced.reviewCache).toEqual(legacy.reviewCache); + }); + + it("owns the full memory trace only in IterationResult and does not invent persistence metadata", async () => { + const traced = await run("memory"); + const legacy = await run("off"); + const tracedReport = traced.report as Record; + + expect(traced.result.policyTrace).toBeDefined(); + expect(traced.result.policySummary).toBeUndefined(); + expect(tracedReport.policy_summary).toBeUndefined(); + expect(traced.result.summary.policy_trace_ref).toBeUndefined(); + expect(traced.result.summary.policy_trace_sha256).toBeUndefined(); + expect(legacy.result.policyTrace).toBeUndefined(); + expect(legacy.result.policySummary).toBeUndefined(); + }); + + it("keeps one-shot output free of an unbound policy summary and artifact reference", async () => { + const traced = await run("memory", false, {}, "one-shot"); + const report = traced.report as Record; + + expect(traced.result.policyTrace).toBeDefined(); + expect(report.policy_summary).toBeUndefined(); + expect(traced.result.summary.policy_trace_ref).toBeUndefined(); + expect(traced.result.summary.policy_trace_sha256).toBeUndefined(); + }); + + it("defaults the ordinary AuditLogger path to persist mode with an empty internal ablation set", async () => { + const production = await run(undefined, true); + + expect(production.result.policyTrace).toBeDefined(); + expect(production.result.policyTrace?.ablated).toEqual([]); + expect(production.result.policySummary).toBeUndefined(); + }); + + it("orders reviewer, grounding, and critic response hashes by logical call order", async () => { + const repo = mkdtempSync(join(tmpdir(), "rg-policy-response-order-")); + writeFileSync(join(repo, "a.ts"), "export const value = 1;\n"); + const reviewerRaw = '{"verdict":"FAIL","findings":[{"source":"reviewer"}]}'; + const groundingRaw = '{"verdicts":[{"signature":"ordered-signature","grounded":true}]}'; + const criticRaw = '{"verdicts":[{"signature":"ordered-signature","verdict":"keep"}]}'; + const critical: Finding = { + ...finding("codex"), + signature: "ordered-signature", + severity: "CRITICAL", + message: "The changed `value` needs review", + }; + const completionAdapter = (id: "gemini" | "opencode", response: string): ProviderAdapter => ({ + id, + async preflight() { + return { available: true, version: "fixture", authMode: "oauth", error: null }; + }, + async review(input) { + return { + reviewerId: input.reviewerId, + verdict: "PASS", + findings: [], + usage: { inputTokens: 0, outputTokens: 0, costUsd: 0, quotaUsedPct: null }, + durationMs: 1, + exitCode: 0, + rawEventsPath: "", + rawText: "", + status: "ok", + } satisfies ReviewResult; + }, + async complete() { + return response; + }, + }); + const reviewer: ProviderAdapter = { + id: "codex", + async preflight() { + return { available: true, version: "fixture", authMode: "oauth", error: null }; + }, + async review(input) { + return { + reviewerId: input.reviewerId, + verdict: "FAIL", + findings: [critical], + usage: { inputTokens: 1, outputTokens: 1, costUsd: 0, quotaUsedPct: null }, + durationMs: 1, + exitCode: 0, + rawEventsPath: "", + rawText: reviewerRaw, + status: "ok", + } satisfies ReviewResult; + }, + }; + const config: ReviewgateConfig = { + ...defaultConfig, + cache: { enabled: false, reviewTtlDays: 7 }, + providers: { + ...defaultConfig.providers, + gemini: { ...defaultConfig.providers.gemini, enabled: true }, + opencode: { ...defaultConfig.providers.opencode, enabled: true }, + }, + phases: { + ...defaultConfig.phases, + review: { + ...defaultConfig.phases.review, + reviewers: [{ provider: "codex" as const, persona: "quality" }], + providerPrecisionContext: false, + }, + brain: null, + critic: { provider: "opencode" as const, persona: "fp-filter" }, + fpLedger: null, + grounding: { provider: "gemini" as const }, + implicitOutcomes: null, + lore: null, + triage: null, + }, + }; + const input: PolicyAwareInput = { + repoRoot: repo, + config, + adapters: { + codex: reviewer, + gemini: completionAdapter("gemini", groundingRaw), + opencode: completionAdapter("opencode", criticRaw), + }, + sandboxMode: "off", + hostTier: "opus", + agentHost: "codex", + diff: DIFF, + reasonOnFailEnabled: true, + disableLastResortFailover: true, + policyExecution: { + trace: "memory", + policyAblations: new Set(), + authoritative: true, + }, + }; + + const result = (await new Orchestrator(input).runIteration({ + runId: "RUN-RESPONSE-ORDER", + iter: 1, + })) as PolicyAwareResult; + + expect(result.policyTrace?.raw_response_sha256).toEqual([ + sha256(reviewerRaw), + sha256(groundingRaw), + sha256(criticRaw), + ]); + }); + + it("marks configured-inactive pre-aggregation passes not-run without evaluations", async () => { + const traced = await run("memory", false, { + selfRefutationFilter: false, + hypotheticalSeverityGuard: false, + }); + const status = new Map( + traced.result.policyTrace?.passes.map((pass) => [ + pass.pass_id, + [pass.status, "reason_code" in pass ? pass.reason_code : undefined], + ]), + ); + + expect(status.get("evidence.self-refutation")).toEqual(["not-run", "configured-off"]); + expect(status.get("judgment.hypothetical")).toEqual(["not-run", "configured-off"]); + expect(status.get("judgment.grounding-llm")).toEqual(["not-run", "configured-off"]); + }); + + it("preserves the exact production demotion when recorder validation fails", async () => { + const execute = async (mode: "off" | "memory") => { + const repo = mkdtempSync(join(tmpdir(), `rg-policy-recorder-error-${mode}-`)); + writeFileSync(join(repo, "a.ts"), "export const value = 1;\n"); + const lowConfidence = { + ...finding("codex"), + signature: "", + severity: "WARN" as const, + confidence: 0.1, + }; + const config: ReviewgateConfig = { + ...defaultConfig, + cache: { enabled: false, reviewTtlDays: 7 }, + phases: { + ...defaultConfig.phases, + review: { + ...defaultConfig.phases.review, + reviewers: [{ provider: "codex" as const, persona: "quality" }], + providerPrecisionContext: false, + }, + brain: null, + critic: null, + fpLedger: null, + grounding: null, + implicitOutcomes: null, + lore: null, + triage: null, + }, + }; + const reviewer: ProviderAdapter = { + id: "codex", + async preflight() { + return { available: true, version: "fixture", authMode: "oauth", error: null }; + }, + async review(input) { + return { + reviewerId: input.reviewerId, + verdict: "FAIL", + findings: [lowConfidence], + usage: { inputTokens: 1, outputTokens: 1, costUsd: 0, quotaUsedPct: null }, + durationMs: 1, + exitCode: 0, + rawEventsPath: "", + rawText: '{"verdict":"FAIL","findings":[]}', + status: "ok", + } satisfies ReviewResult; + }, + }; + const input: PolicyAwareInput = { + repoRoot: repo, + config, + adapters: { codex: reviewer }, + sandboxMode: "off", + hostTier: "opus", + agentHost: "codex", + diff: DIFF, + reasonOnFailEnabled: true, + disableLastResortFailover: true, + policyExecution: { + trace: mode, + policyAblations: new Set(), + authoritative: false, + }, + }; + const result = (await new Orchestrator(input).runIteration({ + runId: "RUN-RECORDER-ERROR", + iter: 1, + })) as PolicyAwareResult; + return { + result, + report: JSON.parse(readFileSync(join(repo, ".reviewgate", "pending.json"), "utf8")) as { + findings: Finding[]; + }, + }; + }; + + const traced = await execute("memory"); + const legacy = await execute("off"); + + expect(traced.result.policyTrace).toBeUndefined(); + expect(traced.report.findings[0]?.severity).toBe("INFO"); + expect(traced.report.findings[0]?.low_confidence).toBe(true); + expect(stripPolicyTelemetry(traced.report)).toEqual(stripPolicyTelemetry(legacy.report)); + }); +}); diff --git a/tests/integration/run-summary-orchestrator.test.ts b/tests/integration/run-summary-orchestrator.test.ts index d4822df..5585630 100644 --- a/tests/integration/run-summary-orchestrator.test.ts +++ b/tests/integration/run-summary-orchestrator.test.ts @@ -89,5 +89,8 @@ describe("orchestrator IterationResult.summary", () => { const result = await orch.runIteration({ runId: "RUN", iter: 1 }); expect(result.summary.source).toBe("skipped"); expect(result.summary.providers).toEqual([]); + expect(result.summary.policy_trace_status).toBe("not-run"); + expect(result.summary.policy_trace_ref).toBeUndefined(); + expect(result.summary.policy_trace_sha256).toBeUndefined(); }); }); diff --git a/tests/unit/orchestrator-policy-trace.test.ts b/tests/unit/orchestrator-policy-trace.test.ts new file mode 100644 index 0000000..e59943d --- /dev/null +++ b/tests/unit/orchestrator-policy-trace.test.ts @@ -0,0 +1,59 @@ +import { describe, expect, it } from "bun:test"; +import { readFileSync } from "node:fs"; +import { join } from "node:path"; +import { + EMPTY_POLICY_ABLATIONS, + resolvePolicyExecutionOptions, +} from "../../src/core/policy/replay.ts"; + +const REPO_ROOT = join(import.meta.dir, "..", ".."); + +describe("internal policy execution selection", () => { + it("keeps legacy direct construction off and defaults the AuditLogger path to persist", () => { + const direct = resolvePolicyExecutionOptions(undefined, false); + const production = resolvePolicyExecutionOptions(undefined, true); + + expect(direct).toEqual({ + trace: "off", + policyAblations: EMPTY_POLICY_ABLATIONS, + authoritative: false, + }); + expect(production).toEqual({ + trace: "persist", + policyAblations: EMPTY_POLICY_ABLATIONS, + authoritative: false, + }); + expect(production.policyAblations.size).toBe(0); + }); + + it("preserves explicit internal memory mode and its ablation identity", () => { + const ablations = new Set(["judgment.confidence"] as const); + const resolved = resolvePolicyExecutionOptions( + { trace: "memory", policyAblations: ablations, authoritative: true }, + false, + ); + + expect(resolved.trace).toBe("memory"); + expect(resolved.policyAblations).toBe(ablations); + expect(resolved.authoritative).toBe(true); + }); +}); + +describe("policy ablations stay internal", () => { + it("has no policyAblations mapping in Gate, Config, Setup, config schemas, or env parsing", async () => { + const guardedFiles = [ + "src/cli/commands/gate.ts", + "src/cli/commands/config.ts", + "src/cli/commands/setup.ts", + ]; + const configFiles = new Bun.Glob("src/config/**/*.ts"); + for await (const path of configFiles.scan({ cwd: REPO_ROOT, onlyFiles: true })) { + guardedFiles.push(path); + } + + for (const path of guardedFiles) { + const source = readFileSync(join(REPO_ROOT, path), "utf8"); + expect(source, path).not.toContain("policyAblations"); + } + }); +}); From b946a2e1e70a65fb7aa852fbdc875853649e1483 Mon Sep 17 00:00:00 2001 From: Codevena Date: Mon, 10 Aug 2026 08:26:51 +0200 Subject: [PATCH 34/55] fix(policy): complete authoritative trace inputs --- src/core/aggregator.ts | 219 +++++++++--------- src/core/critic.ts | 7 + src/core/orchestrator.ts | 46 +++- src/schemas/policy-trace.ts | 59 ++++- .../policy-trace-equivalence.test.ts | 94 ++++++-- tests/unit/critic-runner.test.ts | 22 +- tests/unit/orchestrator-lore.test.ts | 43 ++++ .../unit/policy-aggregator-first-half.test.ts | 43 ++++ tests/unit/policy-trace-schema.test.ts | 118 ++++++++++ 9 files changed, 510 insertions(+), 141 deletions(-) diff --git a/src/core/aggregator.ts b/src/core/aggregator.ts index c8fdd2d..939de19 100644 --- a/src/core/aggregator.ts +++ b/src/core/aggregator.ts @@ -13,10 +13,14 @@ import { ruleIdToken0 } from "./fp-ledger/clusters.ts"; import type { PolicyProtectionCode, PolicyReasonCode } from "./policy/catalog.ts"; import { type PolicyRuntime, mergePolicyEffects, transitionFinding } from "./policy/trace.ts"; +type FirstHalfPolicyPassId = "judgment.critic" | "scope.diff" | "scope.delta" | "scope.session"; +type InactivePolicyReason = Extract; + export interface AggregateInput { findings: Finding[]; reviewersTotal: number; policyRuntime?: PolicyRuntime; + policyInactive?: Partial>; critic?: Map; // M5 Part A: per-file changed new-file line ranges. When provided and // scopeToDiff !== false, findings outside the changed hunks are demoted to INFO. @@ -326,6 +330,17 @@ function sourceSignatures(f: Finding): string[] { // outside the changed hunks. Paths on both sides are normalized so a reviewer's // "./src/x.ts" matches the canonical "src/x.ts" diff key. function scopeFindings(survivors: Finding[], input: AggregateInput): Finding[] { + const inactiveReason = + input.policyInactive?.["scope.diff"] ?? + (input.scopeToDiff === false + ? "configured-off" + : input.changedRanges === undefined + ? "stage-precondition-miss" + : undefined); + if (inactiveReason !== undefined) { + input.policyRuntime?.markInactive("scope.diff", inactiveReason); + return survivors; + } const enabled = input.scopeToDiff !== false && input.changedRanges !== undefined; if (!enabled && input.policyRuntime === undefined) return survivors; const normalizedRanges = new Map(); @@ -689,6 +704,12 @@ export function aggregate(input: AggregateInput): AggregateResult { : deduped; const critic = input.critic; + const criticInactiveReason = + input.policyInactive?.["judgment.critic"] ?? + (critic === undefined ? "stage-precondition-miss" : undefined); + if (criticInactiveReason !== undefined) { + input.policyRuntime?.markInactive("judgment.critic", criticInactiveReason); + } const criticAblated = input.policyRuntime?.isAblated("judgment.critic") ?? false; // #4: a BLOCKING finding whose every contributing base provider is high-precision is // protected from the SOFT demoters. Never protects a self_refuted (T1) or INFO finding — @@ -703,6 +724,10 @@ export function aggregate(input: AggregateInput): AggregateResult { const survivors: Finding[] = []; const criticDropped: Finding[] = []; for (const f of taggedFindings) { + if (criticInactiveReason !== undefined) { + survivors.push(f); + continue; + } // Scan the representative AND every merged member signature (mirror the // fp_ledger_match pass): the critic may have keyed its verdict on a member's // signature, not the promoted representative's — checking only f.signature @@ -820,67 +845,60 @@ export function aggregate(input: AggregateInput): AggregateResult { // blocking; §4.3 pinned recurrences stay; inert when no deltaScope was computed // (missing/corrupt snapshot, iteration 1, one-shot mode, incomplete diff). const deltaScope = input.deltaScope; + const deltaInactiveReason = + input.policyInactive?.["scope.delta"] ?? + (deltaScope === null || deltaScope === undefined ? "stage-precondition-miss" : undefined); + if (deltaInactiveReason !== undefined) { + input.policyRuntime?.markInactive("scope.delta", deltaInactiveReason); + } const deltaScoped: Finding[] = - deltaScope !== null && deltaScope !== undefined - ? scoped.map((f) => { - const opportunity = f.severity !== "INFO"; - const matched = opportunity && !deltaScope.has(normalizeRepoPath(f.file)); - let protectedBy: PolicyProtectionCode | undefined; - if (matched && f.claimed_fixed_recurred) { - protectedBy = "claimed-fixed-pin"; - } else if (matched && touchesSecurityOrCorrectness(f)) { - protectedBy = "security-correctness-floor"; - } else if (matched && f.demoted_from_critical === true) { - // G0 alignment: a from-CRITICAL WARN remains decision-required. - protectedBy = "critical-floor"; - } else if (matched) { - // Honor the same cross-file escape hatch as diff/session scope. - const memberCats = [f.category, ...(f.members?.map((m) => m.category) ?? [])]; - const hatch = new Set(input.outOfDiffBlocking ?? []); - if (memberCats.some((category) => hatch.has(category))) { - protectedBy = "out-of-diff-blocking-hatch"; + deltaInactiveReason !== undefined + ? scoped + : deltaScope !== null && deltaScope !== undefined + ? scoped.map((f) => { + const opportunity = f.severity !== "INFO"; + const matched = opportunity && !deltaScope.has(normalizeRepoPath(f.file)); + let protectedBy: PolicyProtectionCode | undefined; + if (matched && f.claimed_fixed_recurred) { + protectedBy = "claimed-fixed-pin"; + } else if (matched && touchesSecurityOrCorrectness(f)) { + protectedBy = "security-correctness-floor"; + } else if (matched && f.demoted_from_critical === true) { + // G0 alignment: a from-CRITICAL WARN remains decision-required. + protectedBy = "critical-floor"; + } else if (matched) { + // Honor the same cross-file escape hatch as diff/session scope. + const memberCats = [f.category, ...(f.members?.map((m) => m.category) ?? [])]; + const hatch = new Set(input.outOfDiffBlocking ?? []); + if (memberCats.some((category) => hatch.has(category))) { + protectedBy = "out-of-diff-blocking-hatch"; + } } - } - return ( - transitionFinding({ - ...(input.policyRuntime === undefined ? {} : { runtime: input.policyRuntime }), - passId: "scope.delta", - finding: f, - opportunity, - matched, - reasonCode: "outside-delta-scope", - action: "demoted", - ...(protectedBy === undefined ? {} : { protectedBy }), - sourceSignatures: sourceSignatures(f), - proposed: () => { - const note = - "\n\n↓ on content already reviewed in an earlier iteration and unchanged since — advisory only (delta scope)."; - return { - ...f, - severity: "INFO" as const, - delta_scope_demoted: true, - details: `${f.details.slice(0, 2000 - note.length)}${note}`, - }; - }, - }) ?? f - ); - }) - : input.policyRuntime - ? scoped.map( - (f) => + return ( transitionFinding({ ...(input.policyRuntime === undefined ? {} : { runtime: input.policyRuntime }), passId: "scope.delta", finding: f, - opportunity: false, - matched: false, + opportunity, + matched, reasonCode: "outside-delta-scope", action: "demoted", + ...(protectedBy === undefined ? {} : { protectedBy }), sourceSignatures: sourceSignatures(f), - proposed: () => f, - }) ?? f, - ) + proposed: () => { + const note = + "\n\n↓ on content already reviewed in an earlier iteration and unchanged since — advisory only (delta scope)."; + return { + ...f, + severity: "INFO" as const, + delta_scope_demoted: true, + details: `${f.details.slice(0, 2000 - note.length)}${note}`, + }; + }, + }) ?? f + ); + }) : scoped; // Slice A (P1) — session-ownership demote. A blocking finding on a file FOREIGN to this @@ -893,68 +911,61 @@ export function aggregate(input: AggregateInput): AggregateResult { // foreign (still tagged, so the agent can dispose it via an out-of-scope decision). Done as // an INDEPENDENT pass (not inside scopeFindings, which early-returns when scopeToDiff is off). const foreignFiles = input.foreignFiles; + const sessionInactiveReason = + input.policyInactive?.["scope.session"] ?? + (!foreignFiles || foreignFiles.size === 0 ? "stage-precondition-miss" : undefined); + if (sessionInactiveReason !== undefined) { + input.policyRuntime?.markInactive("scope.session", sessionInactiveReason); + } const sessionAblated = input.policyRuntime?.isAblated("scope.session") ?? false; const foreignScoped: Finding[] = - foreignFiles && foreignFiles.size > 0 - ? deltaScoped.map((f) => { - const opportunity = f.severity !== "INFO"; - const isForeign = foreignFiles.has(normalizeRepoPath(f.file)); - const matched = opportunity && isForeign; - const categories = [f.category, ...(f.members?.map((m) => m.category) ?? [])]; - const blocking = new Set(input.outOfDiffBlocking ?? []); - const protectedBy = - matched && categories.some((category) => blocking.has(category)) - ? "out-of-diff-blocking-hatch" - : undefined; - const transitioned = - transitionFinding({ - ...(input.policyRuntime === undefined ? {} : { runtime: input.policyRuntime }), - passId: "scope.session", - finding: f, - opportunity, - matched, - reasonCode: "foreign-to-session", - action: "demoted", - ...(protectedBy === undefined ? {} : { protectedBy }), - sourceSignatures: sourceSignatures(f), - proposed: () => { - const note = - "\n\n↓ on a file this session did not author (parallel agent / pre-existing) — advisory only."; - return { - ...f, - severity: "INFO" as const, - foreign_to_session: true, - details: `${f.details.slice(0, 2000 - note.length)}${note}`, - }; - }, - }) ?? f; - // The legacy INFO marker is explanatory rather than a blocking policy - // transition. A protected blocking finding is likewise tagged so the - // out-of-scope disposition remains available. - if ( - !sessionAblated && - isForeign && - (f.severity === "INFO" || protectedBy !== undefined) - ) { - return { ...transitioned, foreign_to_session: true }; - } - return transitioned; - }) - : input.policyRuntime - ? deltaScoped.map( - (f) => + sessionInactiveReason !== undefined + ? deltaScoped + : foreignFiles && foreignFiles.size > 0 + ? deltaScoped.map((f) => { + const opportunity = f.severity !== "INFO"; + const isForeign = foreignFiles.has(normalizeRepoPath(f.file)); + const matched = opportunity && isForeign; + const categories = [f.category, ...(f.members?.map((m) => m.category) ?? [])]; + const blocking = new Set(input.outOfDiffBlocking ?? []); + const protectedBy = + matched && categories.some((category) => blocking.has(category)) + ? "out-of-diff-blocking-hatch" + : undefined; + const transitioned = transitionFinding({ ...(input.policyRuntime === undefined ? {} : { runtime: input.policyRuntime }), passId: "scope.session", finding: f, - opportunity: false, - matched: false, + opportunity, + matched, reasonCode: "foreign-to-session", action: "demoted", + ...(protectedBy === undefined ? {} : { protectedBy }), sourceSignatures: sourceSignatures(f), - proposed: () => f, - }) ?? f, - ) + proposed: () => { + const note = + "\n\n↓ on a file this session did not author (parallel agent / pre-existing) — advisory only."; + return { + ...f, + severity: "INFO" as const, + foreign_to_session: true, + details: `${f.details.slice(0, 2000 - note.length)}${note}`, + }; + }, + }) ?? f; + // The legacy INFO marker is explanatory rather than a blocking policy + // transition. A protected blocking finding is likewise tagged so the + // out-of-scope disposition remains available. + if ( + !sessionAblated && + isForeign && + (f.severity === "INFO" || protectedBy !== undefined) + ) { + return { ...transitioned, foreign_to_session: true }; + } + return transitioned; + }) : deltaScoped; // M5 Part B1 — reactive FP-ledger demote: a finding whose representative diff --git a/src/core/critic.ts b/src/core/critic.ts index dc71d2c..de63788 100644 --- a/src/core/critic.ts +++ b/src/core/critic.ts @@ -13,6 +13,9 @@ export interface CriticVerdict { export interface CriticRunResult { map: Map; info: { provider: string; status: "ran" | "error" | "empty" | "misconfigured"; verdicts: number }; + /** Ordered hashes of every completion text that actually returned. */ + rawResponseSha256s?: string[]; + /** Last returned response hash, retained for additive caller compatibility. */ rawResponseSha256?: string; } @@ -62,6 +65,7 @@ export async function runCritic( const prompt = buildCriticPrompt(findings); let finalStatus: "error" | "empty" = "empty"; let rawResponseSha256: string | undefined; + const rawResponseSha256s: string[] = []; for (let attempt = 1; attempt <= attemptLimit; attempt++) { try { // Force reasoning OFF: the critic is a keep/demote classification that needs @@ -71,11 +75,13 @@ export async function runCritic( // flag ignore it. const text = await adapter.complete(prompt, { ...opts, disableReasoning: true }); rawResponseSha256 = createHash("sha256").update(Buffer.from(text, "utf8")).digest("hex"); + rawResponseSha256s.push(rawResponseSha256); const map = parseCriticOutput(text); if (map.size > 0) { return { map, info: { provider, status: "ran", verdicts: map.size }, + rawResponseSha256s, rawResponseSha256, }; } @@ -87,6 +93,7 @@ export async function runCritic( return { map: new Map(), info: { provider, status: finalStatus, verdicts: 0 }, + ...(rawResponseSha256s.length === 0 ? {} : { rawResponseSha256s }), ...(rawResponseSha256 === undefined ? {} : { rawResponseSha256 }), }; } diff --git a/src/core/orchestrator.ts b/src/core/orchestrator.ts index a3ce012..e06143c 100644 --- a/src/core/orchestrator.ts +++ b/src/core/orchestrator.ts @@ -63,7 +63,7 @@ import { withTimeout } from "../utils/with-timeout.ts"; import { RG_VERSION } from "../version.ts"; import { type Adjudication, renderAdjudications } from "./adjudications.ts"; import { recurrenceNotesForFindings } from "./agent-lessons/recurrence.ts"; -import { aggregate } from "./aggregator.ts"; +import { type AggregateInput, aggregate } from "./aggregator.ts"; import { CandidateStore } from "./brain/candidate-store.ts"; import { runCurator } from "./brain/curator.ts"; import type { EmbedOptions, Embedder } from "./brain/embeddings.ts"; @@ -2331,11 +2331,17 @@ export class Orchestrator { ); if (groundingResponseSha256 !== undefined) { rawResponseSha256.push(groundingResponseSha256); + groundedFindings = applyGroundingJudgeVerdicts(groundedFindings, map, policyRuntime); + } else { + policyRuntime?.markInactive("judgment.grounding-llm", "stage-precondition-miss"); } - groundedFindings = applyGroundingJudgeVerdicts(groundedFindings, map, policyRuntime); + } else { + policyRuntime?.markInactive("judgment.grounding-llm", "stage-precondition-miss"); } } else if (groundingCfg === null || groundingCfg === undefined) { policyRuntime?.markInactive("judgment.grounding-llm", "configured-off"); + } else { + policyRuntime?.markInactive("judgment.grounding-llm", "stage-precondition-miss"); } // --- Optional critic phase (demote-only) --- @@ -2354,6 +2360,7 @@ export class Orchestrator { // (no usage envelope). Kept as a named field for the IterationResult shape. const criticCostUsd = 0; const criticCfg = this.input.config.phases.critic; + let criticAttempted = false; if (criticCfg && groundedFindings.length > 0) { const criticAdapter = this.input.adapters[criticCfg.provider]; const cProviderCfg = this.input.config.providers[criticCfg.provider] as @@ -2376,6 +2383,7 @@ export class Orchestrator { // review() — review() forces REVIEW_OUTPUT_SCHEMA on codex/openrouter/ollama // and makes the critic a silent no-op. No cost is attributed: complete() // returns only text (no usage envelope), so the critic phase is $0 here. + criticAttempted = true; const r = await runCritic( criticAdapter, criticCfg.provider, @@ -2393,9 +2401,13 @@ export class Orchestrator { groundedFindings, this.input.criticMaxAttempts ?? 1, ); - criticMap = r.map.size > 0 ? r.map : undefined; + criticMap = r.map; criticInfo = r.info; - if (r.rawResponseSha256 !== undefined) rawResponseSha256.push(r.rawResponseSha256); + if (r.rawResponseSha256s !== undefined) { + rawResponseSha256.push(...r.rawResponseSha256s); + } else if (r.rawResponseSha256 !== undefined) { + rawResponseSha256.push(r.rawResponseSha256); + } } else { criticInfo = { provider: criticCfg.provider, status: "misconfigured", verdicts: 0 }; } @@ -2498,6 +2510,29 @@ export class Orchestrator { }) : undefined; + const policyInactive: NonNullable = {}; + if (criticCfg === null || criticCfg === undefined) { + policyInactive["judgment.critic"] = "configured-off"; + } else if (!criticAttempted) { + policyInactive["judgment.critic"] = "stage-precondition-miss"; + } + if (this.input.config.phases.review.scopeToDiff === false) { + policyInactive["scope.diff"] = "configured-off"; + } else if (this.input.reportMode === "one-shot") { + policyInactive["scope.diff"] = "stage-precondition-miss"; + } + if (this.input.config.phases.review.deltaReview === false) { + policyInactive["scope.delta"] = "configured-off"; + } else if (deltaScope === null) { + policyInactive["scope.delta"] = "stage-precondition-miss"; + } + if (!this.input.foreignFiles || this.input.foreignFiles.size === 0) { + policyInactive["scope.session"] = + this.input.config.phases.review.scopeToSession === false + ? "configured-off" + : "stage-precondition-miss"; + } + const agg = aggregate({ findings: groundedFindings, // Distinct reviewer identities, NOT raw slot count: collapsed fallbacks @@ -2537,6 +2572,7 @@ export class Orchestrator { // value feeds regionsSegment in the cache key above. ...(activeRegions ? { rejectedRegions: activeRegions } : {}), ...(policyRuntime === undefined ? {} : { policyRuntime }), + ...(policyRuntime === undefined ? {} : { policyInactive }), }); // Include critic-DROPPED likely_fp findings (INFO → drop): they never reach @@ -2740,7 +2776,7 @@ export class Orchestrator { const policyTrace = policyRuntime?.finalize({ rawResponseSha256, verdict: agg.verdict, - finalFindings: reportFindings, + finalFindings, }); // Banner data for invalid/broad/zero-match entries + the render-budget drop // count — render-only (report-writer.ts), never affects the verdict. diff --git a/src/schemas/policy-trace.ts b/src/schemas/policy-trace.ts index 3d57f5e..2b8cedf 100644 --- a/src/schemas/policy-trace.ts +++ b/src/schemas/policy-trace.ts @@ -42,6 +42,7 @@ const NON_MATERIAL_REASON_CODES = new Set([ "stage-precondition-miss", PASS_ERROR_REASON_CODE, ]); +const LORE_FINAL_SIGNATURE = /^lore:(?:reminder|canon-promotion):[a-z0-9][a-z0-9-]*$/; const VERDICT_BY_REASON = { "hard-critical": "FAIL", "corroborated-warn": "FAIL", @@ -716,6 +717,34 @@ export const PolicyTraceSchema = PolicyTraceObjectSchema.superRefine((trace, ctx let priorEvaluationOrder = -1; const evaluationsByPass = new Map(); const finalSignatures = new Set(trace.final.finding_signatures); + const firstLoreIndex = trace.final.finding_signatures.findIndex((signature) => + LORE_FINAL_SIGNATURE.test(signature), + ); + const policyFinalSignatures = + firstLoreIndex < 0 + ? trace.final.finding_signatures + : trace.final.finding_signatures.slice(0, firstLoreIndex); + const policyFinalSignatureSet = new Set(policyFinalSignatures); + if (firstLoreIndex >= 0) { + for (let index = firstLoreIndex; index < trace.final.finding_signatures.length; index += 1) { + const signature = trace.final.finding_signatures[index]; + const severity = trace.final.finding_severities[index]?.severity; + if (signature === undefined || !LORE_FINAL_SIGNATURE.test(signature)) { + addIssue( + ctx, + ["final", "finding_signatures", index], + "server-owned Lore findings must form one closed suffix", + ); + } + if (severity !== "INFO") { + addIssue( + ctx, + ["final", "finding_severities", index, "severity"], + "server-owned Lore findings must remain INFO", + ); + } + } + } for (const [index, evaluation] of trace.evaluations.entries()) { if (evaluation.order < priorEvaluationOrder) { addIssue(ctx, ["evaluations", index, "order"], "evaluations must remain in catalog order"); @@ -740,6 +769,16 @@ export const PolicyTraceSchema = PolicyTraceObjectSchema.superRefine((trace, ctx ) { addIssue(ctx, ["evaluations", index, "final_signature"], "unknown final signature"); } + if ( + evaluation.final_signature !== undefined && + LORE_FINAL_SIGNATURE.test(evaluation.final_signature) + ) { + addIssue( + ctx, + ["evaluations", index, "final_signature"], + "Lore findings cannot mask policy evaluation lineage", + ); + } } for (const [index, summary] of trace.passes.entries()) { @@ -798,12 +837,26 @@ export const PolicyTraceSchema = PolicyTraceObjectSchema.superRefine((trace, ctx } priorStageOrder = stage.order; if (stage.stage_id === "aggregation.cluster" && stage.output_signature !== undefined) { + if (LORE_FINAL_SIGNATURE.test(stage.output_signature)) { + addIssue( + ctx, + ["stages", index, "output_signature"], + "Lore findings cannot be policy cluster outputs", + ); + } if (clusterOutputSet.has(stage.output_signature)) { addIssue(ctx, ["stages", index, "output_signature"], "duplicate cluster output"); } clusterOutputs.push(stage.output_signature); clusterOutputSet.add(stage.output_signature); for (const [inputIndex, input] of stage.input_signatures.entries()) { + if (LORE_FINAL_SIGNATURE.test(input)) { + addIssue( + ctx, + ["stages", index, "input_signatures", inputIndex], + "Lore findings cannot be policy cluster inputs", + ); + } if (clusterOutputByInput.has(input)) { addIssue( ctx, @@ -830,7 +883,7 @@ export const PolicyTraceSchema = PolicyTraceObjectSchema.superRefine((trace, ctx addIssue(ctx, ["stages"], "a complete trace requires exactly one verdict.compute row"); } - if (!isOrderedSubsequence(trace.final.finding_signatures, clusterOutputs)) { + if (!isOrderedSubsequence(policyFinalSignatures, clusterOutputs)) { addIssue(ctx, ["stages"], "final finding signatures must preserve cluster output order"); } @@ -861,7 +914,7 @@ export const PolicyTraceSchema = PolicyTraceObjectSchema.superRefine((trace, ctx const [output] = lineageOutputs; if (output === undefined) continue; const appliedDrop = evaluation.result === "applied" && evaluation.after === null; - const survives = finalSignatures.has(output); + const survives = policyFinalSignatureSet.has(output); if (appliedDrop) { if (survives) { @@ -899,7 +952,7 @@ export const PolicyTraceSchema = PolicyTraceObjectSchema.superRefine((trace, ctx } for (const [index, signature] of clusterOutputs.entries()) { - if (!finalSignatures.has(signature) && !droppedOutputs.has(signature)) { + if (!policyFinalSignatureSet.has(signature) && !droppedOutputs.has(signature)) { addIssue(ctx, ["stages", index], "a non-final cluster output requires a later applied drop"); } } diff --git a/tests/integration/policy-trace-equivalence.test.ts b/tests/integration/policy-trace-equivalence.test.ts index 4d5a008..8a28be6 100644 --- a/tests/integration/policy-trace-equivalence.test.ts +++ b/tests/integration/policy-trace-equivalence.test.ts @@ -102,6 +102,7 @@ async function run( withAudit = false, reviewOverrides: Partial = {}, reportMode: OrchestratorInput["reportMode"] = "gate", + grounding: ReviewgateConfig["phases"]["grounding"] = null, ): Promise<{ repo: string; calls: string[]; @@ -134,7 +135,7 @@ async function run( brain: null, critic: null, fpLedger: null, - grounding: null, + grounding, implicitOutcomes: null, lore: null, triage: null, @@ -254,35 +255,44 @@ describe("policy trace lifecycle equivalence", () => { writeFileSync(join(repo, "a.ts"), "export const value = 1;\n"); const reviewerRaw = '{"verdict":"FAIL","findings":[{"source":"reviewer"}]}'; const groundingRaw = '{"verdicts":[{"signature":"ordered-signature","grounded":true}]}'; - const criticRaw = '{"verdicts":[{"signature":"ordered-signature","verdict":"keep"}]}'; + const criticRaw = ["not critic json", '{"verdicts":[]}']; const critical: Finding = { ...finding("codex"), signature: "ordered-signature", severity: "CRITICAL", message: "The changed `value` needs review", }; - const completionAdapter = (id: "gemini" | "opencode", response: string): ProviderAdapter => ({ - id, - async preflight() { - return { available: true, version: "fixture", authMode: "oauth", error: null }; - }, - async review(input) { - return { - reviewerId: input.reviewerId, - verdict: "PASS", - findings: [], - usage: { inputTokens: 0, outputTokens: 0, costUsd: 0, quotaUsedPct: null }, - durationMs: 1, - exitCode: 0, - rawEventsPath: "", - rawText: "", - status: "ok", - } satisfies ReviewResult; - }, - async complete() { - return response; - }, - }); + const completionAdapter = ( + id: "gemini" | "opencode", + response: string | readonly string[], + ): ProviderAdapter => { + let calls = 0; + return { + id, + async preflight() { + return { available: true, version: "fixture", authMode: "oauth", error: null }; + }, + async review(input) { + return { + reviewerId: input.reviewerId, + verdict: "PASS", + findings: [], + usage: { inputTokens: 0, outputTokens: 0, costUsd: 0, quotaUsedPct: null }, + durationMs: 1, + exitCode: 0, + rawEventsPath: "", + rawText: "", + status: "ok", + } satisfies ReviewResult; + }, + async complete() { + if (typeof response === "string") return response; + const value = response[calls] ?? ""; + calls += 1; + return value; + }, + }; + }; const reviewer: ProviderAdapter = { id: "codex", async preflight() { @@ -340,6 +350,7 @@ describe("policy trace lifecycle equivalence", () => { diff: DIFF, reasonOnFailEnabled: true, disableLastResortFailover: true, + criticMaxAttempts: 2, policyExecution: { trace: "memory", policyAblations: new Set(), @@ -355,8 +366,11 @@ describe("policy trace lifecycle equivalence", () => { expect(result.policyTrace?.raw_response_sha256).toEqual([ sha256(reviewerRaw), sha256(groundingRaw), - sha256(criticRaw), + ...criticRaw.map(sha256), ]); + expect( + result.policyTrace?.passes.find((pass) => pass.pass_id === "judgment.critic")?.status, + ).toBe("ran"); }); it("marks configured-inactive pre-aggregation passes not-run without evaluations", async () => { @@ -374,6 +388,36 @@ describe("policy trace lifecycle equivalence", () => { expect(status.get("evidence.self-refutation")).toEqual(["not-run", "configured-off"]); expect(status.get("judgment.hypothetical")).toEqual(["not-run", "configured-off"]); expect(status.get("judgment.grounding-llm")).toEqual(["not-run", "configured-off"]); + expect(status.get("judgment.critic")).toEqual(["not-run", "configured-off"]); + expect(status.get("scope.delta")).toEqual(["not-run", "stage-precondition-miss"]); + expect(status.get("scope.session")).toEqual(["not-run", "stage-precondition-miss"]); + }); + + it("distinguishes configured-off scopes from stage-precondition misses", async () => { + const configuredOff = await run("memory", false, { + scopeToDiff: false, + deltaReview: false, + scopeToSession: false, + }); + const noCritical = await run("memory", false, {}, "gate", { provider: "gemini" }); + const configuredOffStatus = new Map( + configuredOff.result.policyTrace?.passes.map((pass) => [ + pass.pass_id, + [pass.status, "reason_code" in pass ? pass.reason_code : undefined], + ]), + ); + const grounding = noCritical.result.policyTrace?.passes.find( + (pass) => pass.pass_id === "judgment.grounding-llm", + ); + + expect(configuredOffStatus.get("scope.diff")).toEqual(["not-run", "configured-off"]); + expect(configuredOffStatus.get("scope.delta")).toEqual(["not-run", "configured-off"]); + expect(configuredOffStatus.get("scope.session")).toEqual(["not-run", "configured-off"]); + expect(grounding).toEqual({ + pass_id: "judgment.grounding-llm", + status: "not-run", + reason_code: "stage-precondition-miss", + }); }); it("preserves the exact production demotion when recorder validation fails", async () => { diff --git a/tests/unit/critic-runner.test.ts b/tests/unit/critic-runner.test.ts index 35d3e35..c12e75a 100644 --- a/tests/unit/critic-runner.test.ts +++ b/tests/unit/critic-runner.test.ts @@ -5,6 +5,7 @@ // model can only emit {verdict,findings} and never the critic's {verdicts:[...]} // shape → parseCriticOutput sees nothing → a silent no-op (zero demotions). import { describe, expect, it } from "bun:test"; +import { createHash } from "node:crypto"; import { runCritic } from "../../src/core/critic.ts"; import type { CompleteOptions, ProviderAdapter } from "../../src/providers/adapter-base.ts"; import type { Finding } from "../../src/schemas/finding.ts"; @@ -30,6 +31,10 @@ function mkFinding(over: Partial = {}): Finding { const OPTS: CompleteOptions = { model: "m" }; +function sha256(text: string): string { + return createHash("sha256").update(Buffer.from(text, "utf8")).digest("hex"); +} + describe("runCritic", () => { it("returns only the SHA-256 of the successful raw critic response", async () => { const raw = '{"verdicts":[{"signature":"sig-hash","verdict":"keep"}]}'; @@ -136,18 +141,27 @@ describe("runCritic", () => { it("retries empty/unparseable output and stops at the first non-empty verdict map", async () => { let calls = 0; + const responses = [ + "not critic json", + JSON.stringify({ verdicts: [{ signature: "sig-1", verdict: "likely_fp" }] }), + ]; const adapter: Pick = { complete: async () => { calls++; - return calls === 1 - ? "not critic json" - : JSON.stringify({ verdicts: [{ signature: "sig-1", verdict: "likely_fp" }] }); + return responses[calls - 1] ?? ""; }, }; - const { map, info } = await runCritic(adapter, "openrouter", OPTS, [mkFinding()], 3); + const { map, info, rawResponseSha256s } = await runCritic( + adapter, + "openrouter", + OPTS, + [mkFinding()], + 3, + ); expect(calls).toBe(2); expect(info.status).toBe("ran"); expect(map.get("sig-1")?.verdict).toBe("likely_fp"); + expect(rawResponseSha256s).toEqual(responses.map(sha256)); }); it("does not spend a retry after the first parseable non-empty verdict map", async () => { diff --git a/tests/unit/orchestrator-lore.test.ts b/tests/unit/orchestrator-lore.test.ts index 9276627..cd57689 100644 --- a/tests/unit/orchestrator-lore.test.ts +++ b/tests/unit/orchestrator-lore.test.ts @@ -191,6 +191,7 @@ function orch( repo: string, adapter: ProviderAdapter, loreConfig: Record = { enabled: true }, + trace = false, ) { return new Orchestrator({ repoRoot: repo, @@ -206,6 +207,15 @@ function orch( hostTier: "opus", diff, reasonOnFailEnabled: true, + ...(trace + ? { + policyExecution: { + trace: "memory" as const, + policyAblations: new Set(), + authoritative: false, + }, + } + : {}), }); } @@ -541,6 +551,39 @@ describe("orchestrator lore integration", () => { expect(promos[0].severity).toBe("INFO"); }); + it("binds trace final identity to panel and additive lore findings", async () => { + const repo = initRepo(); + writeFileSync(join(repo, "foo.ts"), "content"); + commitAll(repo); + writeLoreEntry(repo, { + id: "traced-canon-entry", + status: "canon", + anchors: ["nonexistent-file.ts"], + verifiedTree: "irrelevant", + body: "This unapproved canon entry must remain visible in authoritative trace identity.", + }); + + const state = { calls: 0 }; + const res = await orch(repo, warnFindingStub(state), { enabled: true }, true).runIteration({ + runId: "R-TRACE-LORE", + iter: 1, + loreReminderBudget: { allowed: false, cooldownIds: [] }, + }); + const pending = JSON.parse(readFileSync(pendingJsonPath(repo), "utf8")); + + expect(res.policyTrace?.final.finding_signatures).toEqual( + pending.findings.map((finding: Finding) => finding.signature), + ); + expect(res.policyTrace?.final.finding_severities).toEqual( + pending.findings.map((finding: Finding) => ({ + signature: finding.signature, + severity: finding.severity, + })), + ); + expect(res.policyTrace?.final.counts).toEqual(pending.counts); + expect(res.summary.counts).toEqual(pending.counts); + }); + it("(g) a CRITICAL panel finding suppresses the reminder but NOT the canon-promotion guard", async () => { const repo = initRepo(); writeFileSync(join(repo, "foo.ts"), "content v1"); diff --git a/tests/unit/policy-aggregator-first-half.test.ts b/tests/unit/policy-aggregator-first-half.test.ts index bbb763a..e9dff12 100644 --- a/tests/unit/policy-aggregator-first-half.test.ts +++ b/tests/unit/policy-aggregator-first-half.test.ts @@ -92,6 +92,49 @@ const ABLATED_BLOCKING_PRESERVED = [1, 1, 1, 0, 0, 0, 1, 0] as const; const PROTECTED_BLOCKING_PRESERVED = [1, 1, 1, 0, 1, 0, 1, 0] as const; describe("aggregator policy numeric contracts, orders 60-100", () => { + it("marks fully inactive Critic and scope passes not-run without evaluations", () => { + const recorder = runtime("inactive-first-half"); + aggregate({ + findings: [finding()], + reviewersTotal: 1, + policyRuntime: recorder, + policyInactive: { + "judgment.critic": "configured-off", + "scope.diff": "configured-off", + "scope.delta": "stage-precondition-miss", + "scope.session": "stage-precondition-miss", + }, + }); + + expect(recorder.summary("judgment.critic")).toEqual({ + pass_id: "judgment.critic", + status: "not-run", + reason_code: "configured-off", + }); + expect(recorder.summary("scope.diff")).toEqual({ + pass_id: "scope.diff", + status: "not-run", + reason_code: "configured-off", + }); + expect(recorder.summary("scope.delta")).toEqual({ + pass_id: "scope.delta", + status: "not-run", + reason_code: "stage-precondition-miss", + }); + expect(recorder.summary("scope.session")).toEqual({ + pass_id: "scope.session", + status: "not-run", + reason_code: "stage-precondition-miss", + }); + expect( + recorder + .evaluations() + .filter((row) => + ["judgment.critic", "scope.diff", "scope.delta", "scope.session"].includes(row.pass_id), + ), + ).toEqual([]); + }); + it("records redaction no-opportunity, miss, active, ablated, and protected tuples", () => { const info = run("redaction-info", { findings: [ diff --git a/tests/unit/policy-trace-schema.test.ts b/tests/unit/policy-trace-schema.test.ts index cdbb6e9..730ee0d 100644 --- a/tests/unit/policy-trace-schema.test.ts +++ b/tests/unit/policy-trace-schema.test.ts @@ -970,6 +970,124 @@ describe("PolicyTraceSchema", () => { ).toBe(false); }); + it("accepts only an additive INFO Lore suffix outside policy cluster lineage", () => { + const trace = traceWithSingleFinal("INFO"); + const loreSignature = "lore:canon-promotion:entry-one"; + const final = { + ...trace.final, + counts: { critical: 0, warn: 0, info: 2 }, + finding_signatures: [...trace.final.finding_signatures, loreSignature], + finding_severities: [ + ...trace.final.finding_severities, + { signature: loreSignature, severity: "INFO" as const }, + ], + }; + + expect(PolicyTraceSchema.safeParse({ ...trace, final }).success).toBe(true); + }); + + it("rejects arbitrary, blocking, interleaved, and unknown-kind Lore-like finals", () => { + const trace = traceWithWarnAndInfoFinals(); + const suffix = (signature: string, severity: "WARN" | "INFO" = "INFO") => ({ + ...trace.final, + counts: { + critical: 0, + warn: trace.final.counts.warn + (severity === "WARN" ? 1 : 0), + info: trace.final.counts.info + (severity === "INFO" ? 1 : 0), + }, + finding_signatures: [...trace.final.finding_signatures, signature], + finding_severities: [...trace.final.finding_severities, { signature, severity }], + }); + + expect( + PolicyTraceSchema.safeParse({ ...trace, final: suffix("arbitrary-extra") }).success, + ).toBe(false); + expect( + PolicyTraceSchema.safeParse({ + ...trace, + final: suffix("lore:reminder:entry-one", "WARN"), + }).success, + ).toBe(false); + expect( + PolicyTraceSchema.safeParse({ + ...trace, + final: { + ...suffix("lore:reminder:entry-one"), + finding_signatures: ["sig-info", "lore:reminder:entry-one", "sig-warn"], + finding_severities: [ + { signature: "sig-info", severity: "INFO" }, + { signature: "lore:reminder:entry-one", severity: "INFO" }, + { signature: "sig-warn", severity: "WARN" }, + ], + }, + }).success, + ).toBe(false); + expect( + PolicyTraceSchema.safeParse({ + ...trace, + final: suffix("lore:unknown-kind:entry-one"), + }).success, + ).toBe(false); + }); + + it("rejects Lore signatures used as evaluation lineage or cluster-output masks", () => { + const trace = traceWithSingleFinal("INFO"); + const loreSignature = "lore:reminder:entry-one"; + const withLore = { + ...trace, + final: { + ...trace.final, + counts: { critical: 0, warn: 0, info: 2 }, + finding_signatures: [...trace.final.finding_signatures, loreSignature], + finding_severities: [ + ...trace.final.finding_severities, + { signature: loreSignature, severity: "INFO" as const }, + ], + }, + }; + + expect( + PolicyTraceSchema.safeParse({ + ...withLore, + evaluations: withLore.evaluations.map((evaluation, index) => + index === 0 ? { ...evaluation, final_signature: loreSignature } : evaluation, + ), + }).success, + ).toBe(false); + expect( + PolicyTraceSchema.safeParse({ + ...withLore, + stages: [ + ...withLore.stages.slice(0, -1), + { + stage_id: "aggregation.cluster", + order: 65, + reason_code: "singleton", + member_count: 1, + input_signatures: [loreSignature], + output_signature: loreSignature, + }, + withLore.stages.at(-1), + ], + }).success, + ).toBe(false); + expect( + PolicyTraceSchema.safeParse({ + ...withLore, + stages: withLore.stages.map((stage) => + stage.stage_id === "aggregation.cluster" + ? { + ...stage, + reason_code: "clustered", + member_count: 2, + input_signatures: ["sig-a", loreSignature], + } + : stage, + ), + }).success, + ).toBe(false); + }); + it("cross-checks applied and would-apply evaluations against the ablation set", () => { const applied = traceWithSingleFinal("INFO"); const appliedSummary = { From 66f0a6cab64f5a55908e1e8ba918cb71528380dd Mon Sep 17 00:00:00 2001 From: Codevena Date: Mon, 10 Aug 2026 09:05:00 +0200 Subject: [PATCH 35/55] feat(policy): persist and verify audit traces --- src/audit/canonical.ts | 8 + src/audit/logger.ts | 33 +- src/audit/policy-trace-store.ts | 148 +++++++++ src/audit/verifier.ts | 30 +- src/core/loop-driver.ts | 20 +- src/core/orchestrator.ts | 62 +++- src/core/report-writer.ts | 80 ++++- .../policy-trace-equivalence.test.ts | 25 +- tests/unit/audit-logger-retention.test.ts | 7 +- tests/unit/audit-logger.test.ts | 153 +++++++++- tests/unit/audit-verify-corruption.test.ts | 164 +++++++++- tests/unit/policy-trace-store.test.ts | 283 ++++++++++++++++++ tests/unit/report-writer.test.ts | 229 +++++++++++++- 13 files changed, 1197 insertions(+), 45 deletions(-) create mode 100644 src/audit/canonical.ts create mode 100644 src/audit/policy-trace-store.ts create mode 100644 tests/unit/policy-trace-store.test.ts diff --git a/src/audit/canonical.ts b/src/audit/canonical.ts new file mode 100644 index 0000000..dda3149 --- /dev/null +++ b/src/audit/canonical.ts @@ -0,0 +1,8 @@ +/** Stable JSON-like serialization with object keys sorted at every level. */ +export function canonicalJson(value: unknown): string { + if (value === null || typeof value !== "object") return JSON.stringify(value); + if (Array.isArray(value)) return `[${value.map(canonicalJson).join(",")}]`; + const record = value as Record; + const keys = Object.keys(record).sort(); + return `{${keys.map((key) => `${JSON.stringify(key)}:${canonicalJson(record[key])}`).join(",")}}`; +} diff --git a/src/audit/logger.ts b/src/audit/logger.ts index 47f8bcf..5a78ede 100644 --- a/src/audit/logger.ts +++ b/src/audit/logger.ts @@ -4,19 +4,14 @@ import { appendFileSync, existsSync, mkdirSync, readdirSync, rmSync } from "node import { join } from "node:path"; import type { AuditEvent, EventType, Trigger } from "../schemas/audit-event.ts"; import { AuditEventSchema } from "../schemas/audit-event.ts"; +import type { PolicyTrace } from "../schemas/policy-trace.ts"; +import { canonicalJson } from "./canonical.ts"; +import { type PolicyTraceWriteResult, writePolicyTrace } from "./policy-trace-store.ts"; function sha256(s: string): string { return createHash("sha256").update(s).digest("hex"); } -function canonical(o: unknown): string { - // Stable stringify with sorted keys at every level. - if (o === null || typeof o !== "object") return JSON.stringify(o); - if (Array.isArray(o)) return `[${o.map(canonical).join(",")}]`; - const keys = Object.keys(o as Record).sort(); - return `{${keys.map((k) => `${JSON.stringify(k)}:${canonical((o as Record)[k])}`).join(",")}}`; -} - export type AuditEventInput = { event: EventType; run_id: string; @@ -27,6 +22,7 @@ export type AuditEventInput = { export class AuditLogger { private lastHash = ""; private filePath: string | null = null; + private partitionDate: Date | null = null; private pruned = false; // `retentionDays` enforces config's `audit.retentionDays` (previously declared @@ -43,6 +39,18 @@ export class AuditLogger { return this.filePath; } + writePolicyTrace(trace: PolicyTrace): PolicyTraceWriteResult { + try { + return writePolicyTrace({ + auditDir: this.auditDir, + trace, + now: this.selectedPartitionDate(), + }); + } catch { + return { status: "error" }; + } + } + // Prune whole day-partition directories (audit/YYYY/MM/DD) whose date is older // than `retentionDays` before today (UTC). Day-granularity matches the on-disk // layout written by computePath() and avoids rewriting hash-chained files (which @@ -83,7 +91,7 @@ export class AuditLogger { } private computePath(): string { - const now = new Date(); + const now = this.selectedPartitionDate(); const y = now.getUTCFullYear(); const m = String(now.getUTCMonth() + 1).padStart(2, "0"); const d = String(now.getUTCDate()).padStart(2, "0"); @@ -102,6 +110,11 @@ export class AuditLogger { return join(dir, `${stamp}-p${process.pid}-${randomBytes(16).toString("hex")}.jsonl`); } + private selectedPartitionDate(): Date { + if (this.partitionDate === null) this.partitionDate = new Date(); + return this.partitionDate; + } + async append(input: AuditEventInput): Promise { // Enforce retention before writing (once per logger lifetime). Pruning happens // ON write so the log can't grow forever between sessions without ever being @@ -116,7 +129,7 @@ export class AuditLogger { }; const forHash = { ...base }; (forHash as { this_event_hash?: unknown }).this_event_hash = undefined; - const h = sha256(canonical(forHash)); + const h = sha256(canonicalJson(forHash)); const event = AuditEventSchema.parse({ ...base, this_event_hash: h }); appendFileSync(this.currentFilePath(), `${JSON.stringify(event)}\n`, { mode: 0o600 }); this.lastHash = h; diff --git a/src/audit/policy-trace-store.ts b/src/audit/policy-trace-store.ts new file mode 100644 index 0000000..6000df7 --- /dev/null +++ b/src/audit/policy-trace-store.ts @@ -0,0 +1,148 @@ +import { createHash } from "node:crypto"; +import { existsSync, lstatSync, mkdirSync, readFileSync, realpathSync } from "node:fs"; +import { isAbsolute, join, relative, resolve } from "node:path"; +import { type PolicyTrace, PolicyTraceSchema } from "../schemas/policy-trace.ts"; +import { writeFileAtomic } from "../utils/atomic-write.ts"; +import { canonicalJson } from "./canonical.ts"; + +export const POLICY_TRACE_MAX_BYTES = 1_048_576; + +export type PolicyTraceWriteResult = + | { status: "complete"; ref: string; sha256: string } + | { status: "error" | "overflow" }; + +export type PolicyTraceVerification = + | { ok: true; trace: PolicyTrace } + | { + ok: false; + reason: + | "invalid-reference" + | "path-escape" + | "missing" + | "not-a-file" + | "hash-mismatch" + | "invalid-json" + | "invalid-trace" + | "non-canonical" + | "identity-mismatch" + | "read-error"; + }; + +export interface WritePolicyTraceInput { + auditDir: string; + trace: PolicyTrace; + maxBytes?: number; + now?: Date; +} + +export interface VerifyPolicyTraceReferenceInput { + auditDir: string; + ref: string; + sha256: string; +} + +const POLICY_REF = + /^(\d{4})\/(\d{2})\/(\d{2})\/policy\/([0-9a-f]{12})-i(0|[1-9]\d*)-([0-9a-f]{12})\.json$/; +const FULL_SHA256 = /^[0-9a-f]{64}$/; + +function sha256(value: string | Buffer): string { + return createHash("sha256").update(value).digest("hex"); +} + +function isContained(root: string, candidate: string): boolean { + const rel = relative(root, candidate); + return rel === "" || (!rel.startsWith("..") && !isAbsolute(rel)); +} + +function utcPartition(now: Date): { year: string; month: string; day: string } { + return { + year: String(now.getUTCFullYear()), + month: String(now.getUTCMonth() + 1).padStart(2, "0"), + day: String(now.getUTCDate()).padStart(2, "0"), + }; +} + +export function writePolicyTrace(input: WritePolicyTraceInput): PolicyTraceWriteResult { + try { + const trace = PolicyTraceSchema.parse(input.trace); + const canonical = canonicalJson(trace); + const byteLength = Buffer.byteLength(canonical, "utf8"); + const maxBytes = input.maxBytes ?? POLICY_TRACE_MAX_BYTES; + if (!Number.isSafeInteger(maxBytes) || maxBytes < 0) return { status: "error" }; + + // The complete canonical buffer is bounded before any directory, temp file, + // reference, or content hash is materialized on disk. + if (byteLength > maxBytes) return { status: "overflow" }; + + const now = input.now ?? new Date(); + const { year, month, day } = utcPartition(now); + const contentSha256 = sha256(Buffer.from(canonical, "utf8")); + const runSha12 = sha256(trace.run_id).slice(0, 12); + const filename = `${runSha12}-i${trace.iter}-${contentSha256.slice(0, 12)}.json`; + const ref = `${year}/${month}/${day}/policy/${filename}`; + const auditRoot = resolve(input.auditDir); + const destination = resolve(auditRoot, ...ref.split("/")); + if (!isContained(auditRoot, destination)) return { status: "error" }; + + const policyDir = join(auditRoot, year, month, day, "policy"); + mkdirSync(policyDir, { recursive: true, mode: 0o700 }); + const realRoot = realpathSync(auditRoot); + const realPolicyDir = realpathSync(policyDir); + if (!isContained(realRoot, realPolicyDir)) return { status: "error" }; + + writeFileAtomic(destination, canonical, { mode: 0o600 }); + return { status: "complete", ref, sha256: contentSha256 }; + } catch { + return { status: "error" }; + } +} + +export function verifyPolicyTraceReference( + input: VerifyPolicyTraceReferenceInput, +): PolicyTraceVerification { + if (!FULL_SHA256.test(input.sha256) || isAbsolute(input.ref) || input.ref.includes("\\")) { + return { ok: false, reason: "invalid-reference" }; + } + const match = POLICY_REF.exec(input.ref); + if (!match) return { ok: false, reason: "invalid-reference" }; + + const auditRoot = resolve(input.auditDir); + const candidate = resolve(auditRoot, ...input.ref.split("/")); + if (!isContained(auditRoot, candidate)) return { ok: false, reason: "path-escape" }; + if (!existsSync(candidate)) return { ok: false, reason: "missing" }; + + try { + const realRoot = realpathSync(auditRoot); + const realCandidate = realpathSync(candidate); + if (!isContained(realRoot, realCandidate)) return { ok: false, reason: "path-escape" }; + if (!lstatSync(realCandidate).isFile()) return { ok: false, reason: "not-a-file" }; + + const bytes = readFileSync(realCandidate); + const contentSha256 = sha256(bytes); + if (contentSha256 !== input.sha256) return { ok: false, reason: "hash-mismatch" }; + if (match[6] !== contentSha256.slice(0, 12)) { + return { ok: false, reason: "identity-mismatch" }; + } + + let decoded: unknown; + try { + decoded = JSON.parse(bytes.toString("utf8")); + } catch { + return { ok: false, reason: "invalid-json" }; + } + const parsed = PolicyTraceSchema.safeParse(decoded); + if (!parsed.success) return { ok: false, reason: "invalid-trace" }; + if (canonicalJson(parsed.data) !== bytes.toString("utf8")) { + return { ok: false, reason: "non-canonical" }; + } + if ( + match[4] !== sha256(parsed.data.run_id).slice(0, 12) || + Number(match[5]) !== parsed.data.iter + ) { + return { ok: false, reason: "identity-mismatch" }; + } + return { ok: true, trace: parsed.data }; + } catch { + return { ok: false, reason: "read-error" }; + } +} diff --git a/src/audit/verifier.ts b/src/audit/verifier.ts index 8b15d96..7f5d2c8 100644 --- a/src/audit/verifier.ts +++ b/src/audit/verifier.ts @@ -1,18 +1,14 @@ // src/audit/verifier.ts import { createHash } from "node:crypto"; import { readFile } from "node:fs/promises"; +import { dirname, resolve } from "node:path"; +import { canonicalJson } from "./canonical.ts"; +import { verifyPolicyTraceReference } from "./policy-trace-store.ts"; function sha256(s: string): string { return createHash("sha256").update(s).digest("hex"); } -function canonical(o: unknown): string { - if (o === null || typeof o !== "object") return JSON.stringify(o); - if (Array.isArray(o)) return `[${o.map(canonical).join(",")}]`; - const keys = Object.keys(o as Record).sort(); - return `{${keys.map((k) => `${JSON.stringify(k)}:${canonical((o as Record)[k])}`).join(",")}}`; -} - export interface VerifyResult { ok: boolean; brokenAtLine: number | null; @@ -23,6 +19,7 @@ export async function verifyChain(path: string): Promise { const raw = await readFile(path, "utf8"); const lines = raw.split("\n").filter((l) => l.length > 0); let prev = ""; + const auditDir = resolve(dirname(path), "..", "..", ".."); for (let i = 0; i < lines.length; i++) { // A tampered / truncated / half-flushed line is not valid JSON — treat it as a // BROKEN CHAIN at that line rather than letting JSON.parse throw an uncaught @@ -43,7 +40,7 @@ export async function verifyChain(path: string): Promise { const claimed = obj.this_event_hash as string; const recomputeBase = { ...obj }; recomputeBase.this_event_hash = undefined; - const recompute = sha256(canonical(recomputeBase)); + const recompute = sha256(canonicalJson(recomputeBase)); // Use recompute (not claimed) as the chain link so tampering of THIS line // surfaces at line i+2's prev_event_hash check. This gives the // brokenAtLine semantics the test asserts (brokenAtLine: 2 when line 1 is tampered). @@ -52,6 +49,23 @@ export async function verifyChain(path: string): Promise { if (i === lines.length - 1 && recompute !== claimed) { return { ok: false, brokenAtLine: i + 1, totalLines: lines.length }; } + const runSummary = obj.run_summary; + if (runSummary !== null && typeof runSummary === "object") { + const summary = runSummary as Record; + if (summary.policy_trace_status === "complete") { + if ( + typeof summary.policy_trace_ref !== "string" || + typeof summary.policy_trace_sha256 !== "string" || + !verifyPolicyTraceReference({ + auditDir, + ref: summary.policy_trace_ref, + sha256: summary.policy_trace_sha256, + }).ok + ) { + return { ok: false, brokenAtLine: i + 1, totalLines: lines.length }; + } + } + } } return { ok: true, brokenAtLine: null, totalLines: lines.length }; } diff --git a/src/core/loop-driver.ts b/src/core/loop-driver.ts index 4060dbd..2b29c7b 100644 --- a/src/core/loop-driver.ts +++ b/src/core/loop-driver.ts @@ -9,7 +9,7 @@ import { SETUP_BUDGET_MS_DEFAULT, } from "../config/budgets.ts"; import type { ReviewgateConfig } from "../config/define-config.ts"; -import type { RunSummary } from "../schemas/audit-event.ts"; +import { type RunSummary, RunSummarySchema } from "../schemas/audit-event.ts"; import { type DecisionEntry, DecisionEntrySchema } from "../schemas/decision.ts"; import { type Finding, FindingSchema } from "../schemas/finding.ts"; import { @@ -1900,13 +1900,29 @@ export class LoopDriver { // run.complete audit event. Wrapped in .catch so a logging failure can never // affect the verdict. Emitted on the iteration path only (not on the early // allow/escalation branches, which never run an iteration). + const auditRunSummary = (() => { + try { + const policySummary = result.policySummary; + if (policySummary === undefined) return result.summary; + return RunSummarySchema.parse({ + ...result.summary, + policy_trace_status: policySummary.status, + policy_trace_ref: policySummary.policy_trace_ref, + policy_trace_sha256: policySummary.policy_trace_sha256, + }); + } catch { + // Audit identity binding is telemetry. A malformed optional summary may + // lose that binding, but it must never change the already-computed verdict. + return result.summary; + } + })(); await this.i.audit .append({ event: "run.complete", run_id: state.session_id, iter: nextIter, trigger: "stop-hook", - run_summary: result.summary, + run_summary: auditRunSummary, }) .catch(() => {}); diff --git a/src/core/orchestrator.ts b/src/core/orchestrator.ts index e06143c..c40b617 100644 --- a/src/core/orchestrator.ts +++ b/src/core/orchestrator.ts @@ -52,7 +52,11 @@ import type { RunSummary } from "../schemas/audit-event.ts"; import { type MemoryProposal, VALID_EVIDENCE_KINDS } from "../schemas/brain.ts"; import type { Finding, FindingCategory } from "../schemas/finding.ts"; import { NO_PANEL_REVIEWER_ID } from "../schemas/pending-report.ts"; -import type { PolicySummary, PolicyTrace } from "../schemas/policy-trace.ts"; +import { + type PolicySummary, + PolicySummarySchema, + type PolicyTrace, +} from "../schemas/policy-trace.ts"; import type { PassLedger, ReviewedSnapshot } from "../schemas/state.ts"; import { triageFromFacts } from "../triage/matrix.ts"; import { refineTriage } from "../triage/triage-engine.ts"; @@ -98,6 +102,7 @@ import { orderForBudget, renderLoreBlock, selectForDiff } from "./lore/render.ts import { classifyEntry } from "./lore/staleness.ts"; import { type LoreEntryParsed, loadLore } from "./lore/store.ts"; import { PERSONA_REAFFIRM, reaffirmFor, resolvePersonas } from "./personas.ts"; +import { POLICY_CATALOG_VERSION, POLICY_PASSES } from "./policy/catalog.ts"; import type { PolicyExecutionOptions } from "./policy/replay.ts"; import { resolvePolicyExecutionOptions } from "./policy/replay.ts"; import { OrderedResponseHashes } from "./policy/response-hashes.ts"; @@ -2778,6 +2783,46 @@ export class Orchestrator { verdict: agg.verdict, finalFindings, }); + let policySummary: PolicySummary | undefined; + if (policyExecution.trace === "persist") { + // Keep the gate's abort boundary ahead of all persistence. The complete + // trace is stored before pending.*, so every report/run summary can bind + // one identical content address. + opts.signal?.throwIfAborted(); + const stored = + policyTrace === undefined || policyTrace === null || this.input.audit === undefined + ? ({ status: "error" } as const) + : this.input.audit.writePolicyTrace(policyTrace); + const candidate = { + catalog_version: POLICY_CATALOG_VERSION, + status: stored.status, + passes: + policyTrace?.passes ?? + POLICY_PASSES.map( + (pass) => + policyRuntime?.summary(pass.id) ?? { + pass_id: pass.id, + status: "error" as const, + reason_code: "instrumentation-error" as const, + }, + ), + ...(stored.status === "complete" + ? { policy_trace_ref: stored.ref, policy_trace_sha256: stored.sha256 } + : {}), + }; + const parsed = PolicySummarySchema.safeParse(candidate); + policySummary = parsed.success + ? parsed.data + : PolicySummarySchema.parse({ + catalog_version: POLICY_CATALOG_VERSION, + status: "error", + passes: POLICY_PASSES.map((pass) => ({ + pass_id: pass.id, + status: "error", + reason_code: "instrumentation-error", + })), + }); + } // Banner data for invalid/broad/zero-match entries + the render-budget drop // count — render-only (report-writer.ts), never affects the verdict. const loreBanner = @@ -2810,6 +2855,7 @@ export class Orchestrator { triage.riskClass === "docs", wholeDiffAttributable, loreBanner, + policySummary, ); // --- Brain Curator (Phase 4): non-blocking, best-effort, hard-timeout-bounded. @@ -2970,6 +3016,7 @@ export class Orchestrator { } : {}), ...(policyTrace === undefined || policyTrace === null ? {} : { policyTrace }), + ...(policySummary === undefined ? {} : { policySummary }), summary: buildRunSummary({ verdict: agg.verdict, source: "panel", @@ -2979,6 +3026,17 @@ export class Orchestrator { findings: finalFindings, runs: reviewerOutcomes, ruleUncited, + ...(policySummary === undefined + ? {} + : { + policyTraceStatus: policySummary.status, + ...(policySummary.policy_trace_ref === undefined + ? {} + : { policyTraceRef: policySummary.policy_trace_ref }), + ...(policySummary.policy_trace_sha256 === undefined + ? {} + : { policyTraceSha256: policySummary.policy_trace_sha256 }), + }), }), }; } @@ -3263,6 +3321,7 @@ export class Orchestrator { zero_match: string[]; dropped: number; }, + policySummary?: PolicySummary, ): Promise { // Single chokepoint for the self-deadline: if the gate aborted this run // (loop.runTimeoutMs), NO writeReport branch — early triage ERROR/PASS, cache @@ -3339,6 +3398,7 @@ export class Orchestrator { ? { whole_diff_attributable: wholeDiffAttributable } : {}), ...(loreBanner ? { lore_banner: loreBanner } : {}), + ...(policySummary ? { policy_summary: policySummary } : {}), cost_usd_total: runs.reduce((sum, r) => sum + r.res.usage.costUsd, 0), duration_ms_total: Date.now() - start, generated_at: new Date().toISOString(), diff --git a/src/core/report-writer.ts b/src/core/report-writer.ts index 1c553a6..3dba066 100644 --- a/src/core/report-writer.ts +++ b/src/core/report-writer.ts @@ -4,6 +4,7 @@ import { dirname } from "node:path"; import { neutralizeFences, neutralizeInjectionMarkers } from "../diff/sanitizer.ts"; import type { Finding } from "../schemas/finding.ts"; import type { PendingReport } from "../schemas/pending-report.ts"; +import type { PolicyEffect } from "../schemas/policy-trace.ts"; import type { EscalationReason } from "../schemas/state.ts"; import { writeFileAtomic } from "../utils/atomic-write.ts"; import { @@ -30,6 +31,20 @@ function consensusEmoji(c: Finding["consensus"]): string { return "⚪"; // singleton or minority } +function hasPolicyEffect( + finding: Finding, + passId: PolicyEffect["pass_id"], + predicate: (effect: PolicyEffect) => boolean = () => true, +): boolean { + return (finding.policy_effects ?? []).some( + (effect) => effect.pass_id === passId && predicate(effect), + ); +} + +function hasAppliedPolicyEffect(finding: Finding, passId: PolicyEffect["pass_id"]): boolean { + return hasPolicyEffect(finding, passId, (effect) => effect.action !== "protected"); +} + // Building finding badges: the hard-block 🔒 deterministic badge (for findings // from the deterministic checker tier) AND the demote/suppression badges (scope, // FP-ledger, critic, reputation, …). Builds a blockquote line ONLY when at @@ -40,17 +55,37 @@ export function findingBadges(f: Finding): string | null { const badges: string[] = []; if (f.deterministic) badges.push("🔒 deterministic check — fix it (re-runs automatically; not rejectable)"); - if (f.fact_invalid) badges.push("🔎 cited location not found — likely hallucinated"); + if ( + f.fact_invalid || + hasPolicyEffect( + f, + "evidence.fact-location", + (effect) => effect.reason_code === "location-out-of-range", + ) + ) + badges.push("🔎 cited location not found — likely hallucinated"); // Anchor repair: the counterpart to the badge above — the cited line was wrong, but the // reviewer's quoted evidence (carrying an identifier-like token) matched a real line of this // file, showing the reviewer read real code rather than fabricating one, so it was moved rather // than demoted. That is weaker than proof the defect itself is real — see finding.ts. - if (f.anchor_repaired) + if ( + f.anchor_repaired || + hasPolicyEffect( + f, + "evidence.fact-location", + (effect) => effect.reason_code === "evidence-line-reanchored", + ) + ) badges.push( "⚑ reviewer cited a line that does not exist — re-anchored to the source line it quoted", ); - if (f.grounding_demoted) badges.push("🌫 cited token absent from corpus — likely fabricated"); - if (f.hypothetical_demoted) + if ( + f.grounding_demoted || + hasAppliedPolicyEffect(f, "evidence.grounding-token") || + hasAppliedPolicyEffect(f, "judgment.grounding-llm") + ) + badges.push("🌫 cited token absent from corpus — likely fabricated"); + if (f.hypothetical_demoted || hasAppliedPolicyEffect(f, "judgment.hypothetical")) badges.push( "⏳ demoted CRITICAL→WARN — reviewer text is hypothetical/future, not a present defect", ); @@ -58,30 +93,36 @@ export function findingBadges(f: Finding): string | null { // decision-required on SOFT-PASS (it does NOT silently re-arm). Render only while still blocking // (CRITICAL/WARN): an INFO one was further suppressed by a structural/agent off-ramp (e.g. the // reject → cycleRejected path) and no longer needs a decision, so the prompt would mislead. - if (f.demoted_from_critical && f.severity !== "INFO") + const tracedFromCritical = (f.policy_effects ?? []).some( + (effect) => + effect.action !== "protected" && effect.before === "CRITICAL" && effect.after === "WARN", + ); + if ((f.demoted_from_critical || tracedFromCritical) && f.severity !== "INFO") badges.push( "⬇ was CRITICAL, one-step-demoted — decide before passing (don't reflexively acknowledge)", ); - if (f.scope_demoted) badges.push("📍 outside changed lines"); + if (f.scope_demoted || hasAppliedPolicyEffect(f, "scope.diff")) + badges.push("📍 outside changed lines"); // T4/R2: iteration >= 2 policy demote — fresh nit on content the panel already // reviewed and the agent did not touch since. - if (f.delta_scope_demoted) + if (f.delta_scope_demoted || hasAppliedPolicyEffect(f, "scope.delta")) badges.push("🗂 on content already reviewed and unchanged since — advisory (delta scope)"); // Slice A (P1): on a file this session did not author — advisory (parallel agent / pre-existing). - if (f.foreign_to_session) + if (f.foreign_to_session || hasAppliedPolicyEffect(f, "scope.session")) badges.push( "👥 on a file this session did not edit (parallel agent / pre-existing) — advisory; if it truly isn't yours, record an out-of-scope decision", ); - if (f.test_severity_demoted) badges.push("📁 security finding on a test/fixture file — advisory"); + if (f.test_severity_demoted || hasAppliedPolicyEffect(f, "judgment.test-security")) + badges.push("📁 security finding on a test/fixture file — advisory"); // Slice D (P5): a CRITICAL on a docs/markdown file capped to WARN (stale doc ≠ data-loss bug). - if (f.docs_severity_capped) + if (f.docs_severity_capped || hasAppliedPolicyEffect(f, "judgment.docs-cap")) badges.push("📝 docs file — capped CRITICAL→WARN; still decide before passing"); // Slice C (P4): a lone uncorroborated CRITICAL — honest framing, NOT a downgrade (still blocks). if (f.lone_critical_uncorroborated) badges.push( "🚧 lone CRITICAL — single reviewer, uncorroborated; verify the cited code yourself, then fix (action:fixed) or reject (reviewer_was_wrong) with a concrete reason", ); - if (f.redaction_demoted) + if (f.redaction_demoted || hasAppliedPolicyEffect(f, "evidence.redaction-placeholder")) badges.push( "🙈 targets a placeholder (stripped secret, not real code) — advisory", ); @@ -103,16 +144,22 @@ export function findingBadges(f: Finding): string | null { badges.push( "🔎 the line this finding cites as evidence is not present in the file — likely reasoned on stale or absent context; verify the cited code yourself before acting", ); - if (f.critic_verdict === "likely_fp") badges.push("🧠 critic flagged as likely FP"); - if (f.fp_ledger_match?.suppressed) badges.push("📒 matches known-FP pattern"); + if (f.critic_verdict === "likely_fp" || hasAppliedPolicyEffect(f, "judgment.critic")) + badges.push("🧠 critic flagged as likely FP"); + if (f.fp_ledger_match?.suppressed || hasAppliedPolicyEffect(f, "history.fp-signature")) + badges.push("📒 matches known-FP pattern"); if (f.fp_cluster_match?.suppressed) badges.push(`📚 active FP cluster ${f.fp_cluster_match.cluster_key}`); - if (f.low_confidence) badges.push("🎯 below confidence floor"); + if (f.low_confidence || hasAppliedPolicyEffect(f, "judgment.confidence")) + badges.push("🎯 below confidence floor"); // #4: only assert "kept blocking" while the finding IS still blocking. The protect flag is // stamped in the critic pass BEFORE the hard suppressors (scopeToDiff/fpActive/cycleRejected) // run; if one of them later demotes this finding to advisory INFO, the "kept blocking" badge // would be a lie (codex DoD) — so gate it on a non-INFO severity. - if (f.protected_high_precision && f.severity !== "INFO") + const tracedHighPrecisionProtection = (f.policy_effects ?? []).some( + (effect) => effect.action === "protected" && effect.protected_by === "high-precision-reviewer", + ); + if ((f.protected_high_precision || tracedHighPrecisionProtection) && f.severity !== "INFO") badges.push("🛡 kept blocking — high-track-record reviewer (soft demote overridden)"); // T3/R4 (field report 2026-07-03): region-rejection badge. Suppressed → explains WHY the // finding is advisory; blocking → cites the prior reason and names the fast-path so the @@ -130,7 +177,8 @@ export function findingBadges(f: Finding): string | null { badges.push( "🧷 needs corroboration — CRITICAL claim from a chronically-unreliable reviewer, clamped to WARN; verify the cited code, then fix or reject with evidence", ); - else if (f.reputation_demoted) badges.push("📉 reviewer reputation low"); + else if (f.reputation_demoted || hasAppliedPolicyEffect(f, "judgment.reputation")) + badges.push("📉 reviewer reputation low"); if (f.claimed_fixed_recurred) // A pinned recurrence that survived the demote chain (CRITICAL/WARN) is blocking → // assert the fix failed. One that was scope/fp-demoted to advisory INFO recurred but diff --git a/tests/integration/policy-trace-equivalence.test.ts b/tests/integration/policy-trace-equivalence.test.ts index 8a28be6..0b5b232 100644 --- a/tests/integration/policy-trace-equivalence.test.ts +++ b/tests/integration/policy-trace-equivalence.test.ts @@ -4,6 +4,7 @@ import { mkdtempSync, readFileSync, readdirSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { AuditLogger } from "../../src/audit/logger.ts"; +import { verifyPolicyTraceReference } from "../../src/audit/policy-trace-store.ts"; import { defaultConfig } from "../../src/config/defaults.ts"; import type { ReviewgateConfig } from "../../src/config/define-config.ts"; import { @@ -242,12 +243,32 @@ describe("policy trace lifecycle equivalence", () => { expect(traced.result.summary.policy_trace_sha256).toBeUndefined(); }); - it("defaults the ordinary AuditLogger path to persist mode with an empty internal ablation set", async () => { + it("defaults AuditLogger to persist mode and binds one verified compact identity", async () => { const production = await run(undefined, true); + const policySummary = production.result.policySummary; expect(production.result.policyTrace).toBeDefined(); expect(production.result.policyTrace?.ablated).toEqual([]); - expect(production.result.policySummary).toBeUndefined(); + expect(policySummary?.status).toBe("complete"); + expect((production.report as { policy_summary?: unknown }).policy_summary).toEqual( + policySummary, + ); + expect(production.result.summary.policy_trace_status).toBe(policySummary?.status); + expect(production.result.summary.policy_trace_ref).toBe(policySummary?.policy_trace_ref); + expect(production.result.summary.policy_trace_sha256).toBe(policySummary?.policy_trace_sha256); + if ( + policySummary?.policy_trace_ref === undefined || + policySummary.policy_trace_sha256 === undefined + ) { + throw new Error("persist mode did not produce a complete trace identity"); + } + expect( + verifyPolicyTraceReference({ + auditDir: join(production.repo, ".reviewgate", "audit"), + ref: policySummary.policy_trace_ref, + sha256: policySummary.policy_trace_sha256, + }).ok, + ).toBe(true); }); it("orders reviewer, grounding, and critic response hashes by logical call order", async () => { diff --git a/tests/unit/audit-logger-retention.test.ts b/tests/unit/audit-logger-retention.test.ts index c86a0e1..a66b312 100644 --- a/tests/unit/audit-logger-retention.test.ts +++ b/tests/unit/audit-logger-retention.test.ts @@ -21,6 +21,9 @@ function seedDay(auditDir: string, d: Date): string { mkdirSync(dir, { recursive: true }); const f = join(dir, "120000.jsonl"); writeFileSync(f, "{}\n"); + const policyDir = join(dir, "policy"); + mkdirSync(policyDir, { recursive: true }); + writeFileSync(join(policyDir, "trace.json"), "{}\n"); return dir; } @@ -36,9 +39,11 @@ describe("AuditLogger retention pruning", () => { const log = new AuditLogger(auditDir, 180); await log.append({ event: "session.start", run_id: "r1", iter: 0, trigger: "session-start" }); - // Older-than-180d partition pruned; the recent one and today's new file remain. + // The policy child follows the existing whole-day retention boundary: there is + // no independent artifact-retention implementation to drift from the JSONL log. expect(existsSync(oldDir)).toBe(false); expect(existsSync(recentDir)).toBe(true); + expect(existsSync(join(recentDir, "policy", "trace.json"))).toBe(true); expect(existsSync(log.currentFilePath())).toBe(true); }); diff --git a/tests/unit/audit-logger.test.ts b/tests/unit/audit-logger.test.ts index 00667da..1261dde 100644 --- a/tests/unit/audit-logger.test.ts +++ b/tests/unit/audit-logger.test.ts @@ -1,11 +1,18 @@ // tests/unit/audit-logger.test.ts -import { describe, expect, it } from "bun:test"; -import { mkdtempSync, readFileSync } from "node:fs"; +import { describe, expect, it, setSystemTime } from "bun:test"; +import { mkdtempSync, readFileSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { basename, join } from "node:path"; import { AuditLogger } from "../../src/audit/logger.ts"; import { verifyChain } from "../../src/audit/verifier.ts"; +import { defaultConfig } from "../../src/config/defaults.ts"; +import { LoopDriver } from "../../src/core/loop-driver.ts"; +import { POLICY_PASSES } from "../../src/core/policy/catalog.ts"; +import { PolicyTraceRecorder } from "../../src/core/policy/trace.ts"; +import { StateStore } from "../../src/core/state-store.ts"; +import type { RunSummary } from "../../src/schemas/audit-event.ts"; import { loadAuditWindow } from "../../src/stats/load.ts"; +import { auditDir, dirtyFlagPath } from "../../src/utils/paths.ts"; function tmp() { return mkdtempSync(join(tmpdir(), "rg-audit-")); @@ -27,6 +34,65 @@ describe("AuditLogger", () => { expect(parsed[2].prev_event_hash).toBe(parsed[1].this_event_hash); }); + it("preserves the pre-extraction canonical audit hash bytes", async () => { + setSystemTime(new Date("2026-08-10T12:00:00.000Z")); + try { + const log = new AuditLogger(tmp()); + const event = await log.append({ + event: "session.start", + run_id: "r1", + iter: 0, + trigger: "session-start", + }); + expect(event.this_event_hash).toBe( + "77968fbe0f1179d1c3445b603a476f47d16df8a4d2dcac2c3a96fbc229bacbc6", + ); + } finally { + setSystemTime(); + } + }); + + it("writePolicyTrace never throws when the audit root cannot be a directory", () => { + const blocked = join(tmp(), "blocked-audit-root"); + writeFileSync(blocked, "not a directory"); + const result = new AuditLogger(blocked).writePolicyTrace({} as never); + expect(result).toEqual({ status: "error" }); + }); + + it("stores a policy trace in the logger chain's already-selected UTC day partition", async () => { + setSystemTime(new Date("2026-08-10T23:59:59.000Z")); + try { + const log = new AuditLogger(tmp()); + await log.append({ + event: "session.start", + run_id: "midnight-run", + iter: 0, + trigger: "session-start", + }); + const recorder = PolicyTraceRecorder.start({ runId: "midnight-run", iter: 1, ablated: [] }); + recorder.recordStage({ + stageId: "verdict.compute", + reasonCode: "no-blocking-findings", + inputSignatures: [], + verdict: "PASS", + }); + const trace = recorder.finalize({ + rawResponseSha256: ["a".repeat(64)], + verdict: "PASS", + finalFindings: [], + }); + if (trace === null) throw new Error("fixture trace did not finalize"); + + setSystemTime(new Date("2026-08-11T00:00:01.000Z")); + const stored = log.writePolicyTrace(trace); + expect(stored.status).toBe("complete"); + if (stored.status !== "complete") throw new Error("fixture trace did not persist"); + expect(stored.ref).toStartWith("2026/08/10/policy/"); + } finally { + setSystemTime(); + } + }); + it("verifyChain returns ok=true on a freshly written chain", async () => { const dir = tmp(); const log = new AuditLogger(dir); @@ -106,3 +172,86 @@ describe("AuditLogger", () => { ).toEqual(["same-clock-0", "same-clock-1", "same-clock-2"]); }); }); + +describe("LoopDriver run.complete policy identity", () => { + it("binds run.complete to the exact IterationResult compact summary identity", async () => { + const repo = tmp(); + const state = new StateStore(repo); + await state.initialise("RUN-LOOP-POLICY"); + writeFileSync( + dirtyFlagPath(repo), + JSON.stringify({ diff_hash: "dirty", ts: new Date().toISOString() }), + ); + const audit = new AuditLogger(auditDir(repo)); + const passes = POLICY_PASSES.map((pass) => ({ + pass_id: pass.id, + status: "ran" as const, + considered: 0, + opportunities: 0, + would_apply: 0, + applied: 0, + protected: 0, + blocking_removed: 0, + blocking_preserved: 0, + dropped: 0, + })); + const policySummary = { + catalog_version: "reviewgate.policy-catalog.v1" as const, + status: "complete" as const, + passes, + policy_trace_ref: "2026/08/10/policy/aaaaaaaaaaaa-i1-bbbbbbbbbbbb.json", + policy_trace_sha256: "b".repeat(64), + }; + const summary: RunSummary = { + verdict: "PASS", + source: "panel", + counts: { critical: 0, warn: 0, info: 0 }, + cost_usd: 0, + duration_ms: 1, + demoted: 0, + signatures: [], + providers: [], + from_critical_demoted: 0, + corroboration_clamped: 0, + // Deliberately valid but stale: LoopDriver owns the final audit binding + // and must prefer the compact identity carried beside the pending report. + policy_trace_status: "complete", + policy_trace_ref: "2026/08/10/policy/cccccccccccc-i1-dddddddddddd.json", + policy_trace_sha256: "d".repeat(64), + }; + const driver = new LoopDriver({ + repoRoot: repo, + config: { + ...defaultConfig, + loop: { ...defaultConfig.loop, runTimeoutMs: 0 }, + }, + state, + audit, + orchestrator: { + async runIteration() { + return { + verdict: "PASS" as const, + costUsd: 0, + durationMs: 1, + signaturesThisIter: [], + locationsThisIter: [], + policySummary, + summary, + }; + }, + }, + stopHookActive: false, + freshHeadSha: async () => null, + }); + + await driver.run(); + const events = readFileSync(audit.currentFilePath(), "utf8") + .trim() + .split("\n") + .map((line) => JSON.parse(line)) as Array<{ event: string; run_summary?: RunSummary }>; + const complete = events.find((event) => event.event === "run.complete")?.run_summary; + expect(complete?.policy_trace_status).toBe(policySummary.status); + expect(complete?.policy_trace_ref).toBe(policySummary.policy_trace_ref); + expect(complete?.policy_trace_sha256).toBe(policySummary.policy_trace_sha256); + }); +}); diff --git a/tests/unit/audit-verify-corruption.test.ts b/tests/unit/audit-verify-corruption.test.ts index fd5b2fc..dbb7ee6 100644 --- a/tests/unit/audit-verify-corruption.test.ts +++ b/tests/unit/audit-verify-corruption.test.ts @@ -3,7 +3,7 @@ // malformed/tampered/truncated log line — NOT crash with an uncaught JSON.parse // SyntaxError / raw stack trace. import { describe, expect, it } from "bun:test"; -import { mkdtempSync, readFileSync, writeFileSync } from "node:fs"; +import { mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { AuditLogger } from "../../src/audit/logger.ts"; @@ -48,6 +48,168 @@ describe("verifyChain on a corrupt/non-JSON line", () => { }); }); +describe("verifyChain policy artifact binding", () => { + it("accepts legacy chains and rejects a missing complete policy artifact", async () => { + const legacy = await writeChain(); + expect((await verifyChain(legacy)).ok).toBe(true); + + const auditDir = tmp(); + const log = new AuditLogger(auditDir); + await log.append({ + event: "run.complete", + run_id: "policy-run", + iter: 1, + trigger: "stop-hook", + run_summary: { + verdict: "PASS", + source: "panel", + counts: { critical: 0, warn: 0, info: 0 }, + cost_usd: 0, + duration_ms: 1, + demoted: 0, + signatures: [], + providers: [], + policy_trace_status: "complete", + policy_trace_ref: "2026/08/10/policy/000000000000-i1-000000000000.json", + policy_trace_sha256: "0".repeat(64), + }, + }); + + expect(await verifyChain(log.currentFilePath())).toMatchObject({ + ok: false, + brokenAtLine: 1, + }); + }); + + it("rejects a policy artifact removed after a valid chained append", async () => { + const auditDir = tmp(); + const log = new AuditLogger(auditDir); + const stored = log.writePolicyTrace({ + schema: "reviewgate.policy-trace.v1", + catalog_version: "reviewgate.policy-catalog.v1", + run_id: "policy-run", + iter: 1, + ablated: [], + raw_response_sha256: ["a".repeat(64)], + passes: ( + [ + "evidence.fact-location", + "evidence.self-refutation", + "judgment.hypothetical", + "evidence.grounding-token", + "judgment.grounding-llm", + "evidence.redaction-placeholder", + "judgment.critic", + "scope.diff", + "scope.delta", + "scope.session", + "history.fp-signature", + "history.cycle-rejected", + "history.fp-cluster", + "judgment.confidence", + "judgment.reputation", + "history.region-rejected", + "judgment.test-security", + "judgment.docs-cap", + ] as const + ).map((pass_id) => ({ + pass_id, + status: "ran" as const, + considered: 0, + opportunities: 0, + would_apply: 0, + applied: 0, + protected: 0, + blocking_removed: 0, + blocking_preserved: 0, + dropped: 0, + })), + evaluations: [], + stages: [ + { + stage_id: "verdict.compute", + order: 190, + reason_code: "no-blocking-findings", + input_signatures: [], + verdict: "PASS", + }, + ], + final: { + verdict: "PASS", + counts: { critical: 0, warn: 0, info: 0 }, + finding_signatures: [], + finding_severities: [], + }, + }); + if (stored.status !== "complete") throw new Error("fixture trace did not persist"); + await log.append({ + event: "run.complete", + run_id: "policy-run", + iter: 1, + trigger: "stop-hook", + run_summary: { + verdict: "PASS", + source: "panel", + counts: { critical: 0, warn: 0, info: 0 }, + cost_usd: 0, + duration_ms: 1, + demoted: 0, + signatures: [], + providers: [], + policy_trace_status: stored.status, + policy_trace_ref: stored.ref, + policy_trace_sha256: stored.sha256, + }, + }); + expect((await verifyChain(log.currentFilePath())).ok).toBe(true); + + const artifact = join(auditDir, ...stored.ref.split("/")); + const original = readFileSync(artifact, "utf8"); + writeFileSync(artifact, `${original} `); + expect(await verifyChain(log.currentFilePath())).toMatchObject({ + ok: false, + brokenAtLine: 1, + }); + writeFileSync(artifact, original); + expect((await verifyChain(log.currentFilePath())).ok).toBe(true); + + rmSync(artifact); + expect(await verifyChain(log.currentFilePath())).toMatchObject({ + ok: false, + brokenAtLine: 1, + }); + }); + + it("rejects an escaping complete policy reference even when the audit hash is valid", async () => { + const auditDir = tmp(); + const log = new AuditLogger(auditDir); + await log.append({ + event: "run.complete", + run_id: "policy-run", + iter: 1, + trigger: "stop-hook", + run_summary: { + verdict: "PASS", + source: "panel", + counts: { critical: 0, warn: 0, info: 0 }, + cost_usd: 0, + duration_ms: 1, + demoted: 0, + signatures: [], + providers: [], + policy_trace_status: "complete", + policy_trace_ref: "../2026/08/10/policy/000000000000-i1-000000000000.json", + policy_trace_sha256: "0".repeat(64), + }, + }); + + expect(await verifyChain(log.currentFilePath())).toMatchObject({ + ok: false, + brokenAtLine: 1, + }); + }); +}); + describe("runAuditVerify command exit code + output", () => { it("exits non-zero with a clean message (no stack trace) on a corrupt log", async () => { const path = await writeChain(); diff --git a/tests/unit/policy-trace-store.test.ts b/tests/unit/policy-trace-store.test.ts new file mode 100644 index 0000000..2419d2e --- /dev/null +++ b/tests/unit/policy-trace-store.test.ts @@ -0,0 +1,283 @@ +import { describe, expect, it } from "bun:test"; +import { createHash } from "node:crypto"; +import { + chmodSync, + existsSync, + lstatSync, + mkdirSync, + mkdtempSync, + readFileSync, + readdirSync, + symlinkSync, + writeFileSync, +} from "node:fs"; +import { tmpdir } from "node:os"; +import { dirname, join } from "node:path"; +import { canonicalJson } from "../../src/audit/canonical.ts"; +import { + POLICY_TRACE_MAX_BYTES, + verifyPolicyTraceReference, + writePolicyTrace, +} from "../../src/audit/policy-trace-store.ts"; +import type { PolicyTrace } from "../../src/schemas/policy-trace.ts"; + +const NOW = new Date("2026-08-10T12:34:56.000Z"); +const PASS_IDS = [ + "evidence.fact-location", + "evidence.self-refutation", + "judgment.hypothetical", + "evidence.grounding-token", + "judgment.grounding-llm", + "evidence.redaction-placeholder", + "judgment.critic", + "scope.diff", + "scope.delta", + "scope.session", + "history.fp-signature", + "history.cycle-rejected", + "history.fp-cluster", + "judgment.confidence", + "judgment.reputation", + "history.region-rejected", + "judgment.test-security", + "judgment.docs-cap", +] as const; + +function tmp(prefix = "rg-policy-store-"): string { + return mkdtempSync(join(tmpdir(), prefix)); +} + +function sha256(bytes: string): string { + return createHash("sha256").update(Buffer.from(bytes, "utf8")).digest("hex"); +} + +function emptyRanSummary(pass_id: (typeof PASS_IDS)[number], considered = 0) { + return { + pass_id, + status: "ran" as const, + considered, + opportunities: 0, + would_apply: 0, + applied: 0, + protected: 0, + blocking_removed: 0, + blocking_preserved: 0, + dropped: 0, + }; +} + +function emptyTrace(runId = "run-1"): PolicyTrace { + return { + schema: "reviewgate.policy-trace.v1", + catalog_version: "reviewgate.policy-catalog.v1", + run_id: runId, + iter: 1, + ablated: [], + raw_response_sha256: ["a".repeat(64)], + passes: PASS_IDS.map((passId) => emptyRanSummary(passId)), + evaluations: [], + stages: [ + { + stage_id: "verdict.compute", + order: 190, + reason_code: "no-blocking-findings", + input_signatures: [], + verdict: "PASS", + }, + ], + final: { + verdict: "PASS", + counts: { critical: 0, warn: 0, info: 0 }, + finding_signatures: [], + finding_severities: [], + }, + }; +} + +function maximumSignatureTrace(findingCount: number): PolicyTrace { + const signatures = Array.from({ length: findingCount }, (_, index) => + index.toString(16).padStart(64, "0"), + ); + return { + ...emptyTrace("worst-case-run"), + passes: PASS_IDS.map((passId) => emptyRanSummary(passId, findingCount)), + evaluations: PASS_IDS.flatMap((passId, passIndex) => + signatures.map((signature) => ({ + pass_id: passId, + order: (passIndex + 1) * 10, + result: "no-opportunity" as const, + before: "INFO" as const, + after: "INFO" as const, + reason_code: "ineligible-starting-state" as const, + source_signatures: [signature], + final_signature: signature, + })), + ), + stages: [ + ...signatures.map((signature) => ({ + stage_id: "aggregation.cluster" as const, + order: 65, + reason_code: "singleton" as const, + member_count: 1, + input_signatures: [signature], + output_signature: signature, + })), + { + stage_id: "verdict.compute" as const, + order: 190, + reason_code: "no-blocking-findings" as const, + input_signatures: [], + verdict: "PASS" as const, + }, + ], + final: { + verdict: "PASS", + counts: { critical: 0, warn: 0, info: findingCount }, + finding_signatures: signatures, + finding_severities: signatures.map((signature) => ({ signature, severity: "INFO" as const })), + }, + }; +} + +function allDescendants(root: string): string[] { + if (!existsSync(root) || !lstatSync(root).isDirectory()) return []; + return readdirSync(root, { withFileTypes: true }).flatMap((entry) => { + const path = join(root, entry.name); + return entry.isDirectory() ? [path, ...allDescendants(path)] : [path]; + }); +} + +describe("canonical policy trace storage", () => { + it("keeps the audit canonicalizer sorted recursively and byte-stable", () => { + expect(canonicalJson({ z: 1, a: { d: 4, b: 2 }, c: [2, { y: 1, x: 0 }] })).toBe( + '{"a":{"b":2,"d":4},"c":[2,{"x":0,"y":1}],"z":1}', + ); + }); + + it("stores canonical UTF-8 bytes at a hashed UTC path with mode 0600", () => { + const auditDir = join(tmp(), "audit"); + const trace = emptyTrace("../../escape/raw-run-id"); + const canonical = canonicalJson(trace); + const contentHash = sha256(canonical); + const runHash = sha256(trace.run_id); + + const stored = writePolicyTrace({ auditDir, trace, now: NOW }); + + expect(stored).toEqual({ + status: "complete", + ref: `2026/08/10/policy/${runHash.slice(0, 12)}-i1-${contentHash.slice(0, 12)}.json`, + sha256: contentHash, + }); + if (stored.status !== "complete") throw new Error("fixture trace did not persist"); + const artifact = join(auditDir, ...stored.ref.split("/")); + expect(readFileSync(artifact, "utf8")).toBe(canonical); + expect(lstatSync(artifact).mode & 0o777).toBe(0o600); + expect(stored.ref).not.toContain("escape"); + expect(stored.ref).not.toContain(trace.run_id); + expect( + verifyPolicyTraceReference({ auditDir, ref: stored.ref, sha256: stored.sha256 }), + ).toMatchObject({ ok: true }); + }); + + it("accepts exactly maxBytes and overflows one byte above it before creating a path", () => { + const trace = emptyTrace(); + const bytes = Buffer.byteLength(canonicalJson(trace), "utf8"); + const exactAudit = join(tmp(), "audit-exact"); + const overflowAudit = join(tmp(), "audit-overflow"); + + expect( + writePolicyTrace({ auditDir: exactAudit, trace, maxBytes: bytes, now: NOW }).status, + ).toBe("complete"); + expect( + writePolicyTrace({ auditDir: overflowAudit, trace, maxBytes: bytes - 1, now: NOW }), + ).toEqual({ status: "overflow" }); + expect(existsSync(overflowAudit)).toBe(false); + }); + + it("returns overflow for exactly 1,048,577 canonical bytes with no dir, ref, hash, or temp", () => { + const base = emptyTrace("x"); + const baseBytes = Buffer.byteLength(canonicalJson(base), "utf8"); + const trace = emptyTrace("x".repeat(1 + 1_048_577 - baseBytes)); + expect(Buffer.byteLength(canonicalJson(trace), "utf8")).toBe(1_048_577); + const auditDir = join(tmp(), "never-created-audit"); + + expect(writePolicyTrace({ auditDir, trace, now: NOW })).toEqual({ status: "overflow" }); + expect(POLICY_TRACE_MAX_BYTES).toBe(1_048_576); + expect(existsSync(auditDir)).toBe(false); + }); + + it("cleans its private temp and omits identity when the atomic write fails", () => { + const auditDir = join(tmp(), "audit"); + mkdirSync(auditDir, { recursive: true }); + writeFileSync(join(auditDir, "2026"), "blocks the UTC directory"); + + expect(writePolicyTrace({ auditDir, trace: emptyTrace(), now: NOW })).toEqual({ + status: "error", + }); + expect(allDescendants(auditDir).some((path) => path.endsWith(".tmp"))).toBe(false); + }); +}); + +describe("policy trace reference security", () => { + it("rejects missing, absolute, traversing, wrong-hash, tampered, and symlink-escaping refs", () => { + const root = tmp(); + const auditDir = join(root, "audit"); + const stored = writePolicyTrace({ auditDir, trace: emptyTrace(), now: NOW }); + if (stored.status !== "complete") throw new Error("fixture trace did not persist"); + const artifact = join(auditDir, ...stored.ref.split("/")); + + expect( + verifyPolicyTraceReference({ + auditDir, + ref: "2026/08/10/policy/000000000000-i1-000000000000.json", + sha256: "0".repeat(64), + }).ok, + ).toBe(false); + expect(verifyPolicyTraceReference({ auditDir, ref: artifact, sha256: stored.sha256 }).ok).toBe( + false, + ); + expect( + verifyPolicyTraceReference({ + auditDir, + ref: `../${stored.ref}`, + sha256: stored.sha256, + }).ok, + ).toBe(false); + expect( + verifyPolicyTraceReference({ auditDir, ref: stored.ref, sha256: "0".repeat(64) }).ok, + ).toBe(false); + + const symlinkAudit = join(root, "symlink-audit"); + const symlinkDay = join(symlinkAudit, "2026", "08", "10"); + mkdirSync(symlinkDay, { recursive: true }); + symlinkSync(dirname(artifact), join(symlinkDay, "policy")); + expect( + verifyPolicyTraceReference({ + auditDir: symlinkAudit, + ref: stored.ref, + sha256: stored.sha256, + }).ok, + ).toBe(false); + + chmodSync(artifact, 0o600); + writeFileSync(artifact, `${readFileSync(artifact, "utf8")} `); + expect( + verifyPolicyTraceReference({ auditDir, ref: stored.ref, sha256: stored.sha256 }).ok, + ).toBe(false); + }); + + it("stores the 1,046,855-byte maximum-signature all-pass trace and overflows the 1,053,027-byte next finding", () => { + const below = maximumSignatureTrace(169); + const above = maximumSignatureTrace(170); + const belowBytes = Buffer.byteLength(canonicalJson(below), "utf8"); + const aboveBytes = Buffer.byteLength(canonicalJson(above), "utf8"); + expect(belowBytes).toBe(1_046_855); + expect(aboveBytes).toBe(1_053_027); + expect( + writePolicyTrace({ auditDir: join(tmp(), "below"), trace: below, now: NOW }).status, + ).toBe("complete"); + expect(writePolicyTrace({ auditDir: join(tmp(), "above"), trace: above, now: NOW })).toEqual({ + status: "overflow", + }); + }); +}); diff --git a/tests/unit/report-writer.test.ts b/tests/unit/report-writer.test.ts index 46908a2..73a715b 100644 --- a/tests/unit/report-writer.test.ts +++ b/tests/unit/report-writer.test.ts @@ -1,9 +1,16 @@ // tests/unit/report-writer.test.ts import { describe, expect, it } from "bun:test"; -import { mkdtempSync, readFileSync } from "node:fs"; +import { mkdtempSync, readFileSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; -import { ReportWriter } from "../../src/core/report-writer.ts"; +import { AuditLogger } from "../../src/audit/logger.ts"; +import { verifyPolicyTraceReference } from "../../src/audit/policy-trace-store.ts"; +import { defaultConfig } from "../../src/config/defaults.ts"; +import { Orchestrator } from "../../src/core/orchestrator.ts"; +import { POLICY_PASSES } from "../../src/core/policy/catalog.ts"; +import { ReportWriter, findingBadges } from "../../src/core/report-writer.ts"; +import type { ProviderAdapter, ReviewResult } from "../../src/providers/adapter-base.ts"; +import type { Finding } from "../../src/schemas/finding.ts"; import type { PendingReport } from "../../src/schemas/pending-report.ts"; const baseReport: PendingReport = { @@ -61,6 +68,39 @@ describe("ReportWriter", () => { expect(json.findings[0].id).toBe("F-001"); }); + it("adds the compact policy summary to JSON without changing one Markdown byte", async () => { + const dir = mkdtempSync(join(tmpdir(), "rg-rep-policy-summary-")); + const writer = new ReportWriter(dir); + await writer.write(baseReport); + const legacyMarkdown = readFileSync(join(dir, ".reviewgate", "pending.md")); + const policySummary = { + catalog_version: "reviewgate.policy-catalog.v1" as const, + status: "complete" as const, + passes: POLICY_PASSES.map((pass) => ({ + pass_id: pass.id, + status: "ran" as const, + considered: 0, + opportunities: 0, + would_apply: 0, + applied: 0, + protected: 0, + blocking_removed: 0, + blocking_preserved: 0, + dropped: 0, + })), + policy_trace_ref: "2026/08/10/policy/aaaaaaaaaaaa-i1-bbbbbbbbbbbb.json", + policy_trace_sha256: "b".repeat(64), + }; + + await writer.write({ ...baseReport, policy_summary: policySummary }); + const tracedMarkdown = readFileSync(join(dir, ".reviewgate", "pending.md")); + const pending = JSON.parse( + readFileSync(join(dir, ".reviewgate", "pending.json"), "utf8"), + ) as PendingReport; + expect(tracedMarkdown.equals(legacyMarkdown)).toBe(true); + expect(pending.policy_summary).toEqual(policySummary); + }); + it("renders a line RANGE for a multi-line finding (line_start-line_end)", async () => { const dir = mkdtempSync(join(tmpdir(), "rg-rep-")); const w = new ReportWriter(dir); @@ -288,6 +328,52 @@ describe("ReportWriter", () => { expect(badgeLine).toContain("🎯"); }); + it("derives existing badge copy from policy effects with legacy marker fallback", () => { + const markerFinding = { + ...f0, + severity: "INFO" as const, + low_confidence: true, + }; + const tracedFinding = { + ...f0, + severity: "INFO" as const, + policy_effects: [ + { + pass_id: "judgment.confidence" as const, + order: 140, + action: "demoted" as const, + before: "WARN" as const, + after: "INFO" as const, + reason_code: "below-confidence-floor" as const, + source_signatures: [f0.signature], + }, + ], + }; + expect(findingBadges(tracedFinding)).toBe(findingBadges(markerFinding)); + expect(findingBadges(tracedFinding)).toContain("🎯 below confidence floor"); + }); + + it("derives existing high-precision protection copy from a protected effect", () => { + const markerFinding = { ...f0, severity: "WARN" as const, protected_high_precision: true }; + const tracedFinding = { + ...f0, + severity: "WARN" as const, + policy_effects: [ + { + pass_id: "judgment.critic" as const, + order: 70, + action: "protected" as const, + before: "WARN" as const, + after: "WARN" as const, + reason_code: "critic-likely-fp" as const, + protected_by: "high-precision-reviewer" as const, + source_signatures: [f0.signature], + }, + ], + }; + expect(findingBadges(tracedFinding)).toBe(findingBadges(markerFinding)); + }); + it("claimed_fixed_recurred (blocking CRITICAL/WARN) → asserts the fix did not resolve it", async () => { const md = await renderFinding({ claimed_fixed_recurred: { iter: 2 } }); expect(md).toContain("claimed fixed @ iter 2"); @@ -346,3 +432,142 @@ describe("ReportWriter", () => { }); }); }); + +const POLICY_DIFF = [ + "diff --git a/a.ts b/a.ts", + "--- a/a.ts", + "+++ b/a.ts", + "@@ -1 +1 @@", + "-export const value = 0;", + "+export const value = 1;", + "", +].join("\n"); + +function policyAdapter(): ProviderAdapter { + const finding: Finding = { + id: "F-001", + signature: "a".repeat(64), + severity: "INFO", + category: "quality", + rule_id: "fixture", + file: "a.ts", + line_start: 1, + line_end: 1, + message: "fixture advisory", + details: "fixture advisory", + reviewer: { provider: "codex", model: "fixture", persona: "quality" }, + confidence: 0.9, + consensus: "singleton", + }; + return { + id: "codex", + async preflight() { + return { available: true, version: "fixture", authMode: "oauth", error: null }; + }, + async review(input) { + return { + reviewerId: input.reviewerId, + verdict: "PASS", + findings: [finding], + usage: { inputTokens: 1, outputTokens: 1, costUsd: 0, quotaUsedPct: null }, + durationMs: 1, + exitCode: 0, + rawEventsPath: "", + rawText: '{"verdict":"PASS","findings":[]}', + status: "ok", + } satisfies ReviewResult; + }, + }; +} + +function policyConfig() { + return { + ...defaultConfig, + cache: { enabled: false, reviewTtlDays: 7 }, + phases: { + ...defaultConfig.phases, + review: { + ...defaultConfig.phases.review, + reviewers: [{ provider: "codex" as const, persona: "quality" }], + providerPrecisionContext: false, + }, + brain: null, + critic: null, + fpLedger: null, + grounding: null, + implicitOutcomes: null, + lore: null, + triage: null, + }, + }; +} + +describe("Orchestrator persisted policy identity", () => { + it("uses one compact identity in Pending, IterationResult, and RunSummary", async () => { + const repo = mkdtempSync(join(tmpdir(), "rg-orch-policy-persist-")); + writeFileSync(join(repo, "a.ts"), "export const value = 1;\n"); + const auditDir = join(repo, ".reviewgate", "audit"); + const result = await new Orchestrator({ + repoRoot: repo, + config: policyConfig(), + audit: new AuditLogger(auditDir), + adapters: { codex: policyAdapter() }, + sandboxMode: "off", + hostTier: "opus", + diff: POLICY_DIFF, + reasonOnFailEnabled: true, + disableLastResortFailover: true, + }).runIteration({ runId: "RUN-PERSISTED-POLICY", iter: 1 }); + const pending = JSON.parse( + readFileSync(join(repo, ".reviewgate", "pending.json"), "utf8"), + ) as PendingReport; + + expect(result.policySummary).toEqual(pending.policy_summary); + expect(result.policySummary?.status).toBe("complete"); + expect(result.summary.policy_trace_status).toBe(result.policySummary?.status); + expect(result.summary.policy_trace_ref).toBe(result.policySummary?.policy_trace_ref); + expect(result.summary.policy_trace_sha256).toBe(result.policySummary?.policy_trace_sha256); + if ( + result.policySummary?.policy_trace_ref === undefined || + result.policySummary.policy_trace_sha256 === undefined + ) { + throw new Error("complete persistence identity missing"); + } + expect( + verifyPolicyTraceReference({ + auditDir, + ref: result.policySummary.policy_trace_ref, + sha256: result.policySummary.policy_trace_sha256, + }).ok, + ).toBe(true); + }); + + it("keeps the production verdict/findings when trace persistence fails", async () => { + const repo = mkdtempSync(join(tmpdir(), "rg-orch-policy-error-")); + writeFileSync(join(repo, "a.ts"), "export const value = 1;\n"); + const blockedAudit = join(repo, "blocked-audit"); + writeFileSync(blockedAudit, "not a directory"); + const result = await new Orchestrator({ + repoRoot: repo, + config: policyConfig(), + audit: new AuditLogger(blockedAudit), + adapters: { codex: policyAdapter() }, + sandboxMode: "off", + hostTier: "opus", + diff: POLICY_DIFF, + reasonOnFailEnabled: true, + disableLastResortFailover: true, + }).runIteration({ runId: "RUN-PERSISTENCE-FAILS", iter: 1 }); + const pending = JSON.parse( + readFileSync(join(repo, ".reviewgate", "pending.json"), "utf8"), + ) as PendingReport; + + expect(result.verdict).toBe("PASS"); + expect(pending.verdict).toBe("PASS"); + expect(pending.findings).toHaveLength(1); + expect(result.policySummary?.status).toBe("error"); + expect(result.summary.policy_trace_status).toBe("error"); + expect(result.summary.policy_trace_ref).toBeUndefined(); + expect(result.summary.policy_trace_sha256).toBeUndefined(); + }); +}); From 62a39866f6b0242f39c1a798b6b8045bf9d927be Mon Sep 17 00:00:00 2001 From: Codevena Date: Mon, 10 Aug 2026 09:46:07 +0200 Subject: [PATCH 36/55] fix(audit): close policy trace storage boundaries --- src/audit/policy-trace-store.ts | 186 +++++++++++++++++++++--- tests/unit/policy-trace-store.test.ts | 196 +++++++++++++++++++++++++- 2 files changed, 361 insertions(+), 21 deletions(-) diff --git a/src/audit/policy-trace-store.ts b/src/audit/policy-trace-store.ts index 6000df7..de5cb7f 100644 --- a/src/audit/policy-trace-store.ts +++ b/src/audit/policy-trace-store.ts @@ -1,8 +1,18 @@ import { createHash } from "node:crypto"; -import { existsSync, lstatSync, mkdirSync, readFileSync, realpathSync } from "node:fs"; -import { isAbsolute, join, relative, resolve } from "node:path"; +import { + constants, + closeSync, + existsSync, + fstatSync, + lstatSync, + mkdirSync, + openSync, + readFileSync, + realpathSync, +} from "node:fs"; +import { dirname, isAbsolute, join, relative, resolve } from "node:path"; import { type PolicyTrace, PolicyTraceSchema } from "../schemas/policy-trace.ts"; -import { writeFileAtomic } from "../utils/atomic-write.ts"; +import { writeFileIfAbsent } from "../utils/atomic-write.ts"; import { canonicalJson } from "./canonical.ts"; export const POLICY_TRACE_MAX_BYTES = 1_048_576; @@ -20,7 +30,9 @@ export type PolicyTraceVerification = | "path-escape" | "missing" | "not-a-file" + | "too-large" | "hash-mismatch" + | "invalid-encoding" | "invalid-json" | "invalid-trace" | "non-canonical" @@ -54,6 +66,106 @@ function isContained(root: string, candidate: string): boolean { return rel === "" || (!rel.startsWith("..") && !isAbsolute(rel)); } +function realNonSymlinkDirectory(path: string): string | null { + const stat = lstatSync(path); + if (stat.isSymbolicLink() || !stat.isDirectory()) return null; + return realpathSync(path); +} + +function ensureAuditRoot(auditRoot: string): string | null { + if (!existsSync(auditRoot)) { + const parent = dirname(auditRoot); + if (realNonSymlinkDirectory(parent) === null) return null; + mkdirSync(auditRoot, { mode: 0o700 }); + } + return realNonSymlinkDirectory(auditRoot); +} + +function ensureContainedDirectory( + auditRoot: string, + realAuditRoot: string, + parent: string, + name: string, +): string | null { + const realParent = realNonSymlinkDirectory(parent); + if (realParent === null || !isContained(realAuditRoot, realParent)) return null; + const path = join(parent, name); + if (!existsSync(path)) mkdirSync(path, { mode: 0o700 }); + const realPath = realNonSymlinkDirectory(path); + if (realPath === null || !isContained(realAuditRoot, realPath)) return null; + if (!isContained(auditRoot, path)) return null; + return path; +} + +function isExactRegularArtifact(path: string, realAuditRoot: string, expected: Buffer): boolean { + const read = readBoundedRegularArtifact(path, realAuditRoot, 0o600); + return read.ok && read.bytes.equals(expected); +} + +function readBoundedRegularArtifact( + path: string, + realAuditRoot: string, + requiredMode?: number, +): + | { ok: true; bytes: Buffer } + | { ok: false; reason: "not-a-file" | "path-escape" | "too-large" | "read-error" } { + const pathBefore = lstatSync(path); + if (pathBefore.isSymbolicLink() || !pathBefore.isFile() || pathBefore.nlink !== 1) { + return { ok: false, reason: "not-a-file" }; + } + if (requiredMode !== undefined && (pathBefore.mode & 0o777) !== requiredMode) { + return { ok: false, reason: "not-a-file" }; + } + if (pathBefore.size > POLICY_TRACE_MAX_BYTES) return { ok: false, reason: "too-large" }; + const realPath = realpathSync(path); + if (!isContained(realAuditRoot, realPath)) return { ok: false, reason: "path-escape" }; + + const fd = openSync(path, constants.O_RDONLY | constants.O_NOFOLLOW); + try { + const openedBefore = fstatSync(fd); + if (!openedBefore.isFile() || openedBefore.nlink !== 1) { + return { ok: false, reason: "not-a-file" }; + } + if (requiredMode !== undefined && (openedBefore.mode & 0o777) !== requiredMode) { + return { ok: false, reason: "not-a-file" }; + } + if (openedBefore.size > POLICY_TRACE_MAX_BYTES) { + return { ok: false, reason: "too-large" }; + } + if (openedBefore.dev !== pathBefore.dev || openedBefore.ino !== pathBefore.ino) { + return { ok: false, reason: "read-error" }; + } + + const bytes = readFileSync(fd); + if (bytes.length > POLICY_TRACE_MAX_BYTES) return { ok: false, reason: "too-large" }; + const openedAfter = fstatSync(fd); + if ( + openedAfter.dev !== openedBefore.dev || + openedAfter.ino !== openedBefore.ino || + openedAfter.size !== openedBefore.size || + openedAfter.mtimeMs !== openedBefore.mtimeMs || + openedAfter.ctimeMs !== openedBefore.ctimeMs + ) { + return { ok: false, reason: "read-error" }; + } + const pathAfter = lstatSync(path); + if ( + pathAfter.isSymbolicLink() || + !pathAfter.isFile() || + pathAfter.nlink !== 1 || + (requiredMode !== undefined && (pathAfter.mode & 0o777) !== requiredMode) || + pathAfter.dev !== openedAfter.dev || + pathAfter.ino !== openedAfter.ino || + realpathSync(path) !== realPath + ) { + return { ok: false, reason: "read-error" }; + } + return { ok: true, bytes }; + } finally { + closeSync(fd); + } +} + function utcPartition(now: Date): { year: string; month: string; day: string } { return { year: String(now.getUTCFullYear()), @@ -66,7 +178,8 @@ export function writePolicyTrace(input: WritePolicyTraceInput): PolicyTraceWrite try { const trace = PolicyTraceSchema.parse(input.trace); const canonical = canonicalJson(trace); - const byteLength = Buffer.byteLength(canonical, "utf8"); + const canonicalBytes = Buffer.from(canonical, "utf8"); + const byteLength = canonicalBytes.length; const maxBytes = input.maxBytes ?? POLICY_TRACE_MAX_BYTES; if (!Number.isSafeInteger(maxBytes) || maxBytes < 0) return { status: "error" }; @@ -76,7 +189,7 @@ export function writePolicyTrace(input: WritePolicyTraceInput): PolicyTraceWrite const now = input.now ?? new Date(); const { year, month, day } = utcPartition(now); - const contentSha256 = sha256(Buffer.from(canonical, "utf8")); + const contentSha256 = sha256(canonicalBytes); const runSha12 = sha256(trace.run_id).slice(0, 12); const filename = `${runSha12}-i${trace.iter}-${contentSha256.slice(0, 12)}.json`; const ref = `${year}/${month}/${day}/policy/${filename}`; @@ -84,13 +197,33 @@ export function writePolicyTrace(input: WritePolicyTraceInput): PolicyTraceWrite const destination = resolve(auditRoot, ...ref.split("/")); if (!isContained(auditRoot, destination)) return { status: "error" }; - const policyDir = join(auditRoot, year, month, day, "policy"); - mkdirSync(policyDir, { recursive: true, mode: 0o700 }); - const realRoot = realpathSync(auditRoot); - const realPolicyDir = realpathSync(policyDir); - if (!isContained(realRoot, realPolicyDir)) return { status: "error" }; + const realRoot = ensureAuditRoot(auditRoot); + if (realRoot === null) return { status: "error" }; + let policyDir = auditRoot; + for (const component of [year, month, day, "policy"]) { + const next = ensureContainedDirectory(auditRoot, realRoot, policyDir, component); + if (next === null) return { status: "error" }; + policyDir = next; + } - writeFileAtomic(destination, canonical, { mode: 0o600 }); + const finalPolicyDir = realNonSymlinkDirectory(policyDir); + if (finalPolicyDir === null || !isContained(realRoot, finalPolicyDir)) { + return { status: "error" }; + } + if (existsSync(destination)) { + return isExactRegularArtifact(destination, realRoot, canonicalBytes) + ? { status: "complete", ref, sha256: contentSha256 } + : { status: "error" }; + } + + // Publish without replacement: if another writer or an attacker creates the + // final path after the existence check, link(2) returns EEXIST and their path + // is validated below rather than overwritten. + const created = writeFileIfAbsent(destination, canonical, { mode: 0o600 }); + if (!created && !existsSync(destination)) return { status: "error" }; + if (!isExactRegularArtifact(destination, realRoot, canonicalBytes)) { + return { status: "error" }; + } return { status: "complete", ref, sha256: contentSha256 }; } catch { return { status: "error" }; @@ -112,27 +245,42 @@ export function verifyPolicyTraceReference( if (!existsSync(candidate)) return { ok: false, reason: "missing" }; try { - const realRoot = realpathSync(auditRoot); - const realCandidate = realpathSync(candidate); - if (!isContained(realRoot, realCandidate)) return { ok: false, reason: "path-escape" }; - if (!lstatSync(realCandidate).isFile()) return { ok: false, reason: "not-a-file" }; - - const bytes = readFileSync(realCandidate); + const realRoot = realNonSymlinkDirectory(auditRoot); + if (realRoot === null) return { ok: false, reason: "path-escape" }; + let parent = auditRoot; + for (const component of input.ref.split("/").slice(0, -1)) { + parent = join(parent, component); + const realParent = realNonSymlinkDirectory(parent); + if (realParent === null || !isContained(realRoot, realParent)) { + return { ok: false, reason: "path-escape" }; + } + } + const read = readBoundedRegularArtifact(candidate, realRoot); + if (!read.ok) return read; + const { bytes } = read; const contentSha256 = sha256(bytes); if (contentSha256 !== input.sha256) return { ok: false, reason: "hash-mismatch" }; if (match[6] !== contentSha256.slice(0, 12)) { return { ok: false, reason: "identity-mismatch" }; } + let text: string; + try { + text = new TextDecoder("utf-8", { fatal: true }).decode(bytes); + } catch { + return { ok: false, reason: "invalid-encoding" }; + } + let decoded: unknown; try { - decoded = JSON.parse(bytes.toString("utf8")); + decoded = JSON.parse(text); } catch { return { ok: false, reason: "invalid-json" }; } const parsed = PolicyTraceSchema.safeParse(decoded); if (!parsed.success) return { ok: false, reason: "invalid-trace" }; - if (canonicalJson(parsed.data) !== bytes.toString("utf8")) { + const canonicalBytes = Buffer.from(canonicalJson(parsed.data), "utf8"); + if (!canonicalBytes.equals(bytes)) { return { ok: false, reason: "non-canonical" }; } if ( diff --git a/tests/unit/policy-trace-store.test.ts b/tests/unit/policy-trace-store.test.ts index 2419d2e..1a2e99a 100644 --- a/tests/unit/policy-trace-store.test.ts +++ b/tests/unit/policy-trace-store.test.ts @@ -3,11 +3,14 @@ import { createHash } from "node:crypto"; import { chmodSync, existsSync, + linkSync, lstatSync, mkdirSync, mkdtempSync, readFileSync, readdirSync, + rmSync, + statSync, symlinkSync, writeFileSync, } from "node:fs"; @@ -47,8 +50,10 @@ function tmp(prefix = "rg-policy-store-"): string { return mkdtempSync(join(tmpdir(), prefix)); } -function sha256(bytes: string): string { - return createHash("sha256").update(Buffer.from(bytes, "utf8")).digest("hex"); +function sha256(bytes: string | Buffer): string { + return createHash("sha256") + .update(typeof bytes === "string" ? Buffer.from(bytes, "utf8") : bytes) + .digest("hex"); } function emptyRanSummary(pass_id: (typeof PASS_IDS)[number], considered = 0) { @@ -147,6 +152,26 @@ function allDescendants(root: string): string[] { }); } +function writeUncheckedArtifact( + auditDir: string, + trace: PolicyTrace, + bytes: Buffer, +): { ref: string; sha256: string; path: string } { + const contentSha256 = sha256(bytes); + const ref = `2026/08/10/policy/${sha256(trace.run_id).slice(0, 12)}-i${trace.iter}-${contentSha256.slice(0, 12)}.json`; + const path = join(auditDir, ...ref.split("/")); + mkdirSync(dirname(path), { recursive: true }); + writeFileSync(path, bytes); + return { ref, sha256: contentSha256, path }; +} + +function artifactDescriptor(auditDir: string, trace: PolicyTrace) { + const canonical = canonicalJson(trace); + const sha = sha256(canonical); + const ref = `2026/08/10/policy/${sha256(trace.run_id).slice(0, 12)}-i${trace.iter}-${sha.slice(0, 12)}.json`; + return { canonical, ref, sha256: sha, path: join(auditDir, ...ref.split("/")) }; +} + describe("canonical policy trace storage", () => { it("keeps the audit canonicalizer sorted recursively and byte-stable", () => { expect(canonicalJson({ z: 1, a: { d: 4, b: 2 }, c: [2, { y: 1, x: 0 }] })).toBe( @@ -216,6 +241,94 @@ describe("canonical policy trace storage", () => { }); expect(allDescendants(auditDir).some((path) => path.endsWith(".tmp"))).toBe(false); }); + + it("rejects a symlinked year parent before creating a policy path outside the audit root", () => { + const root = tmp(); + const auditDir = join(root, "audit"); + const outside = join(root, "outside"); + mkdirSync(auditDir); + mkdirSync(outside); + symlinkSync(outside, join(auditDir, "2026")); + + expect(writePolicyTrace({ auditDir, trace: emptyTrace(), now: NOW })).toEqual({ + status: "error", + }); + expect(existsSync(join(outside, "08"))).toBe(false); + expect(existsSync(join(auditDir, "2026", "08", "10", "policy"))).toBe(false); + expect(allDescendants(outside)).toEqual([]); + }); + + it("rejects a symlinked audit root without mutating its target", () => { + const root = tmp(); + const outside = join(root, "outside"); + const auditDir = join(root, "audit"); + mkdirSync(outside); + symlinkSync(outside, auditDir); + + expect(writePolicyTrace({ auditDir, trace: emptyTrace(), now: NOW })).toEqual({ + status: "error", + }); + expect(allDescendants(outside)).toEqual([]); + }); + + it("never follows or replaces a pre-existing final symlink, hardlink, or wrong file", () => { + const trace = emptyTrace(); + + const symlinkRoot = tmp(); + const symlinkAudit = join(symlinkRoot, "audit"); + const symlinkArtifact = artifactDescriptor(symlinkAudit, trace); + const symlinkVictim = join(symlinkRoot, "victim.json"); + mkdirSync(dirname(symlinkArtifact.path), { recursive: true }); + writeFileSync(symlinkVictim, "victim-must-not-change"); + symlinkSync(symlinkVictim, symlinkArtifact.path); + expect(writePolicyTrace({ auditDir: symlinkAudit, trace, now: NOW })).toEqual({ + status: "error", + }); + expect(lstatSync(symlinkArtifact.path).isSymbolicLink()).toBe(true); + expect(readFileSync(symlinkVictim, "utf8")).toBe("victim-must-not-change"); + + const hardlinkRoot = tmp(); + const hardlinkAudit = join(hardlinkRoot, "audit"); + const hardlinkArtifact = artifactDescriptor(hardlinkAudit, trace); + const hardlinkVictim = join(hardlinkRoot, "victim.json"); + mkdirSync(dirname(hardlinkArtifact.path), { recursive: true }); + writeFileSync(hardlinkVictim, "hardlink-must-not-change"); + linkSync(hardlinkVictim, hardlinkArtifact.path); + expect(writePolicyTrace({ auditDir: hardlinkAudit, trace, now: NOW })).toEqual({ + status: "error", + }); + expect(readFileSync(hardlinkArtifact.path, "utf8")).toBe("hardlink-must-not-change"); + expect(readFileSync(hardlinkVictim, "utf8")).toBe("hardlink-must-not-change"); + + const wrongRoot = tmp(); + const wrongAudit = join(wrongRoot, "audit"); + const wrongArtifact = artifactDescriptor(wrongAudit, trace); + mkdirSync(dirname(wrongArtifact.path), { recursive: true }); + writeFileSync(wrongArtifact.path, "wrong-existing-content"); + const wrongInode = statSync(wrongArtifact.path).ino; + expect(writePolicyTrace({ auditDir: wrongAudit, trace, now: NOW })).toEqual({ + status: "error", + }); + expect(readFileSync(wrongArtifact.path, "utf8")).toBe("wrong-existing-content"); + expect(statSync(wrongArtifact.path).ino).toBe(wrongInode); + }); + + it("reuses a pre-existing exact regular artifact without replacing its inode", () => { + const auditDir = join(tmp(), "audit"); + const trace = emptyTrace(); + const artifact = artifactDescriptor(auditDir, trace); + mkdirSync(dirname(artifact.path), { recursive: true }); + writeFileSync(artifact.path, artifact.canonical, { mode: 0o600 }); + const inode = statSync(artifact.path).ino; + + expect(writePolicyTrace({ auditDir, trace, now: NOW })).toEqual({ + status: "complete", + ref: artifact.ref, + sha256: artifact.sha256, + }); + expect(statSync(artifact.path).ino).toBe(inode); + expect(readFileSync(artifact.path, "utf8")).toBe(artifact.canonical); + }); }); describe("policy trace reference security", () => { @@ -280,4 +393,83 @@ describe("policy trace reference security", () => { status: "overflow", }); }); + + it("rejects invalid UTF-8 even when the raw hash, filename, and lossy JSON are consistent", () => { + const auditDir = join(tmp(), "audit"); + const trace = emptyTrace("invalid-\uFFFD-byte"); + const canonical = Buffer.from(canonicalJson(trace), "utf8"); + const replacement = Buffer.from("\uFFFD", "utf8"); + const replacementAt = canonical.indexOf(replacement); + expect(replacementAt).toBeGreaterThan(-1); + const raw = Buffer.concat([ + canonical.subarray(0, replacementAt), + Buffer.from([0xff]), + canonical.subarray(replacementAt + replacement.length), + ]); + expect(raw.toString("utf8")).toBe(canonical.toString("utf8")); + const artifact = writeUncheckedArtifact(auditDir, trace, raw); + + const verified = verifyPolicyTraceReference({ + auditDir, + ref: artifact.ref, + sha256: artifact.sha256, + }); + expect(verified.ok).toBe(false); + if (!verified.ok) expect(verified.reason).toBe("invalid-encoding"); + }); + + it("rejects a schema-valid hash-consistent artifact above the verification byte limit", () => { + const auditDir = join(tmp(), "audit"); + const trace = maximumSignatureTrace(170); + const bytes = Buffer.from(canonicalJson(trace), "utf8"); + expect(bytes.length).toBe(1_053_027); + const artifact = writeUncheckedArtifact(auditDir, trace, bytes); + + const verified = verifyPolicyTraceReference({ + auditDir, + ref: artifact.ref, + sha256: artifact.sha256, + }); + expect(verified.ok).toBe(false); + if (!verified.ok) expect(verified.reason).toBe("too-large"); + }); + + it("rejects final policy files that are symlinks or have another hardlink", () => { + const symlinkAudit = join(tmp(), "audit"); + const symlinkStored = writePolicyTrace({ + auditDir: symlinkAudit, + trace: emptyTrace(), + now: NOW, + }); + if (symlinkStored.status !== "complete") throw new Error("fixture trace did not persist"); + const symlinkArtifact = join(symlinkAudit, ...symlinkStored.ref.split("/")); + const containedTarget = join(symlinkAudit, "contained-copy.json"); + writeFileSync(containedTarget, readFileSync(symlinkArtifact)); + rmSync(symlinkArtifact); + symlinkSync(containedTarget, symlinkArtifact); + const symlinkVerified = verifyPolicyTraceReference({ + auditDir: symlinkAudit, + ref: symlinkStored.ref, + sha256: symlinkStored.sha256, + }); + expect(symlinkVerified.ok).toBe(false); + if (!symlinkVerified.ok) expect(symlinkVerified.reason).toBe("not-a-file"); + + const hardlinkAudit = join(tmp(), "audit"); + const hardlinkStored = writePolicyTrace({ + auditDir: hardlinkAudit, + trace: emptyTrace(), + now: NOW, + }); + if (hardlinkStored.status !== "complete") throw new Error("fixture trace did not persist"); + const hardlinkArtifact = join(hardlinkAudit, ...hardlinkStored.ref.split("/")); + linkSync(hardlinkArtifact, join(hardlinkAudit, "alias.json")); + const hardlinkVerified = verifyPolicyTraceReference({ + auditDir: hardlinkAudit, + ref: hardlinkStored.ref, + sha256: hardlinkStored.sha256, + }); + expect(hardlinkVerified.ok).toBe(false); + if (!hardlinkVerified.ok) expect(hardlinkVerified.reason).toBe("not-a-file"); + }); }); From d76e591e851667180d28fa0eaae280a967b38361 Mon Sep 17 00:00:00 2001 From: Codevena Date: Mon, 10 Aug 2026 10:09:27 +0200 Subject: [PATCH 37/55] fix(audit): tolerate verified concurrent partitions --- src/audit/policy-trace-store.ts | 13 +- tests/unit/policy-trace-store.test.ts | 198 +++++++++++++++++++++++++- 2 files changed, 208 insertions(+), 3 deletions(-) diff --git a/src/audit/policy-trace-store.ts b/src/audit/policy-trace-store.ts index de5cb7f..2a18fe3 100644 --- a/src/audit/policy-trace-store.ts +++ b/src/audit/policy-trace-store.ts @@ -72,11 +72,20 @@ function realNonSymlinkDirectory(path: string): string | null { return realpathSync(path); } +function mkdirIfMissing(path: string): void { + if (existsSync(path)) return; + try { + mkdirSync(path, { mode: 0o700 }); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "EEXIST") throw error; + } +} + function ensureAuditRoot(auditRoot: string): string | null { if (!existsSync(auditRoot)) { const parent = dirname(auditRoot); if (realNonSymlinkDirectory(parent) === null) return null; - mkdirSync(auditRoot, { mode: 0o700 }); + mkdirIfMissing(auditRoot); } return realNonSymlinkDirectory(auditRoot); } @@ -90,7 +99,7 @@ function ensureContainedDirectory( const realParent = realNonSymlinkDirectory(parent); if (realParent === null || !isContained(realAuditRoot, realParent)) return null; const path = join(parent, name); - if (!existsSync(path)) mkdirSync(path, { mode: 0o700 }); + mkdirIfMissing(path); const realPath = realNonSymlinkDirectory(path); if (realPath === null || !isContained(realAuditRoot, realPath)) return null; if (!isContained(auditRoot, path)) return null; diff --git a/tests/unit/policy-trace-store.test.ts b/tests/unit/policy-trace-store.test.ts index 1a2e99a..9512206 100644 --- a/tests/unit/policy-trace-store.test.ts +++ b/tests/unit/policy-trace-store.test.ts @@ -15,10 +15,11 @@ import { writeFileSync, } from "node:fs"; import { tmpdir } from "node:os"; -import { dirname, join } from "node:path"; +import { basename, dirname, join } from "node:path"; import { canonicalJson } from "../../src/audit/canonical.ts"; import { POLICY_TRACE_MAX_BYTES, + type PolicyTraceWriteResult, verifyPolicyTraceReference, writePolicyTrace, } from "../../src/audit/policy-trace-store.ts"; @@ -172,6 +173,127 @@ function artifactDescriptor(auditDir: string, trace: PolicyTrace) { return { canonical, ref, sha256: sha, path: join(auditDir, ...ref.split("/")) }; } +async function runSynchronizedWriters( + inputs: Array<{ auditDir: string; trace: PolicyTrace }>, +): Promise { + const barrierDir = tmp("rg-policy-writer-barrier-"); + const startPath = join(barrierDir, "start"); + const childSource = ` + const input = JSON.parse(process.env.RG_POLICY_WRITER_INPUT); + const fs = require("node:fs"); + const mkdirSync = fs.mkdirSync; + const wait = new Int32Array(new SharedArrayBuffer(Int32Array.BYTES_PER_ELEMENT)); + fs.mkdirSync = (path, options) => { + if (path === input.auditDir) { + fs.writeFileSync(input.readyPath, "ready"); + while (!fs.existsSync(input.startPath)) Atomics.wait(wait, 0, 0, 1); + } + try { + return mkdirSync(path, options); + } finally { + if (path === input.auditDir) Atomics.wait(wait, 0, 0, input.releaseDelayMs); + } + }; + const { writePolicyTrace } = await import(process.env.RG_POLICY_STORE_URL); + process.stdout.write(JSON.stringify(writePolicyTrace({ + auditDir: input.auditDir, + trace: input.trace, + now: new Date(input.now), + }))); + `; + const storeUrl = new URL("../../src/audit/policy-trace-store.ts", import.meta.url).href; + const children = inputs.map((input, index) => { + const payload = JSON.stringify({ + ...input, + now: NOW.toISOString(), + readyPath: join(barrierDir, `ready-${index}`), + releaseDelayMs: index * 4, + startPath, + }); + return Bun.spawn([process.execPath, "-e", childSource], { + env: { + ...process.env, + RG_POLICY_STORE_URL: storeUrl, + RG_POLICY_WRITER_INPUT: payload, + }, + stdout: "pipe", + stderr: "pipe", + }); + }); + + try { + const readyDeadline = Date.now() + 15_000; + while (readdirSync(barrierDir).length !== children.length) { + if (Date.now() >= readyDeadline) throw new Error("parallel writer ready barrier timed out"); + await Bun.sleep(5); + } + writeFileSync(startPath, "start"); + + return await Promise.all( + children.map(async (child) => { + const [stdout, stderr, exitCode] = await Promise.all([ + new Response(child.stdout).text(), + new Response(child.stderr).text(), + child.exited, + ]); + if (exitCode !== 0) throw new Error(`parallel writer exited ${exitCode}: ${stderr}`); + return JSON.parse(stdout) as PolicyTraceWriteResult; + }), + ); + } catch (error) { + for (const child of children) child.kill(); + await Promise.allSettled(children.map((child) => child.exited)); + throw error; + } +} + +async function runWriterWithCreatedRootFault( + auditDir: string, + trace: PolicyTrace, +): Promise { + const storeUrl = new URL("../../src/audit/policy-trace-store.ts", import.meta.url).href; + const childSource = ` + const input = JSON.parse(process.env.RG_POLICY_WRITER_INPUT); + const fs = require("node:fs"); + const mkdirSync = fs.mkdirSync; + fs.mkdirSync = (path, options) => { + if (path === input.auditDir) { + mkdirSync(path, options); + const error = new Error("injected non-EEXIST mkdir failure"); + error.code = "EACCES"; + throw error; + } + return mkdirSync(path, options); + }; + const { writePolicyTrace } = await import(process.env.RG_POLICY_STORE_URL); + process.stdout.write(JSON.stringify(writePolicyTrace({ + auditDir: input.auditDir, + trace: input.trace, + now: new Date(input.now), + }))); + `; + const child = Bun.spawn([process.execPath, "-e", childSource], { + env: { + ...process.env, + RG_POLICY_STORE_URL: storeUrl, + RG_POLICY_WRITER_INPUT: JSON.stringify({ + auditDir, + trace, + now: NOW.toISOString(), + }), + }, + stdout: "pipe", + stderr: "pipe", + }); + const [stdout, stderr, exitCode] = await Promise.all([ + new Response(child.stdout).text(), + new Response(child.stderr).text(), + child.exited, + ]); + if (exitCode !== 0) throw new Error(`faulted writer exited ${exitCode}: ${stderr}`); + return JSON.parse(stdout) as PolicyTraceWriteResult; +} + describe("canonical policy trace storage", () => { it("keeps the audit canonicalizer sorted recursively and byte-stable", () => { expect(canonicalJson({ z: 1, a: { d: 4, b: 2 }, c: [2, { y: 1, x: 0 }] })).toBe( @@ -329,6 +451,80 @@ describe("canonical policy trace storage", () => { expect(statSync(artifact.path).ino).toBe(inode); expect(readFileSync(artifact.path, "utf8")).toBe(artifact.canonical); }); + + it("lets synchronized identical writers share a freshly-created verified partition", async () => { + const root = tmp(); + const auditDir = join(root, "audit"); + const outside = join(root, "outside"); + mkdirSync(outside); + const trace = emptyTrace("parallel-identical"); + const artifact = artifactDescriptor(auditDir, trace); + + const results = await runSynchronizedWriters( + Array.from({ length: 64 }, () => ({ auditDir, trace })), + ); + + expect(results).toEqual( + Array.from({ length: 64 }, () => ({ + status: "complete", + ref: artifact.ref, + sha256: artifact.sha256, + })), + ); + expect(readdirSync(dirname(artifact.path))).toEqual([basename(artifact.path)]); + expect(readFileSync(artifact.path, "utf8")).toBe(artifact.canonical); + const finalStat = lstatSync(artifact.path); + expect(finalStat.mode & 0o777).toBe(0o600); + expect(finalStat.nlink).toBe(1); + expect(allDescendants(root).some((path) => path.endsWith(".tmp"))).toBe(false); + expect(allDescendants(outside)).toEqual([]); + }, 30_000); + + it("lets synchronized distinct writers populate one freshly-created verified partition", async () => { + const root = tmp(); + const auditDir = join(root, "audit"); + const outside = join(root, "outside"); + mkdirSync(outside); + const traces = Array.from({ length: 32 }, (_, index) => + emptyTrace(`parallel-distinct-${index}`), + ); + const artifacts = traces.map((trace) => artifactDescriptor(auditDir, trace)); + + const results = await runSynchronizedWriters(traces.map((trace) => ({ auditDir, trace }))); + + expect(results.every((result) => result.status === "complete")).toBe(true); + const complete = results.filter( + (result): result is Extract => + result.status === "complete", + ); + expect(new Set(complete.map(({ ref }) => ref)).size).toBe(traces.length); + expect(new Set(complete.map(({ sha256 }) => sha256)).size).toBe(traces.length); + const firstArtifact = artifacts[0]; + if (firstArtifact === undefined) throw new Error("parallel fixture was empty"); + expect(readdirSync(dirname(firstArtifact.path)).sort()).toEqual( + artifacts.map(({ path }) => basename(path)).sort(), + ); + for (const artifact of artifacts) { + expect(readFileSync(artifact.path, "utf8")).toBe(artifact.canonical); + const finalStat = lstatSync(artifact.path); + expect(finalStat.mode & 0o777).toBe(0o600); + expect(finalStat.nlink).toBe(1); + } + expect(allDescendants(root).some((path) => path.endsWith(".tmp"))).toBe(false); + expect(allDescendants(outside)).toEqual([]); + }, 30_000); + + it("does not forgive a non-EEXIST mkdir failure when the root appeared", async () => { + const root = tmp(); + const auditDir = join(root, "audit"); + + expect(await runWriterWithCreatedRootFault(auditDir, emptyTrace())).toEqual({ + status: "error", + }); + expect(lstatSync(auditDir).isDirectory()).toBe(true); + expect(existsSync(join(auditDir, "2026"))).toBe(false); + expect(allDescendants(root).some((path) => path.endsWith(".tmp"))).toBe(false); + }); }); describe("policy trace reference security", () => { From 1611fa38140b6e4d6b69e9fcae11be085956b788 Mon Sep 17 00:00:00 2001 From: Codevena Date: Mon, 10 Aug 2026 10:47:44 +0200 Subject: [PATCH 38/55] feat(bench): run exact policy ablations --- src/bench/report.ts | 38 +++ src/bench/runner.ts | 280 ++++++++++++++++ src/cli/commands/bench.ts | 444 +++++++++++++++++++------ src/schemas/bench-result.ts | 147 ++++++++ tests/unit/bench-matrix.test.ts | 278 ++++++++++++++-- tests/unit/bench-report.test.ts | 81 ++++- tests/unit/bench-result-schema.test.ts | 183 +++++++++- 7 files changed, 1328 insertions(+), 123 deletions(-) diff --git a/src/bench/report.ts b/src/bench/report.ts index ac3b19b..3de90c0 100644 --- a/src/bench/report.ts +++ b/src/bench/report.ts @@ -255,12 +255,38 @@ function fmtPoint(m: Metric): string { */ export function renderBenchMatrix(matrix: BenchMatrix): string { const p = matrix.provenance; + const policyRows = matrix.variants.flatMap((variant) => + variant.policy === undefined ? [] : [{ label: variant.label, policy: variant.policy }], + ); + const policyCatalogs = [...new Set(policyRows.map((row) => row.policy.catalog_version))]; + const invalidPolicyRows = policyRows.filter((row) => !row.policy.authoritative); + const legacyPolicyRows = matrix.variants + .filter((variant) => variant.policy === undefined) + .map((variant) => ({ + label: variant.label, + reason: "legacy result has no policy trace provenance", + })); + const policyInvalidities = [ + ...invalidPolicyRows.map((row) => ({ + label: row.label, + reason: row.policy.reason ?? `trace status ${row.policy.trace_status}`, + })), + ...legacyPolicyRows, + ]; const L: string[] = []; L.push("Reviewgate bench matrix — ablation (baseline = full suppression)"); L.push("================================================================"); L.push( `roster: ${p.providers.map((r) => r.id).join(", ")} · repeat ${p.repeat} · corpus ${p.corpus_commit}${p.corpus_dirty ? " (dirty)" : ""}`, ); + if (policyCatalogs.length > 0) L.push(`policy catalog: ${policyCatalogs.join(", ")}`); + if (matrix.authoritative === false || policyInvalidities.length > 0) { + L.push(""); + L.push("⚠ NON-AUTHORITATIVE policy matrix:"); + for (const row of policyInvalidities) { + L.push(` · ${row.label}: ${row.reason}`); + } + } L.push(""); const head = ` ${pad("variant", 16)} ${pad("class", 6)} ${pad("precision", 10)} ${pad("recall", 8)} ${pad("clean-FP", 9)} ${pad("Δprec", 7)} ${pad("Δrecall", 8)} Δcleanfp`; L.push(head); @@ -278,6 +304,18 @@ export function renderBenchMatrix(matrix: BenchMatrix): string { const M: string[] = []; M.push("### Reviewgate bench — ablation matrix"); M.push(""); + if (policyCatalogs.length > 0) { + M.push(`_Policy catalog: \`${policyCatalogs.join(", ")}\`._`); + M.push(""); + } + if (matrix.authoritative === false || policyInvalidities.length > 0) { + M.push( + `> ⚠ **NON-AUTHORITATIVE policy matrix** — ${policyInvalidities + .map((row) => `${row.label}: ${row.reason}`) + .join("; ")}.`, + ); + M.push(""); + } M.push("| variant | class | precision | recall | clean-FP | Δprec | Δrecall | Δclean-FP |"); M.push("| --- | --- | --- | --- | --- | --- | --- | --- |"); for (const v of matrix.variants) { diff --git a/src/bench/runner.ts b/src/bench/runner.ts index 3c02e2a..fbd1f5d 100644 --- a/src/bench/runner.ts +++ b/src/bench/runner.ts @@ -7,6 +7,7 @@ // diffs are UNTRUSTED, so the case is hydrated defensively (path-safety + git apply // to an empty tree) and anything unparseable / unsafe / non-applyable is `invalid`. +import { createHash } from "node:crypto"; import { existsSync, lstatSync, @@ -19,10 +20,17 @@ import { } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; +import { canonicalJson } from "../audit/canonical.ts"; import { buildAdapters } from "../cli/build-adapters.ts"; import { defaultConfig } from "../config/defaults.ts"; import { ConfigSchema, type ReviewgateConfig } from "../config/define-config.ts"; import { Orchestrator } from "../core/orchestrator.ts"; +import { + POLICY_CATALOG_VERSION, + POLICY_PASS_IDS, + type PolicyPassId, +} from "../core/policy/catalog.ts"; +import type { PolicyExecutionOptions } from "../core/policy/replay.ts"; import type { OpenRouterProviderRouting, ProviderAdapter, @@ -32,6 +40,7 @@ import type { ProviderId } from "../providers/registry.ts"; import type { BenchCase } from "../schemas/bench-case.ts"; import type { Finding } from "../schemas/finding.ts"; import { type PendingReport, PendingReportSchema } from "../schemas/pending-report.ts"; +import { type PolicyTrace, PolicyTraceSchema } from "../schemas/policy-trace.ts"; import type { GitInfo } from "../utils/git.ts"; import { planReviewJsonPath, reviewgateDir } from "../utils/paths.ts"; import { spawnCapture } from "../utils/spawn-capture.ts"; @@ -210,6 +219,197 @@ export interface CaseRunOutcome { verdicts: number; demoted: number; }; + /** Complete in-memory policy trace identity for authoritative matrix pairing. */ + policy?: AuthoritativeTraceRun; +} + +export type AuthoritativeTraceInvalidityCode = + | "missing-trace" + | "missing-pass-row" + | "missing-counter" + | "pass-not-run" + | "trace-status" + | "trace-reference" + | "trace-hash" + | "catalog-mismatch" + | "requested-pass-mismatch" + | "response-hash-mismatch" + | "request-identity-mismatch" + | "config-mismatch" + | "final-identity-mismatch" + | "non-authoritative-execution" + | "invalid-trace"; + +export interface AuthoritativeTraceRun { + authoritative: boolean; + status: "complete" | "not-run" | "error" | "overflow"; + catalogVersion: string; + requestedAblations: PolicyPassId[]; + trace?: PolicyTrace | undefined; + traceRef?: string | undefined; + traceSha256?: string | undefined; + requestIdentitySha256: string; + effectiveConfigSha256: string; + finalIdentitySha256: string; +} + +export type AuthoritativeTracePairValidation = + | { ok: true } + | { ok: false; code: AuthoritativeTraceInvalidityCode; reason: string }; + +const TRACE_COUNTERS = [ + "considered", + "opportunities", + "would_apply", + "applied", + "protected", + "blocking_removed", + "blocking_preserved", + "dropped", +] as const; + +function invalidTracePair( + code: AuthoritativeTraceInvalidityCode, + reason: string, +): AuthoritativeTracePairValidation { + return { ok: false, code, reason }; +} + +function sameStrings(left: readonly string[], right: readonly string[]): boolean { + return left.length === right.length && left.every((value, index) => value === right[index]); +} + +function sha256Text(value: string): string { + return createHash("sha256").update(Buffer.from(value, "utf8")).digest("hex"); +} + +/** Fail-closed validation used before an ablation pair can contribute metrics. */ +export function validateAuthoritativeTracePair( + baseline: AuthoritativeTraceRun, + counterfactual: AuthoritativeTraceRun, +): AuthoritativeTracePairValidation { + const runs = [ + ["baseline", baseline], + ["counterfactual", counterfactual], + ] as const; + + for (const [label, run] of runs) { + if (!run.authoritative) { + return invalidTracePair( + "non-authoritative-execution", + `${label} policy execution was not authoritative`, + ); + } + if (run.trace === undefined) { + return invalidTracePair("missing-trace", `${label} policy trace is missing`); + } + if (run.status !== "complete") { + return invalidTracePair("trace-status", `${label} policy trace status is ${run.status}`); + } + if ( + run.catalogVersion !== POLICY_CATALOG_VERSION || + run.trace.catalog_version !== POLICY_CATALOG_VERSION + ) { + return invalidTracePair( + "catalog-mismatch", + `${label} policy catalog is not ${POLICY_CATALOG_VERSION}`, + ); + } + if ( + run.trace.passes.length !== POLICY_PASS_IDS.length || + POLICY_PASS_IDS.some((passId, index) => run.trace?.passes[index]?.pass_id !== passId) + ) { + return invalidTracePair( + "missing-pass-row", + `${label} policy trace does not contain the exact configured pass inventory`, + ); + } + for (const row of run.trace.passes) { + if (row.status !== "ran") continue; + const record = row as unknown as Record; + const missing = TRACE_COUNTERS.find((counter) => typeof record[counter] !== "number"); + if (missing !== undefined) { + return invalidTracePair( + "missing-counter", + `${label} pass ${row.pass_id} is missing numeric counter ${missing}`, + ); + } + } + if (!sameStrings(run.requestedAblations, run.trace.ablated)) { + return invalidTracePair( + "requested-pass-mismatch", + `${label} requested ablations do not match its trace`, + ); + } + for (const passId of run.requestedAblations) { + const row = run.trace.passes.find((candidate) => candidate.pass_id === passId); + if (row?.status !== "ran") { + return invalidTracePair("pass-not-run", `${label} requested pass ${passId} did not run`); + } + } + const parsed = PolicyTraceSchema.safeParse(run.trace); + if (!parsed.success) { + return invalidTracePair( + "invalid-trace", + `${label} policy trace is invalid: ${parsed.error.issues[0]?.message ?? "schema mismatch"}`, + ); + } + } + + if (!sameStrings(baseline.requestedAblations, [])) { + return invalidTracePair( + "requested-pass-mismatch", + "baseline must not request a policy ablation", + ); + } + if (counterfactual.requestedAblations.length !== 1) { + return invalidTracePair( + "requested-pass-mismatch", + "counterfactual must request exactly one policy ablation", + ); + } + if (baseline.effectiveConfigSha256 !== counterfactual.effectiveConfigSha256) { + return invalidTracePair("config-mismatch", "effective policy configs differ across the pair"); + } + if (baseline.requestIdentitySha256 !== counterfactual.requestIdentitySha256) { + return invalidTracePair( + "request-identity-mismatch", + "review request identities differ across the pair", + ); + } + const baselineTrace = baseline.trace as PolicyTrace; + const counterfactualTrace = counterfactual.trace as PolicyTrace; + if (!sameStrings(baselineTrace.raw_response_sha256, counterfactualTrace.raw_response_sha256)) { + return invalidTracePair( + "response-hash-mismatch", + "ordered raw response hashes differ across the pair", + ); + } + + for (const [label, run] of runs) { + const trace = run.trace as PolicyTrace; + if (run.finalIdentitySha256 !== sha256Text(canonicalJson(trace.final))) { + return invalidTracePair( + "final-identity-mismatch", + `${label} final finding identity does not match its trace`, + ); + } + if (run.traceRef === undefined || run.traceSha256 === undefined) { + return invalidTracePair( + "trace-reference", + `${label} complete policy trace is missing its inline reference or hash`, + ); + } + const actualSha256 = sha256Text(canonicalJson(trace)); + if (run.traceSha256 !== actualSha256) { + return invalidTracePair("trace-hash", `${label} policy trace content hash mismatches`); + } + if (run.traceRef !== `inline-policy-trace/${actualSha256}.json`) { + return invalidTracePair("trace-reference", `${label} policy trace reference mismatches`); + } + } + + return { ok: true }; } export interface RunBenchCaseInput { @@ -227,6 +427,72 @@ export interface RunBenchCaseInput { providerAvailable?: (id: ProviderId, apiKeyEnv?: string) => boolean; /** Benchmark-only critic completion limit; omitted means the runtime default (1). */ criticMaxAttempts?: number; + /** Bench/Rig-only trace and ablation control. Normal gate/config paths never set this. */ + policyExecution?: PolicyExecutionOptions; +} + +function buildAuthoritativeTraceRun(input: { + trace: PolicyTrace | undefined; + execution: PolicyExecutionOptions; + config: ReviewgateConfig; + benchCase: BenchCase; + diffPatch: string; + report: PendingReport; + verdict: "PASS" | "SOFT-PASS" | "FAIL" | "ERROR"; +}): AuthoritativeTraceRun { + const requestedAblations = [...input.execution.policyAblations].sort( + (left, right) => POLICY_PASS_IDS.indexOf(left) - POLICY_PASS_IDS.indexOf(right), + ); + const finalIdentity = { + verdict: input.verdict, + counts: { + critical: input.report.counts.critical, + warn: input.report.counts.warn, + info: input.report.counts.info, + }, + finding_signatures: input.report.findings.map((finding) => finding.signature), + finding_severities: input.report.findings.map((finding) => ({ + signature: finding.signature, + severity: finding.severity, + })), + }; + const effectiveConfigSha256 = sha256Text(canonicalJson(input.config)); + const requestIdentitySha256 = sha256Text( + canonicalJson({ + bench_case: input.benchCase, + diff_sha256: sha256Text(input.diffPatch), + git: FIXED_SYNTHETIC_GIT_INFO, + reviewers: input.config.phases.review.reviewers, + grounding: input.config.phases.grounding, + critic: input.config.phases.critic, + effective_config_sha256: effectiveConfigSha256, + state: "per-case-fresh", + }), + ); + if (input.trace === undefined) { + return { + authoritative: input.execution.authoritative, + status: "not-run", + catalogVersion: POLICY_CATALOG_VERSION, + requestedAblations, + requestIdentitySha256, + effectiveConfigSha256, + finalIdentitySha256: sha256Text(canonicalJson(finalIdentity)), + }; + } + const traceSha256 = sha256Text(canonicalJson(input.trace)); + return { + authoritative: input.execution.authoritative, + status: "complete", + catalogVersion: input.trace.catalog_version, + requestedAblations, + trace: input.trace, + traceRef: `inline-policy-trace/${traceSha256}.json`, + traceSha256, + requestIdentitySha256, + effectiveConfigSha256, + finalIdentitySha256: sha256Text(canonicalJson(finalIdentity)), + }; } /** Adapt a persisted Finding to the matcher's shape. Index-derived id guarantees @@ -323,6 +589,7 @@ export async function runBenchCase(input: RunBenchCaseInput): Promise Date; /** injectable quota-failover availability probe (tests); production probes real CLIs. */ providerAvailable?: (id: ProviderId, apiKeyEnv?: string) => boolean; + /** Internal matrix-only policy trace/ablation options. */ + policyExecution?: PolicyExecutionOptions; } export interface BenchRunnerInfo { @@ -427,6 +438,25 @@ function outcomeToCaseResult( latency_ms: out.latencyMs, error: out.error, ...(out.critic ? { critic: out.critic } : {}), + ...(out.policy + ? { + policy_trace: { + authoritative: out.policy.authoritative, + status: out.policy.status, + catalog_version: out.policy.catalogVersion, + requested_ablations: out.policy.requestedAblations, + ...(out.policy.trace === undefined ? {} : { trace: out.policy.trace }), + ...(out.policy.traceRef === undefined ? {} : { trace_ref: out.policy.traceRef }), + ...(out.policy.traceSha256 === undefined + ? {} + : { trace_sha256: out.policy.traceSha256 }), + request_identity_sha256: out.policy.requestIdentitySha256, + effective_config_sha256: out.policy.effectiveConfigSha256, + final_identity_sha256: out.policy.finalIdentitySha256, + reason: out.policy.authoritative ? null : `policy trace status ${out.policy.status}`, + }, + } + : {}), }; } @@ -690,6 +720,7 @@ async function runBenchRunInternal(input: BenchRunInput): Promise = { - critic: { klass: "A", off: { critic: null } }, - "confidence-floor": { klass: "A", off: { confidenceFloor: 0 } }, - reputation: { klass: "A", off: { reputation: false } }, - "scope-to-diff": { klass: "A", off: { scopeToDiff: false } }, +/** Legacy CLI labels accepted at the boundary and normalized to the closed policy catalog. */ +const MATRIX_ABLATION_ALIASES: Readonly> = { + critic: "judgment.critic", + "confidence-floor": "judgment.confidence", + reputation: "judgment.reputation", + "scope-to-diff": "scope.diff", }; +function normalizeMatrixAblation(value: string): PolicyPassId | null { + const alias = MATRIX_ABLATION_ALIASES[value]; + if (alias !== undefined) return alias; + return (POLICY_PASS_IDS as readonly string[]).includes(value) ? (value as PolicyPassId) : null; +} + export interface BenchMatrixInput { repoRoot: string; corpus: string; @@ -1192,16 +1227,21 @@ export function validateMatrixPreregistration( interface CapturedReviewEntry { provider: ProviderId; - reviewer_id: string; + kind: "review" | "complete"; ordinal: number; request_sha256: string; response_sha256: string; + outcome: "return" | "throw"; } interface ReviewCaptureState { entries: CapturedReviewEntry[]; - responses: Map; - ordinals: Map; + responses: Map< + number, + { kind: "review"; value: ReviewResult } | { kind: "complete"; value: string } + >; + errors: Map; + nextOrdinal: number; mismatch: string | null; } @@ -1214,8 +1254,10 @@ function normalizedReview(result: ReviewResult): ReviewResult { durationMs: result.durationMs, exitCode: result.exitCode, rawEventsPath: "", + ...(result.rawText === undefined ? {} : { rawText: result.rawText }), status: result.status, ...(result.statusDetail ? { statusDetail: result.statusDetail } : {}), + ...(result.quotaInferred === undefined ? {} : { quotaInferred: result.quotaInferred }), }; } @@ -1246,36 +1288,108 @@ function reviewRequestHash( ); } +function completionRequestHash( + provider: ProviderId, + ordinal: number, + prompt: string, + opts: Parameters>[1], +): string { + return sha256( + stableJson({ + provider, + kind: "complete", + ordinal, + prompt_sha256: sha256(prompt), + options: { + model: opts.model, + apiKeyEnv: opts.apiKeyEnv ?? null, + timeoutMs: opts.timeoutMs ?? null, + maxTokens: opts.maxTokens ?? null, + auth: opts.auth ?? null, + openrouterProvider: opts.openrouterProvider ?? null, + baseUrl: opts.baseUrl ?? null, + disableReasoning: opts.disableReasoning ?? null, + }, + }), + ); +} + function captureReviewerAdapters( adapters: Partial>, - reviewers: ReadonlySet, state: ReviewCaptureState, ): Partial> { - const out: Partial> = { ...adapters }; - for (const provider of reviewers) { - const adapter = adapters[provider]; + const out: Partial> = {}; + for (const [provider, adapter] of Object.entries(adapters) as Array< + [ProviderId, ProviderAdapter | undefined] + >) { if (!adapter) continue; const complete = adapter.complete?.bind(adapter); out[provider] = { id: adapter.id, preflight: (cfg) => adapter.preflight(cfg), async review(input) { - const ordinal = (state.ordinals.get(provider) ?? 0) + 1; - state.ordinals.set(provider, ordinal); + const ordinal = state.nextOrdinal++; const requestHash = reviewRequestHash(provider, ordinal, input); - const response = normalizedReview(await adapter.review(input)); - const responseHash = sha256(stableJson(response)); - state.responses.set(requestHash, response); - state.entries.push({ - provider, - reviewer_id: input.reviewerId, - ordinal, - request_sha256: requestHash, - response_sha256: responseHash, - }); - return structuredClone(response); + try { + const response = normalizedReview(await adapter.review(input)); + const responseHash = sha256(stableJson(response)); + state.responses.set(ordinal, { kind: "review", value: response }); + state.entries.push({ + provider, + kind: "review", + ordinal, + request_sha256: requestHash, + response_sha256: responseHash, + outcome: "return", + }); + return structuredClone(response); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + state.errors.set(ordinal, message); + state.entries.push({ + provider, + kind: "review", + ordinal, + request_sha256: requestHash, + response_sha256: sha256(message), + outcome: "throw", + }); + throw error; + } }, - ...(complete ? { complete: (prompt, opts) => complete(prompt, opts) } : {}), + ...(complete + ? { + async complete(prompt, opts) { + const ordinal = state.nextOrdinal++; + const requestHash = completionRequestHash(provider, ordinal, prompt, opts); + try { + const response = await complete(prompt, opts); + state.responses.set(ordinal, { kind: "complete", value: response }); + state.entries.push({ + provider, + kind: "complete", + ordinal, + request_sha256: requestHash, + response_sha256: sha256(response), + outcome: "return", + }); + return response; + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + state.errors.set(ordinal, message); + state.entries.push({ + provider, + kind: "complete", + ordinal, + request_sha256: requestHash, + response_sha256: sha256(message), + outcome: "throw", + }); + throw error; + } + }, + } + : {}), }; } return out; @@ -1283,28 +1397,50 @@ function captureReviewerAdapters( function replayReviewerAdapters( adapters: Partial>, - reviewers: ReadonlySet, capture: ReviewCaptureState, -): Partial> { - const out: Partial> = { ...adapters }; - const ordinals = new Map(); - for (const provider of reviewers) { - const adapter = adapters[provider]; +): { adapters: Partial>; consumed: () => boolean } { + const out: Partial> = {}; + let cursor = 0; + const mismatch = (message: string): void => { + capture.mismatch ??= message; + }; + for (const [provider, adapter] of Object.entries(adapters) as Array< + [ProviderId, ProviderAdapter | undefined] + >) { if (!adapter) continue; - const complete = adapter.complete?.bind(adapter); out[provider] = { id: adapter.id, preflight: (cfg) => adapter.preflight(cfg), async review(input) { - const ordinal = (ordinals.get(provider) ?? 0) + 1; - ordinals.set(provider, ordinal); + const ordinal = cursor++; const requestHash = reviewRequestHash(provider, ordinal, input); - const expected = capture.entries.find( - (entry) => entry.provider === provider && entry.ordinal === ordinal, - ); - const response = expected ? capture.responses.get(expected.request_sha256) : undefined; - if (!expected || expected.request_sha256 !== requestHash || !response) { - capture.mismatch = `${provider} reviewer request ${ordinal} did not match baseline`; + const expected = capture.entries[ordinal]; + const stored = capture.responses.get(ordinal); + if ( + !expected || + expected.kind !== "review" || + expected.provider !== provider || + expected.request_sha256 !== requestHash + ) { + mismatch(`${provider} review request ${ordinal} did not match baseline order/identity`); + return { + reviewerId: input.reviewerId, + verdict: "ERROR", + findings: [], + usage: { inputTokens: 0, outputTokens: 0, costUsd: 0, quotaUsedPct: null }, + durationMs: 0, + exitCode: 1, + rawEventsPath: "", + status: "error", + statusDetail: capture.mismatch ?? "replay mismatch", + }; + } + if (expected.outcome === "throw") throw new Error(capture.errors.get(ordinal)); + if ( + stored?.kind !== "review" || + sha256(stableJson(stored.value)) !== expected.response_sha256 + ) { + mismatch(`${provider} review response ${ordinal} failed baseline hash validation`); return { reviewerId: input.reviewerId, verdict: "ERROR", @@ -1314,15 +1450,53 @@ function replayReviewerAdapters( exitCode: 1, rawEventsPath: "", status: "error", - statusDetail: capture.mismatch, + statusDetail: capture.mismatch ?? "replay response mismatch", }; } - return structuredClone(response); + return structuredClone(stored.value); }, - ...(complete ? { complete: (prompt, opts) => complete(prompt, opts) } : {}), + ...(adapter.complete + ? { + async complete(prompt, opts) { + const ordinal = cursor++; + const requestHash = completionRequestHash(provider, ordinal, prompt, opts); + const expected = capture.entries[ordinal]; + const stored = capture.responses.get(ordinal); + if ( + !expected || + expected.kind !== "complete" || + expected.provider !== provider || + expected.request_sha256 !== requestHash + ) { + mismatch( + `${provider} complete request ${ordinal} did not match baseline order/identity`, + ); + throw new Error(capture.mismatch ?? "replay mismatch"); + } + if (expected.outcome === "throw") throw new Error(capture.errors.get(ordinal)); + if ( + stored?.kind !== "complete" || + sha256(stored.value) !== expected.response_sha256 + ) { + mismatch( + `${provider} complete response ${ordinal} failed baseline hash validation`, + ); + throw new Error(capture.mismatch ?? "replay response mismatch"); + } + return stored.value; + }, + } + : {}), }; } - return out; + return { + adapters: out, + consumed: () => { + if (cursor === capture.entries.length) return true; + mismatch(`replay consumed ${cursor}/${capture.entries.length} captured provider responses`); + return false; + }, + }; } function relativeArtifact(fromDir: string, path: string): string { @@ -1351,6 +1525,77 @@ function matrixVariantProvenanceMismatch(baseline: BenchResult, variant: BenchRe return reasons; } +function authoritativeTraceRunFromCase(caseResult: CaseResult): AuthoritativeTraceRun | null { + const policy = caseResult.policy_trace; + if (policy === undefined) return null; + return { + authoritative: policy.authoritative, + status: policy.status, + catalogVersion: policy.catalog_version, + requestedAblations: [...policy.requested_ablations], + ...(policy.trace === undefined ? {} : { trace: policy.trace }), + ...(policy.trace_ref === undefined ? {} : { traceRef: policy.trace_ref }), + ...(policy.trace_sha256 === undefined ? {} : { traceSha256: policy.trace_sha256 }), + requestIdentitySha256: policy.request_identity_sha256, + effectiveConfigSha256: policy.effective_config_sha256, + finalIdentitySha256: policy.final_identity_sha256, + }; +} + +function validateBenchResultTracePairs( + baseline: BenchResult, + counterfactual: BenchResult, +): { ok: true } | { ok: false; reason: string } { + if (baseline.cases.length !== counterfactual.cases.length) { + return { ok: false, reason: "case identity mismatch: result cardinality differs" }; + } + for (const [index, baselineCase] of baseline.cases.entries()) { + const counterfactualCase = counterfactual.cases[index]; + if ( + counterfactualCase === undefined || + baselineCase.id !== counterfactualCase.id || + (baselineCase.repeat ?? 1) !== (counterfactualCase.repeat ?? 1) || + baselineCase.content_hash !== counterfactualCase.content_hash + ) { + return { ok: false, reason: `case identity mismatch at row ${index}` }; + } + const baselineTrace = authoritativeTraceRunFromCase(baselineCase); + const counterfactualTrace = authoritativeTraceRunFromCase(counterfactualCase); + if (baselineTrace === null || counterfactualTrace === null) { + return { ok: false, reason: `missing-trace: case ${baselineCase.id}` }; + } + const validation = validateAuthoritativeTracePair(baselineTrace, counterfactualTrace); + if (!validation.ok) { + return { + ok: false, + reason: `${validation.code}: case ${baselineCase.id}: ${validation.reason}`, + }; + } + } + return { ok: true }; +} + +function matrixPolicyProvenance(result: BenchResult, ablatedPassId: PolicyPassId | null) { + const policyRows = result.cases.map((caseResult) => caseResult.policy_trace); + const complete = policyRows.length > 0 && policyRows.every((row) => row?.authoritative === true); + const rawResponseSha256 = policyRows.flatMap((row) => row?.trace?.raw_response_sha256 ?? []); + const traceSha256 = sha256(canonicalJson(policyRows)); + return { + catalog_version: POLICY_CATALOG_VERSION, + ablated_pass_id: ablatedPassId, + trace_status: complete ? ("complete" as const) : ("not-run" as const), + ...(complete + ? { + trace_ref: `inline-policy-trace-set/${traceSha256}.json`, + trace_sha256: traceSha256, + } + : {}), + raw_response_sha256: rawResponseSha256, + authoritative: complete, + reason: complete ? null : "one or more case traces are non-authoritative", + }; +} + /** * Ablation matrix (spec §8): run the corpus once as a BASELINE (full suppression) * and once per `--ablate` layer with that ONE layer turned off, then report the @@ -1361,23 +1606,26 @@ export async function runBenchMatrix(input: BenchMatrixInput): Promise !(a in MATRIX_ABLATIONS)); + const normalizedAblations = input.ablate.map(normalizeMatrixAblation); + const unknown = input.ablate.filter((_value, index) => normalizedAblations[index] === null); if (unknown.length > 0) { return { exitCode: 2, stdout: "", - stderr: `bench matrix: unknown ablation(s): ${unknown.join(",")} (known: ${Object.keys(MATRIX_ABLATIONS).join(",")})\n`, + stderr: `bench matrix: unknown ablation(s): ${unknown.join(",")} (known catalog IDs plus aliases: ${Object.keys(MATRIX_ABLATION_ALIASES).join(",")})\n`, }; } - if (input.authoritative && (input.ablate.length !== 1 || input.ablate[0] !== "critic")) { + const ablatedPassIds = normalizedAblations.filter( + (value): value is PolicyPassId => value !== null, + ); + if (new Set(ablatedPassIds).size !== ablatedPassIds.length) { return { exitCode: 2, stdout: "", - stderr: - "bench matrix: authoritative paired mode currently supports exactly --ablate critic\n", + stderr: "bench matrix: duplicate ablations resolve to the same policy catalog ID\n", }; } - if (input.ablate.includes("critic") && !input.criticProvider) { + if (ablatedPassIds.includes("judgment.critic") && !input.criticProvider) { return { exitCode: 2, stdout: "", @@ -1392,7 +1640,7 @@ export async function runBenchMatrix(input: BenchMatrixInput): Promise [layer, join(artifactDir, `no-${layer}.result.json`)]), + ablatedPassIds.map((passId) => [passId, join(artifactDir, `no-${passId}.result.json`)]), ); for (const path of [matrixPath, baselinePath, responseManifestPath, ...variantPaths.values()]) { if (existsSync(path)) { @@ -1457,31 +1705,22 @@ export async function runBenchMatrix(input: BenchMatrixInput): Promise [ - reviewer.provider, - ...(reviewer.fallback ?? []), - ]), - ); const underlying = buildAdapters(baselineConfig, input.adapters); const capture: ReviewCaptureState = { entries: [], responses: new Map(), - ordinals: new Map(), + errors: new Map(), + nextOrdinal: 0, mismatch: null, }; - const capturingAdapters = captureReviewerAdapters(underlying, reviewerIds, capture); + const capturingAdapters = captureReviewerAdapters(underlying, capture); const budget = createCallBudget(input.maxProviderCalls); const runnerInfo = input.runnerInfo ?? detectRunnerInfo(input.adapters); const work = mkdtempSync(join(tmpdir(), "rg-bench-matrix-")); try { const runVariant = async ( label: string, - suppressors: SuppressorConfig, - ablationLabels: string[], + requestedAblations: PolicyPassId[], adapters: Partial>, countProviderCalls: boolean, ): Promise<{ result?: BenchResult; output: BenchRunOutput; tempPath: string }> => { @@ -1518,12 +1757,17 @@ export async function runBenchMatrix(input: BenchMatrixInput): Promise left.ordinal - right.ordinal); const manifest = { schema: "reviewgate.bench.reviewer-response-hashes.v1", - entries: [...capture.entries].sort( - (a, b) => a.provider.localeCompare(b.provider) || a.ordinal - b.ordinal, - ), + entries: [...capture.entries], }; const responseManifestTempPath = join(work, "reviewer-responses.sha256.json"); writeFileSync(responseManifestTempPath, `${JSON.stringify(manifest, null, 2)}\n`); @@ -1572,24 +1809,18 @@ export async function runBenchMatrix(input: BenchMatrixInput): Promise = []; const completedVariantArtifacts: Array<{ tempPath: string; finalPath: string }> = []; - for (const layer of input.ablate) { - const spec = MATRIX_ABLATIONS[layer]; - if (!spec) continue; // validated above - const replayAdapters = replayReviewerAdapters(underlying, reviewerIds, capture); - const variantRun = await runVariant( - `no-${layer}`, - { ...baselineSuppressors, ...spec.off }, - [layer], - replayAdapters, - false, - ); - const finalPath = variantPaths.get(layer); - if (!finalPath) throw new Error(`missing artifact path for ${layer}`); + for (const passId of ablatedPassIds) { + const replay = replayReviewerAdapters(underlying, capture); + const variantRun = await runVariant(`no-${passId}`, [passId], replay.adapters, false); + replay.consumed(); + const finalPath = variantPaths.get(passId); + if (!finalPath) throw new Error(`missing artifact path for ${passId}`); if (variantRun.output.exitCode !== 0 || !variantRun.result || capture.mismatch) { mkdirSync(artifactDir, { recursive: true }); writeFileSync(baselinePath, readFileSync(baselineRun.tempPath)); @@ -1618,6 +1849,20 @@ export async function runBenchMatrix(input: BenchMatrixInput): Promise { + const completeFields = [value.trace, value.trace_ref, value.trace_sha256]; + if (value.status === "complete") { + if (completeFields.some((field) => field === undefined)) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + path: ["trace"], + message: "complete policy trace requires trace/ref/hash", + }); + } + if (value.trace !== undefined && value.trace_sha256 !== undefined) { + const actualSha256 = createHash("sha256") + .update(Buffer.from(canonicalJson(value.trace), "utf8")) + .digest("hex"); + if (value.trace_sha256 !== actualSha256) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + path: ["trace_sha256"], + message: "embedded policy trace hash mismatch", + }); + } + if (value.trace_ref !== `inline-policy-trace/${actualSha256}.json`) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + path: ["trace_ref"], + message: "embedded policy trace reference mismatch", + }); + } + const finalSha256 = createHash("sha256") + .update(Buffer.from(canonicalJson(value.trace.final), "utf8")) + .digest("hex"); + if (value.final_identity_sha256 !== finalSha256) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + path: ["final_identity_sha256"], + message: "embedded policy final identity mismatch", + }); + } + if ( + value.requested_ablations.length !== value.trace.ablated.length || + value.requested_ablations.some((passId, index) => passId !== value.trace?.ablated[index]) + ) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + path: ["requested_ablations"], + message: "requested ablations must equal the embedded trace profile", + }); + } + } + } else if (completeFields.some((field) => field !== undefined)) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + path: ["trace"], + message: `${value.status} policy trace forbids trace/ref/hash`, + }); + } + if (value.authoritative !== (value.status === "complete" && value.reason === null)) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + path: ["authoritative"], + message: "authoritative requires complete trace and no invalidity reason", + }); + } + if ( + value.authoritative && + (value.catalog_version !== POLICY_CATALOG_VERSION || + value.trace?.catalog_version !== POLICY_CATALOG_VERSION) + ) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + path: ["catalog_version"], + message: "authoritative trace requires the current policy catalog", + }); + } + }); + export const CaseResultSchema = z .object({ id: z.string(), @@ -211,6 +309,9 @@ export const CaseResultSchema = z latency_ms: z.number().nonnegative().nullable(), error: z.string().nullable(), critic: CaseCriticSchema.optional(), + // Additive: legacy BenchResult v1 rows without traces remain parseable, but + // exact policy matrix runs require this block before they can be scored. + policy_trace: BenchPolicyTraceRunSchema.optional(), }) .strict(); @@ -398,6 +499,51 @@ export const MatrixVariantSchema = z authoritative: z.boolean().optional(), result_ref: z.string().optional(), result_sha256: z.string().optional(), + policy: z + .object({ + catalog_version: z.string().min(1), + ablated_pass_id: PolicyPassIdSchema.nullable(), + trace_status: z.enum(["complete", "not-run", "error", "overflow"]), + trace_ref: z.string().min(1).optional(), + trace_sha256: Sha256Schema.optional(), + raw_response_sha256: z.array(Sha256Schema), + authoritative: z.boolean(), + reason: z.string().min(1).nullable(), + }) + .strict() + .superRefine((value, ctx) => { + if (value.authoritative && value.catalog_version !== POLICY_CATALOG_VERSION) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + path: ["catalog_version"], + message: "authoritative matrix policy requires the current catalog", + }); + } + const hasIdentity = value.trace_ref !== undefined && value.trace_sha256 !== undefined; + const hasAnyIdentity = value.trace_ref !== undefined || value.trace_sha256 !== undefined; + if (value.trace_status === "complete" && !hasIdentity) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + path: ["trace_ref"], + message: "complete matrix policy requires trace ref/hash", + }); + } + if (value.trace_status !== "complete" && hasAnyIdentity) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + path: ["trace_ref"], + message: `${value.trace_status} matrix policy forbids trace ref/hash`, + }); + } + if (value.authoritative !== (value.trace_status === "complete" && value.reason === null)) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + path: ["authoritative"], + message: "authoritative matrix policy requires complete trace and no reason", + }); + } + }) + .optional(), }) .strict(); @@ -426,6 +572,7 @@ export type BenchMatrix = z.infer; export type PhasesSnapshot = z.infer; export type Provenance = z.infer; export type CaseResult = z.infer; +export type BenchPolicyTraceRun = z.infer; export type SpreadStat = z.infer; export type Stability = z.infer; export type ProviderResult = z.infer; diff --git a/tests/unit/bench-matrix.test.ts b/tests/unit/bench-matrix.test.ts index a79fb73..701f2c9 100644 --- a/tests/unit/bench-matrix.test.ts +++ b/tests/unit/bench-matrix.test.ts @@ -5,13 +5,22 @@ // baseline floor and survives when the floor is ablated → a real Δ. import { describe, expect, it } from "bun:test"; import { execFileSync } from "node:child_process"; +import { createHash } from "node:crypto"; import { existsSync, mkdirSync, mkdtempSync, readFileSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; +import { canonicalJson } from "../../src/audit/canonical.ts"; +import { + type AuthoritativeTraceInvalidityCode, + type AuthoritativeTraceRun, + validateAuthoritativeTracePair, +} from "../../src/bench/runner.ts"; import { runBenchMatrix } from "../../src/cli/commands/bench.ts"; +import { POLICY_CATALOG_VERSION, POLICY_PASS_IDS } from "../../src/core/policy/catalog.ts"; import type { ProviderAdapter, ReviewResult } from "../../src/providers/adapter-base.ts"; import { BenchMatrixSchema, BenchResultSchema } from "../../src/schemas/bench-result.ts"; import type { Finding } from "../../src/schemas/finding.ts"; +import type { PolicyTrace } from "../../src/schemas/policy-trace.ts"; const DB_DIFF = [ "diff --git a/src/db.ts b/src/db.ts", @@ -148,8 +157,77 @@ function initGitRepo(dir: string): void { execFileSync("git", ["commit", "-m", "initial"], { cwd: dir, stdio: "ignore" }); } +function sha256(value: string): string { + return createHash("sha256").update(value).digest("hex"); +} + +function emptyTrace(ablated: PolicyTrace["ablated"]): PolicyTrace { + const passes = POLICY_PASS_IDS.map((passId) => + passId === "judgment.confidence" + ? { + pass_id: passId, + status: "ran" as const, + considered: 0, + opportunities: 0, + would_apply: 0, + applied: 0, + protected: 0, + blocking_removed: 0, + blocking_preserved: 0, + dropped: 0, + } + : { + pass_id: passId, + status: "not-run" as const, + reason_code: "configured-off" as const, + }, + ); + return { + schema: "reviewgate.policy-trace.v1", + catalog_version: POLICY_CATALOG_VERSION, + run_id: "bench-case", + iter: 1, + ablated, + raw_response_sha256: ["a".repeat(64), "b".repeat(64)], + passes, + evaluations: [], + stages: [ + { + stage_id: "verdict.compute", + order: 190, + reason_code: "no-blocking-findings", + input_signatures: [], + verdict: "PASS", + }, + ], + final: { + verdict: "PASS", + counts: { critical: 0, warn: 0, info: 0 }, + finding_signatures: [], + finding_severities: [], + }, + }; +} + +function traceRun(ablated: PolicyTrace["ablated"]): AuthoritativeTraceRun { + const trace = emptyTrace(ablated); + const traceSha256 = sha256(canonicalJson(trace)); + return { + authoritative: true, + status: "complete", + catalogVersion: POLICY_CATALOG_VERSION, + requestedAblations: ablated, + trace, + traceRef: `inline-policy-trace/${traceSha256}.json`, + traceSha256, + requestIdentitySha256: "c".repeat(64), + effectiveConfigSha256: "d".repeat(64), + finalIdentitySha256: sha256(canonicalJson(trace.final)), + }; +} + describe("runBenchMatrix", () => { - it("reports the confidence-floor ablation Δ (baseline demotes a low-conf FP; ablated keeps it)", async () => { + it("ablates confidence internally while effective config and captured responses remain identical", async () => { const corpus = newCorpus(); const out = join(corpus, "matrix.json"); const res = await runBenchMatrix({ @@ -160,11 +238,12 @@ describe("runBenchMatrix", () => { adapters: { codex: stub() }, now: () => new Date("2026-07-01T00:00:00Z"), }); + expect(res.stderr).toBe(""); expect(res.exitCode).toBe(0); const m = BenchMatrixSchema.parse(JSON.parse(readFileSync(out, "utf8"))); expect(m.variants).toHaveLength(2); const baseline = m.variants[0]; - const ablated = m.variants.find((v) => v.ablation === "confidence-floor"); + const ablated = m.variants.find((v) => v.ablation === "judgment.confidence"); expect(baseline?.ablation).toBe(""); expect(baseline?.delta).toBeNull(); // baseline: floor demotes the low-conf FP → clean-FP 0, precision 1. @@ -177,11 +256,169 @@ describe("runBenchMatrix", () => { expect(ablated?.class).toBe("A"); expect(ablated?.delta?.precision).toBeCloseTo(0.5, 10); expect(ablated?.delta?.clean_fp_rate).toBeCloseTo(-1, 10); + const baselineResult = BenchResultSchema.parse( + JSON.parse(readFileSync(join(corpus, "baseline.result.json"), "utf8")), + ); + const ablatedResult = BenchResultSchema.parse( + JSON.parse(readFileSync(join(corpus, "no-judgment.confidence.result.json"), "utf8")), + ); + expect(baselineResult.provenance.config_hash).toBe(ablatedResult.provenance.config_hash); + expect(baselineResult.provenance.phases.confidence_floor).toBeGreaterThan(0); + expect(ablatedResult.provenance.phases.confidence_floor).toBe( + baselineResult.provenance.phases.confidence_floor, + ); + expect(baseline?.policy?.raw_response_sha256).toEqual(ablated?.policy?.raw_response_sha256); + expect(baseline?.policy?.authoritative).toBe(true); + expect(ablated?.policy?.authoritative).toBe(true); + expect(ablated?.policy?.ablated_pass_id).toBe("judgment.confidence"); // The Δ table renders. expect(res.stdout).toContain("ablation"); expect(res.stdout.toLowerCase()).toContain("baseline"); }); + it("rejects every non-authoritative trace-pair boundary with a precise closed reason", () => { + const baseline = traceRun([]); + const counterfactual = traceRun(["judgment.confidence"]); + expect(validateAuthoritativeTracePair(baseline, counterfactual)).toEqual({ ok: true }); + + const cases: Array<{ + name: string; + mutate: (base: AuthoritativeTraceRun, variant: AuthoritativeTraceRun) => void; + code: AuthoritativeTraceInvalidityCode; + }> = [ + { + name: "missing trace", + mutate: (_base, variant) => { + variant.trace = undefined; + }, + code: "missing-trace", + }, + { + name: "missing configured pass row", + mutate: (_base, variant) => { + if (variant.trace) variant.trace.passes = variant.trace.passes.slice(1); + }, + code: "missing-pass-row", + }, + { + name: "ablated pass not run", + mutate: (_base, variant) => { + if (variant.trace) { + variant.trace.passes = variant.trace.passes.map((row) => + row.pass_id === "judgment.confidence" + ? { + pass_id: "judgment.confidence", + status: "not-run", + reason_code: "configured-off", + } + : row, + ); + } + }, + code: "pass-not-run", + }, + { + name: "trace error", + mutate: (_base, variant) => { + variant.status = "error"; + }, + code: "trace-status", + }, + { + name: "trace overflow", + mutate: (_base, variant) => { + variant.status = "overflow"; + }, + code: "trace-status", + }, + { + name: "missing content ref", + mutate: (_base, variant) => { + variant.traceRef = undefined; + }, + code: "trace-reference", + }, + { + name: "content hash mismatch", + mutate: (_base, variant) => { + variant.traceSha256 = "e".repeat(64); + }, + code: "trace-hash", + }, + { + name: "catalog mismatch", + mutate: (_base, variant) => { + variant.catalogVersion = "reviewgate.policy-catalog.v0"; + }, + code: "catalog-mismatch", + }, + { + name: "requested pass mismatch", + mutate: (_base, variant) => { + variant.requestedAblations = ["judgment.critic"]; + }, + code: "requested-pass-mismatch", + }, + { + name: "ordered response mismatch", + mutate: (_base, variant) => { + if (variant.trace) variant.trace.raw_response_sha256.reverse(); + }, + code: "response-hash-mismatch", + }, + { + name: "request mismatch", + mutate: (_base, variant) => { + variant.requestIdentitySha256 = "f".repeat(64); + }, + code: "request-identity-mismatch", + }, + { + name: "config mismatch", + mutate: (_base, variant) => { + variant.effectiveConfigSha256 = "1".repeat(64); + }, + code: "config-mismatch", + }, + { + name: "final identity mismatch", + mutate: (_base, variant) => { + variant.finalIdentitySha256 = "2".repeat(64); + }, + code: "final-identity-mismatch", + }, + { + name: "non-authoritative execution", + mutate: (_base, variant) => { + variant.authoritative = false; + }, + code: "non-authoritative-execution", + }, + { + name: "missing counters", + mutate: (_base, variant) => { + const row = variant.trace?.passes.find( + (candidate) => candidate.pass_id === "judgment.confidence", + ); + if (row?.status === "ran") Reflect.deleteProperty(row, "opportunities"); + }, + code: "missing-counter", + }, + ]; + + for (const testCase of cases) { + const nextBaseline = structuredClone(baseline); + const nextVariant = structuredClone(counterfactual); + testCase.mutate(nextBaseline, nextVariant); + const result = validateAuthoritativeTracePair(nextBaseline, nextVariant); + expect(result.ok, testCase.name).toBe(false); + if (!result.ok) { + expect(result.code, testCase.name).toBe(testCase.code); + expect(result.reason.length, testCase.name).toBeGreaterThan(0); + } + } + }); + it("exits 2 with no --ablate layers", async () => { const corpus = newCorpus(); const res = await runBenchMatrix({ @@ -253,20 +490,21 @@ describe("runBenchMatrix", () => { now: () => new Date("2026-07-01T00:00:00Z"), }); + expect(res.stderr).toBe(""); expect(res.exitCode).toBe(0); expect(reviewCalls).toBe(2); // baseline only; the variant is deterministic replay expect(criticCalls).toBe(2); expect(existsSync(join(artifactDir, "baseline.result.json"))).toBe(true); - expect(existsSync(join(artifactDir, "no-critic.result.json"))).toBe(true); + expect(existsSync(join(artifactDir, "no-judgment.critic.result.json"))).toBe(true); const manifest = JSON.parse( readFileSync(join(artifactDir, "reviewer-responses.sha256.json"), "utf8"), ) as { entries: Array<{ request_sha256: string; response_sha256: string }> }; - expect(manifest.entries).toHaveLength(2); + expect(manifest.entries).toHaveLength(4); expect(manifest.entries.every((e) => e.request_sha256.length === 64)).toBe(true); expect(manifest.entries.every((e) => e.response_sha256.length === 64)).toBe(true); const matrix = BenchMatrixSchema.parse(JSON.parse(readFileSync(out, "utf8"))); expect(matrix.artifacts?.baseline.path).toBe("baseline.result.json"); - expect(matrix.artifacts?.variants[0]?.path).toBe("no-critic.result.json"); + expect(matrix.artifacts?.variants[0]?.path).toBe("no-judgment.critic.result.json"); expect(matrix.artifacts?.reviewer_responses.path).toBe("reviewer-responses.sha256.json"); }); @@ -315,7 +553,7 @@ describe("runBenchMatrix", () => { JSON.parse(readFileSync(join(artifactDir, "baseline.result.json"), "utf8")), ); const variant = BenchResultSchema.parse( - JSON.parse(readFileSync(join(artifactDir, "no-confidence-floor.result.json"), "utf8")), + JSON.parse(readFileSync(join(artifactDir, "no-judgment.confidence.result.json"), "utf8")), ); expect(baseline.providers[0]?.coverage.value).toBe(1); expect(variant.providers[0]?.coverage.value).toBe(1); @@ -376,7 +614,7 @@ describe("runBenchMatrix", () => { expect(res.stderr).toContain("variant corpus commit differs from baseline"); expect(res.stderr).toContain("variant source commit differs from baseline"); expect(existsSync(join(artifactDir, "baseline.result.json"))).toBe(true); - expect(existsSync(join(artifactDir, "no-critic.result.json"))).toBe(true); + expect(existsSync(join(artifactDir, "no-judgment.critic.result.json"))).toBe(true); expect(existsSync(join(artifactDir, "matrix.json"))).toBe(false); }); @@ -440,10 +678,10 @@ describe("runBenchMatrix", () => { expect(res.exitCode).toBe(0); expect(adapter.reviewCalls).toBe(2); - expect(adapter.completeCalls).toBe(4); + expect(adapter.completeCalls).toBe(2); }); - it("counts live critic completions in reviewer-replay variants against the hard call ceiling", async () => { + it("never makes live critic completions in a replay variant", async () => { const corpus = newCorpus(); let reviewCalls = 0; let criticCalls = 0; @@ -483,17 +721,16 @@ describe("runBenchMatrix", () => { criticModel: "deepseek/deepseek-v4-flash", criticOpenrouterProvider: { only: ["alibaba"] }, maxOutputTokens: 128, - // baseline = 2 reviewer + 2 critic calls; the replay variant has two more - // live critic calls. The second one must be refused, never hidden as replay. - maxProviderCalls: 5, + // Exactly the baseline's 2 reviewer + 2 critic calls fit. Any live call in + // the variant exhausts the ceiling and kills this contract test. + maxProviderCalls: 4, adapters: { codex: countedReviewer, openrouter: critic }, now: () => new Date("2026-07-01T00:00:00Z"), }); - expect(res.exitCode).toBe(4); - expect(res.stderr).toContain("provider-call ceiling exhausted"); + expect(res.exitCode).toBe(0); expect(reviewCalls).toBe(2); - expect(criticCalls).toBe(3); + expect(criticCalls).toBe(2); }); it("captures declared fallback reviewers and replays them without untracked live calls", async () => { @@ -553,7 +790,7 @@ describe("runBenchMatrix", () => { expect(manifest.entries.filter((entry) => entry.provider === "gemini")).toHaveLength(2); }); - it("classifies scope-to-diff as a deterministic post-review ablation", async () => { + it("fails authoritative pairing when the requested scope pass did not run", async () => { const corpus = newCorpus(); const out = join(corpus, "scope-matrix", "matrix.json"); const res = await runBenchMatrix({ @@ -565,10 +802,9 @@ describe("runBenchMatrix", () => { now: () => new Date("2026-07-01T00:00:00Z"), }); - expect(res.exitCode).toBe(0); - const matrix = BenchMatrixSchema.parse(JSON.parse(readFileSync(out, "utf8"))); - expect(matrix.variants.find((variant) => variant.ablation === "scope-to-diff")?.class).toBe( - "A", - ); + expect(res.exitCode).toBe(4); + expect(res.stderr).toContain("pass-not-run"); + expect(res.stderr).toContain("scope.diff"); + expect(existsSync(out)).toBe(false); }); }); diff --git a/tests/unit/bench-report.test.ts b/tests/unit/bench-report.test.ts index ae7950b..d32dca6 100644 --- a/tests/unit/bench-report.test.ts +++ b/tests/unit/bench-report.test.ts @@ -6,8 +6,8 @@ // seeded cases) is flagged non-authoritative and its headline rates are withheld. import { describe, expect, it } from "bun:test"; import { makeMetric } from "../../src/bench/metrics.ts"; -import { isAuthoritative, renderBenchReport } from "../../src/bench/report.ts"; -import type { BenchResult } from "../../src/schemas/bench-result.ts"; +import { isAuthoritative, renderBenchMatrix, renderBenchReport } from "../../src/bench/report.ts"; +import type { BenchMatrix, BenchResult } from "../../src/schemas/bench-result.ts"; function baseResult(over: Partial = {}): BenchResult { const result: BenchResult = { @@ -285,3 +285,80 @@ describe("renderBenchReport", () => { expect(table).toContain("n/a"); }); }); + +describe("renderBenchMatrix policy provenance", () => { + it("renders normalized pass IDs and precise authoritative invalidity reasons", () => { + const base = baseResult(); + const metric = makeMetric(1, 1); + const matrix: BenchMatrix = { + schema: "reviewgate.bench.matrix.v1", + provenance: base.provenance, + variants: [ + { + label: "baseline", + ablation: "", + class: "baseline", + precision: metric, + recall: metric, + clean_fp_rate: makeMetric(0, 1), + delta: null, + policy: { + catalog_version: "reviewgate.policy-catalog.v1", + ablated_pass_id: null, + trace_status: "complete", + trace_ref: `inline-policy-trace-set/${"a".repeat(64)}.json`, + trace_sha256: "a".repeat(64), + raw_response_sha256: ["b".repeat(64)], + authoritative: true, + reason: null, + }, + }, + { + label: "-scope.diff", + ablation: "scope.diff", + class: "A", + precision: metric, + recall: metric, + clean_fp_rate: makeMetric(0, 1), + delta: { precision: 0, recall: 0, clean_fp_rate: 0 }, + policy: { + catalog_version: "reviewgate.policy-catalog.v1", + ablated_pass_id: "scope.diff", + trace_status: "not-run", + raw_response_sha256: ["b".repeat(64)], + authoritative: false, + reason: "pass-not-run: scope.diff had no configured opportunity", + }, + }, + ], + authoritative: false, + }; + const output = renderBenchMatrix(matrix); + expect(output).toContain("scope.diff"); + expect(output).toContain("NON-AUTHORITATIVE"); + expect(output).toContain("pass-not-run: scope.diff had no configured opportunity"); + expect(output).toContain("reviewgate.policy-catalog.v1"); + }); + + it("labels a legacy matrix without trace provenance as non-authoritative for policy", () => { + const base = baseResult(); + const metric = makeMetric(1, 1); + const output = renderBenchMatrix({ + schema: "reviewgate.bench.matrix.v1", + provenance: base.provenance, + variants: [ + { + label: "baseline", + ablation: "", + class: "baseline", + precision: metric, + recall: metric, + clean_fp_rate: makeMetric(0, 1), + delta: null, + }, + ], + }); + expect(output).toContain("NON-AUTHORITATIVE"); + expect(output).toContain("legacy result has no policy trace provenance"); + }); +}); diff --git a/tests/unit/bench-result-schema.test.ts b/tests/unit/bench-result-schema.test.ts index 87d8589..1cc83ca 100644 --- a/tests/unit/bench-result-schema.test.ts +++ b/tests/unit/bench-result-schema.test.ts @@ -1,5 +1,66 @@ import { describe, expect, it } from "bun:test"; -import { BenchResultSchema } from "../../src/schemas/bench-result.ts"; +import { createHash } from "node:crypto"; +import { canonicalJson } from "../../src/audit/canonical.ts"; +import { POLICY_CATALOG_VERSION, POLICY_PASS_IDS } from "../../src/core/policy/catalog.ts"; +import { BenchMatrixSchema, BenchResultSchema } from "../../src/schemas/bench-result.ts"; + +function emptyPolicyTrace(ablated: string[] = []) { + return { + schema: "reviewgate.policy-trace.v1", + catalog_version: POLICY_CATALOG_VERSION, + run_id: "bench-case", + iter: 1, + ablated, + raw_response_sha256: ["a".repeat(64)], + passes: POLICY_PASS_IDS.map((passId) => ({ + pass_id: passId, + status: "ran", + considered: 0, + opportunities: 0, + would_apply: 0, + applied: 0, + protected: 0, + blocking_removed: 0, + blocking_preserved: 0, + dropped: 0, + })), + evaluations: [], + stages: [ + { + stage_id: "verdict.compute", + order: 190, + reason_code: "no-blocking-findings", + input_signatures: [], + verdict: "PASS", + }, + ], + final: { + verdict: "PASS", + counts: { critical: 0, warn: 0, info: 0 }, + finding_signatures: [], + finding_severities: [], + }, + }; +} + +function completePolicyRecord() { + const trace = emptyPolicyTrace(); + const traceSha256 = createHash("sha256").update(canonicalJson(trace)).digest("hex"); + const finalIdentitySha256 = createHash("sha256").update(canonicalJson(trace.final)).digest("hex"); + return { + authoritative: true, + status: "complete", + catalog_version: POLICY_CATALOG_VERSION, + requested_ablations: [], + trace, + trace_ref: `inline-policy-trace/${traceSha256}.json`, + trace_sha256: traceSha256, + request_identity_sha256: "c".repeat(64), + effective_config_sha256: "d".repeat(64), + final_identity_sha256: finalIdentitySha256, + reason: null, + }; +} const validResult = { schema: "reviewgate.bench.result.v1", @@ -76,6 +137,126 @@ describe("BenchResultSchema", () => { expect(r.success).toBe(true); }); + it("accepts complete embedded policy trace provenance additively", () => { + const parsed = BenchResultSchema.safeParse({ + ...validResult, + cases: [{ ...validResult.cases[0], policy_trace: completePolicyRecord() }], + }); + if (!parsed.success) console.error(parsed.error); + expect(parsed.success).toBe(true); + }); + + it("keeps legacy BenchResult v1 artifacts explicitly parseable without policy traces", () => { + expect(BenchResultSchema.safeParse(validResult).success).toBe(true); + }); + + it("rejects complete trace provenance with a missing ref/hash instead of defaulting it", () => { + const policy = completePolicyRecord(); + const { trace_ref: _missing, ...withoutRef } = policy; + expect( + BenchResultSchema.safeParse({ + ...validResult, + cases: [{ ...validResult.cases[0], policy_trace: withoutRef }], + }).success, + ).toBe(false); + }); + + it("rejects a tampered embedded trace whose immutable hash/ref no longer match", () => { + const policy = completePolicyRecord(); + policy.trace.raw_response_sha256 = ["f".repeat(64)]; + expect( + BenchResultSchema.safeParse({ + ...validResult, + cases: [{ ...validResult.cases[0], policy_trace: policy }], + }).success, + ).toBe(false); + }); + + it("rejects a missing ran-pass counter rather than coercing it to zero", () => { + const policy = completePolicyRecord(); + const first = policy.trace.passes[0]; + if (first) Reflect.deleteProperty(first, "opportunities"); + expect( + BenchResultSchema.safeParse({ + ...validResult, + cases: [{ ...validResult.cases[0], policy_trace: policy }], + }).success, + ).toBe(false); + }); + + it("strictly validates normalized matrix policy provenance", () => { + const metric = validResult.aggregate.precision; + const matrix = { + schema: "reviewgate.bench.matrix.v1", + provenance: validResult.provenance, + variants: [ + { + label: "-judgment.confidence", + ablation: "judgment.confidence", + class: "A", + precision: metric, + recall: metric, + clean_fp_rate: metric, + delta: { precision: 0, recall: 0, clean_fp_rate: 0 }, + policy: { + catalog_version: POLICY_CATALOG_VERSION, + ablated_pass_id: "judgment.confidence", + trace_status: "complete", + trace_ref: `inline-policy-trace-set/${"f".repeat(64)}.json`, + trace_sha256: "f".repeat(64), + raw_response_sha256: ["a".repeat(64)], + authoritative: true, + reason: null, + }, + }, + ], + }; + expect(BenchMatrixSchema.safeParse(matrix).success).toBe(true); + expect( + BenchMatrixSchema.safeParse({ + ...matrix, + variants: [ + { + ...matrix.variants[0], + policy: { ...matrix.variants[0]?.policy, ablated_pass_id: "confidence-floor" }, + }, + ], + }).success, + ).toBe(false); + expect( + BenchMatrixSchema.safeParse({ + ...matrix, + variants: [ + { + ...matrix.variants[0], + policy: { + ...matrix.variants[0]?.policy, + catalog_version: "reviewgate.policy-catalog.v0", + }, + }, + ], + }).success, + ).toBe(false); + + expect( + BenchMatrixSchema.safeParse({ + ...matrix, + variants: [ + { + ...matrix.variants[0], + policy: { + ...matrix.variants[0]?.policy, + authoritative: false, + trace_status: "error", + trace_sha256: undefined, + reason: "trace failed", + }, + }, + ], + }).success, + ).toBe(false); + }); + it("accepts Alpha.12 integrity, critic coverage and honest unknown costs additively", () => { const r = BenchResultSchema.safeParse({ ...validResult, From bccc36f64edf9df2d723f04b12d99d4e24d56744 Mon Sep 17 00:00:00 2001 From: Codevena Date: Mon, 10 Aug 2026 11:39:30 +0200 Subject: [PATCH 39/55] fix(bench): persist and replay authoritative evidence --- src/bench/runner.ts | 66 +- src/cli/commands/bench.ts | 1093 +++++++++++++++++++++--- src/schemas/bench-result.ts | 325 ++++++- tests/unit/bench-matrix.test.ts | 478 ++++++++++- tests/unit/bench-result-schema.test.ts | 140 ++- 5 files changed, 1944 insertions(+), 158 deletions(-) diff --git a/src/bench/runner.ts b/src/bench/runner.ts index fbd1f5d..5b88a1c 100644 --- a/src/bench/runner.ts +++ b/src/bench/runner.ts @@ -21,6 +21,7 @@ import { import { tmpdir } from "node:os"; import { join } from "node:path"; import { canonicalJson } from "../audit/canonical.ts"; +import { verifyPolicyTraceReference, writePolicyTrace } from "../audit/policy-trace-store.ts"; import { buildAdapters } from "../cli/build-adapters.ts"; import { defaultConfig } from "../config/defaults.ts"; import { ConfigSchema, type ReviewgateConfig } from "../config/define-config.ts"; @@ -397,14 +398,22 @@ export function validateAuthoritativeTracePair( if (run.traceRef === undefined || run.traceSha256 === undefined) { return invalidTracePair( "trace-reference", - `${label} complete policy trace is missing its inline reference or hash`, + `${label} complete policy trace is missing its persisted reference or hash`, ); } const actualSha256 = sha256Text(canonicalJson(trace)); if (run.traceSha256 !== actualSha256) { return invalidTracePair("trace-hash", `${label} policy trace content hash mismatches`); } - if (run.traceRef !== `inline-policy-trace/${actualSha256}.json`) { + const refMatch = run.traceRef.match( + /^artifacts\/policy-traces\/\d{4}\/\d{2}\/\d{2}\/policy\/([0-9a-f]{12})-i(0|[1-9]\d*)-([0-9a-f]{12})\.json$/, + ); + if ( + refMatch === null || + refMatch[1] !== sha256Text(trace.run_id).slice(0, 12) || + Number(refMatch[2]) !== trace.iter || + refMatch[3] !== actualSha256.slice(0, 12) + ) { return invalidTracePair("trace-reference", `${label} policy trace reference mismatches`); } } @@ -429,6 +438,8 @@ export interface RunBenchCaseInput { criticMaxAttempts?: number; /** Bench/Rig-only trace and ablation control. Normal gate/config paths never set this. */ policyExecution?: PolicyExecutionOptions; + /** Matrix-owned contained store for authoritative trace artifacts. */ + policyTraceStore?: { root: string; refPrefix: string; now?: Date }; } function buildAuthoritativeTraceRun(input: { @@ -439,6 +450,7 @@ function buildAuthoritativeTraceRun(input: { diffPatch: string; report: PendingReport; verdict: "PASS" | "SOFT-PASS" | "FAIL" | "ERROR"; + traceStore?: { root: string; refPrefix: string; now?: Date }; }): AuthoritativeTraceRun { const requestedAblations = [...input.execution.policyAblations].sort( (left, right) => POLICY_PASS_IDS.indexOf(left) - POLICY_PASS_IDS.indexOf(right), @@ -471,7 +483,7 @@ function buildAuthoritativeTraceRun(input: { ); if (input.trace === undefined) { return { - authoritative: input.execution.authoritative, + authoritative: false, status: "not-run", catalogVersion: POLICY_CATALOG_VERSION, requestedAblations, @@ -481,14 +493,57 @@ function buildAuthoritativeTraceRun(input: { }; } const traceSha256 = sha256Text(canonicalJson(input.trace)); + if (input.traceStore === undefined) { + return { + authoritative: false, + status: "error", + catalogVersion: input.trace.catalog_version, + requestedAblations, + requestIdentitySha256, + effectiveConfigSha256, + finalIdentitySha256: sha256Text(canonicalJson(finalIdentity)), + }; + } + const stored = writePolicyTrace({ + auditDir: input.traceStore.root, + trace: input.trace, + ...(input.traceStore.now === undefined ? {} : { now: input.traceStore.now }), + }); + if (stored.status !== "complete") { + return { + authoritative: false, + status: stored.status, + catalogVersion: input.trace.catalog_version, + requestedAblations, + requestIdentitySha256, + effectiveConfigSha256, + finalIdentitySha256: sha256Text(canonicalJson(finalIdentity)), + }; + } + const verified = verifyPolicyTraceReference({ + auditDir: input.traceStore.root, + ref: stored.ref, + sha256: stored.sha256, + }); + if (!verified.ok || stored.sha256 !== traceSha256) { + return { + authoritative: false, + status: "error", + catalogVersion: input.trace.catalog_version, + requestedAblations, + requestIdentitySha256, + effectiveConfigSha256, + finalIdentitySha256: sha256Text(canonicalJson(finalIdentity)), + }; + } return { authoritative: input.execution.authoritative, status: "complete", catalogVersion: input.trace.catalog_version, requestedAblations, trace: input.trace, - traceRef: `inline-policy-trace/${traceSha256}.json`, - traceSha256, + traceRef: `${input.traceStore.refPrefix}/${stored.ref}`, + traceSha256: stored.sha256, requestIdentitySha256, effectiveConfigSha256, finalIdentitySha256: sha256Text(canonicalJson(finalIdentity)), @@ -708,6 +763,7 @@ export async function runBenchCase(input: RunBenchCaseInput): Promise boolean; /** Internal matrix-only policy trace/ablation options. */ policyExecution?: PolicyExecutionOptions; + /** Internal Matrix-owned real artifact sink for per-case policy traces. */ + policyTraceStore?: { root: string; refPrefix: string; now?: Date }; } export interface BenchRunnerInfo { @@ -222,7 +240,7 @@ export async function runBenchReport(input: BenchReportInput): Promise(CAPTURED_THROWABLE_FIELD_KEYS); + +function hasUnsafeControlCharacter(value: string): boolean { + return Array.from(value).some((character) => { + const codePoint = character.codePointAt(0) ?? 0; + return ( + codePoint <= 8 || + codePoint === 11 || + codePoint === 12 || + (codePoint >= 14 && codePoint <= 31) || + codePoint === 127 + ); + }); +} + +function safeThrowableString(value: string): boolean { + return !UNSAFE_THROW_STRING.test(value) && !hasUnsafeControlCharacter(value); +} + +type SafeValueCapture = + | { ok: true; value: import("../../schemas/bench-result.ts").ThrowableSafeValue } + | { ok: false; reason: ThrowableCaptureFailureReason }; + +function captureSafeThrowableValue( + value: unknown, + seen: Set, + depth: number, +): SafeValueCapture { + if (depth > 12) return { ok: false, reason: "unsupported-field" }; + if (value === null || typeof value === "boolean") return { ok: true, value }; + if (typeof value === "string") { + return safeThrowableString(value) + ? { ok: true, value } + : { ok: false, reason: "unsafe-string" }; + } + if (typeof value === "number" && Number.isFinite(value)) return { ok: true, value }; + if (typeof value !== "object") return { ok: false, reason: "unsupported-field" }; + if (seen.has(value)) return { ok: false, reason: "cyclic-value" }; + seen.add(value); + try { + if (Array.isArray(value)) { + const captured: import("../../schemas/bench-result.ts").ThrowableSafeValue[] = []; + for (const item of value) { + const next = captureSafeThrowableValue(item, seen, depth + 1); + if (!next.ok) return next; + captured.push(next.value); + } + return { ok: true, value: captured }; + } + const prototype = Object.getPrototypeOf(value); + if (prototype !== Object.prototype && prototype !== null) { + return { ok: false, reason: "unsupported-field" }; + } + const captured: Record = {}; + for (const key of Object.keys(value).sort()) { + if (SENSITIVE_THROW_FIELD.test(key)) return { ok: false, reason: "sensitive-field" }; + const descriptor = Object.getOwnPropertyDescriptor(value, key); + if (descriptor === undefined || !("value" in descriptor)) { + return { ok: false, reason: "unsupported-field" }; + } + const next = captureSafeThrowableValue(descriptor.value, seen, depth + 1); + if (!next.ok) return next; + captured[key] = next.value; + } + return { ok: true, value: captured }; + } finally { + seen.delete(value); + } +} + +function captureThrowableSnapshotInner( + thrown: unknown, + seen: Set, + depth: number, +): ThrowableCaptureResult { + if (typeof thrown === "string") { + if (!safeThrowableString(thrown)) return { ok: false, reason: "unsafe-string" }; + const snapshot = { + kind: "primitive" as const, + primitive_type: "string" as const, + value: thrown, + }; + return { ok: true, snapshot, sha256: sha256(canonicalJson(snapshot)) }; + } + if (thrown === undefined || thrown === null) { + const snapshot = { + kind: "primitive" as const, + primitive_type: thrown === undefined ? ("undefined" as const) : ("null" as const), + }; + return { ok: true, snapshot, sha256: sha256(canonicalJson(snapshot)) }; + } + if (!(thrown instanceof Error)) return { ok: false, reason: "unsupported-thrown-value" }; + if (depth > 12 || seen.has(thrown)) return { ok: false, reason: "cyclic-value" }; + const errorType = + thrown.constructor === Error + ? "Error" + : thrown.constructor === SandboxUnavailableError + ? "SandboxUnavailableError" + : null; + if (errorType === null) return { ok: false, reason: "unsupported-error-type" }; + if (!safeThrowableString(thrown.name) || !safeThrowableString(thrown.message)) { + return { ok: false, reason: "unsafe-string" }; + } + seen.add(thrown); + try { + let cause: CapturedThrowableSnapshot | undefined; + if (Object.hasOwn(thrown, "cause")) { + const causeDescriptor = Object.getOwnPropertyDescriptor(thrown, "cause"); + if (causeDescriptor === undefined || !("value" in causeDescriptor)) { + return { ok: false, reason: "unsupported-field" }; + } + const capturedCause = captureThrowableSnapshotInner(causeDescriptor.value, seen, depth + 1); + if (!capturedCause.ok) return capturedCause; + cause = capturedCause.snapshot; + } + const fields: Array<{ + key: string; + value: import("../../schemas/bench-result.ts").ThrowableSafeValue; + enumerable: boolean; + }> = []; + for (const key of Object.getOwnPropertyNames(thrown).sort()) { + if (["cause", "message", "name", "stack"].includes(key)) continue; + if (SENSITIVE_THROW_FIELD.test(key)) return { ok: false, reason: "sensitive-field" }; + const descriptor = Object.getOwnPropertyDescriptor(thrown, key); + if (descriptor === undefined || !("value" in descriptor)) { + return { ok: false, reason: "unsupported-field" }; + } + if (!CAPTURED_THROWABLE_FIELD_KEY_SET.has(key)) { + if (descriptor.enumerable) return { ok: false, reason: "unsupported-field" }; + continue; + } + const captured = captureSafeThrowableValue(descriptor.value, seen, depth + 1); + if (!captured.ok) return captured; + fields.push({ key, value: captured.value, enumerable: descriptor.enumerable ?? false }); + } + const snapshot: CapturedThrowableSnapshot = { + kind: "error", + error_type: errorType, + name: thrown.name, + message: thrown.message, + ...(cause === undefined ? {} : { cause }), + fields, + }; + const parsed = CapturedThrowableSnapshotSchema.safeParse(snapshot); + if (!parsed.success) { + return { ok: false, reason: "unsupported-field" }; + } + return { ok: true, snapshot: parsed.data, sha256: sha256(canonicalJson(parsed.data)) }; + } finally { + seen.delete(thrown); + } +} + +function freezeThrowableSnapshot(value: T): T { + if (value !== null && typeof value === "object" && !Object.isFrozen(value)) { + for (const child of Object.values(value as Record)) { + freezeThrowableSnapshot(child); + } + Object.freeze(value); + } + return value; +} + +export function captureThrowableSnapshot(thrown: unknown): ThrowableCaptureResult { + const captured = captureThrowableSnapshotInner(thrown, new Set(), 0); + if (!captured.ok) return captured; + freezeThrowableSnapshot(captured.snapshot); + return captured; +} + +function cloneThrowableSafeValue(value: T): T { + return structuredClone(value); +} + +export function replayThrowableSnapshot(snapshot: CapturedThrowableSnapshot): unknown { + if (snapshot.kind === "primitive") { + if (snapshot.primitive_type === "string") return snapshot.value; + if (snapshot.primitive_type === "undefined") return undefined; + return null; + } + const error: Error = + snapshot.error_type === "SandboxUnavailableError" + ? new SandboxUnavailableError(snapshot.message) + : new Error(snapshot.message); + error.name = snapshot.name; + if (snapshot.cause !== undefined) { + Object.defineProperty(error, "cause", { + value: replayThrowableSnapshot(snapshot.cause), + enumerable: false, + configurable: true, + writable: true, + }); + } + for (const field of snapshot.fields) { + Object.defineProperty(error, field.key, { + value: cloneThrowableSafeValue(field.value), + enumerable: field.enumerable, + configurable: true, + writable: true, + }); + } + return error; +} + +export type BenchArtifactKind = + | "policy-trace" + | "policy-trace-set" + | "bench-result" + | "response-manifest"; + +export type BenchArtifactVerification = + | { ok: true; value: unknown } + | { + ok: false; + reason: + | "invalid-reference" + | "path-escape" + | "missing" + | "not-a-file" + | "too-large" + | "hash-mismatch" + | "invalid-encoding" + | "invalid-json" + | "invalid-trace" + | "invalid-schema" + | "non-canonical" + | "identity-mismatch" + | "read-error"; + }; + +const BENCH_ARTIFACT_MAX_BYTES = 128 * 1024 * 1024; +const FULL_SHA256 = /^[0-9a-f]{64}$/; + +function artifactRefFor(kind: Exclude, sha: string): string { + const dir = + kind === "policy-trace-set" + ? "policy-trace-sets" + : kind === "bench-result" + ? "results" + : "responses"; + return `artifacts/${dir}/${sha}.json`; +} + +function isContainedPath(root: string, candidate: string): boolean { + const rel = relative(root, candidate); + return rel === "" || (!rel.startsWith("..") && !isAbsolute(rel)); +} + +export function verifyBenchArtifactReference(input: { + root: string; + ref: string; + sha256: string; + kind: BenchArtifactKind; +}): BenchArtifactVerification { + if ( + !FULL_SHA256.test(input.sha256) || + isAbsolute(input.ref) || + input.ref.includes("\\") || + input.ref + .split("/") + .some((component) => component === "" || component === "." || component === "..") + ) { + return { ok: false, reason: "invalid-reference" }; + } + if (input.kind === "policy-trace") { + const prefix = "artifacts/policy-traces/"; + if (!input.ref.startsWith(prefix)) return { ok: false, reason: "invalid-reference" }; + const verified = verifyPolicyTraceReference({ + auditDir: join(input.root, "artifacts", "policy-traces"), + ref: input.ref.slice(prefix.length), + sha256: input.sha256, + }); + return verified.ok ? { ok: true, value: verified.trace } : verified; + } + if (input.ref !== artifactRefFor(input.kind, input.sha256)) { + return { ok: false, reason: "identity-mismatch" }; + } + const root = resolve(input.root); + const candidate = resolve(root, ...input.ref.split("/")); + if (!isContainedPath(root, candidate)) return { ok: false, reason: "path-escape" }; + if (!existsSync(candidate)) return { ok: false, reason: "missing" }; + let fd: number | undefined; + try { + const rootStat = lstatSync(root); + if (rootStat.isSymbolicLink() || !rootStat.isDirectory()) { + return { ok: false, reason: "path-escape" }; + } + const realRoot = realpathSync(root); + let parent = root; + for (const component of input.ref.split("/").slice(0, -1)) { + parent = join(parent, component); + const parentStat = lstatSync(parent); + if ( + parentStat.isSymbolicLink() || + !parentStat.isDirectory() || + !isContainedPath(realRoot, realpathSync(parent)) + ) { + return { ok: false, reason: "path-escape" }; + } + } + const before = lstatSync(candidate); + if (before.isSymbolicLink() || !before.isFile() || before.nlink !== 1) { + return { ok: false, reason: "not-a-file" }; + } + if (before.size > BENCH_ARTIFACT_MAX_BYTES) return { ok: false, reason: "too-large" }; + if (!isContainedPath(realRoot, realpathSync(candidate))) { + return { ok: false, reason: "path-escape" }; + } + fd = openSync(candidate, constants.O_RDONLY | constants.O_NOFOLLOW); + const opened = fstatSync(fd); + if ( + !opened.isFile() || + opened.nlink !== 1 || + opened.dev !== before.dev || + opened.ino !== before.ino + ) { + return { ok: false, reason: "not-a-file" }; + } + if (opened.size > BENCH_ARTIFACT_MAX_BYTES) return { ok: false, reason: "too-large" }; + const bytes = readFileSync(fd); + if (bytes.length > BENCH_ARTIFACT_MAX_BYTES) return { ok: false, reason: "too-large" }; + const after = fstatSync(fd); + const pathAfter = lstatSync(candidate); + if ( + after.dev !== opened.dev || + after.ino !== opened.ino || + after.size !== opened.size || + after.mtimeMs !== opened.mtimeMs || + after.ctimeMs !== opened.ctimeMs || + pathAfter.isSymbolicLink() || + !pathAfter.isFile() || + pathAfter.nlink !== 1 || + pathAfter.dev !== after.dev || + pathAfter.ino !== after.ino + ) { + return { ok: false, reason: "read-error" }; + } + if (sha256(bytes) !== input.sha256) return { ok: false, reason: "hash-mismatch" }; + let text: string; + try { + text = new TextDecoder("utf-8", { fatal: true }).decode(bytes); + } catch { + return { ok: false, reason: "invalid-encoding" }; + } + let decoded: unknown; + try { + decoded = JSON.parse(text); + } catch { + return { ok: false, reason: "invalid-json" }; + } + const parsed = + input.kind === "bench-result" + ? BenchResultSchema.safeParse(decoded) + : input.kind === "policy-trace-set" + ? BenchPolicyTraceSetSchema.safeParse(decoded) + : BenchResponseManifestSchema.safeParse(decoded); + if (!parsed.success) return { ok: false, reason: "invalid-schema" }; + if (canonicalJson(parsed.data) !== text) return { ok: false, reason: "non-canonical" }; + return { ok: true, value: parsed.data }; + } catch { + return { ok: false, reason: "read-error" }; + } finally { + if (fd !== undefined) closeSync(fd); + } +} + +function ensureDirectoryWithoutSymlinks(path: string): boolean { + const target = resolve(path); + const missing: string[] = []; + let cursor = target; + while (!existsSync(cursor)) { + const parent = dirname(cursor); + if (parent === cursor) return false; + missing.unshift(basename(cursor)); + cursor = parent; + } + try { + const existing = lstatSync(cursor); + if (existing.isSymbolicLink() || !existing.isDirectory()) return false; + for (const component of missing) { + cursor = join(cursor, component); + try { + mkdirSync(cursor, { mode: 0o700 }); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "EEXIST") return false; + } + const created = lstatSync(cursor); + if (created.isSymbolicLink() || !created.isDirectory()) return false; + } + return true; + } catch { + return false; + } +} + +function ensureContainedArtifactParent(root: string, ref: string): boolean { + if (!ensureDirectoryWithoutSymlinks(root)) return false; + try { + const realRoot = realpathSync(root); + let parent = resolve(root); + for (const component of ref.split("/").slice(0, -1)) { + parent = join(parent, component); + if (!existsSync(parent)) { + try { + mkdirSync(parent, { mode: 0o700 }); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "EEXIST") return false; + } + } + const stat = lstatSync(parent); + if ( + stat.isSymbolicLink() || + !stat.isDirectory() || + !isContainedPath(realRoot, realpathSync(parent)) + ) { + return false; + } + } + return true; + } catch { + return false; + } +} + +type PersistedBenchArtifact = + | { ok: true; ref: string; sha256: string } + | { ok: false; reason: Exclude["reason"] }; + +function persistBenchArtifact(input: { + root: string; + kind: Exclude; + value: BenchResult | BenchResponseManifest | BenchPolicyTraceSet; +}): PersistedBenchArtifact { + const parsed = + input.kind === "bench-result" + ? BenchResultSchema.safeParse(input.value) + : input.kind === "response-manifest" + ? BenchResponseManifestSchema.safeParse(input.value) + : BenchPolicyTraceSetSchema.safeParse(input.value); + if (!parsed.success) return { ok: false, reason: "invalid-schema" }; + const canonical = canonicalJson(parsed.data); + const contentSha256 = sha256(canonical); + const ref = artifactRefFor(input.kind, contentSha256); + if (!ensureContainedArtifactParent(input.root, ref)) { + return { ok: false, reason: "path-escape" }; + } + const destination = resolve(input.root, ...ref.split("/")); + try { + writeFileIfAbsent(destination, canonical, { mode: 0o600 }); + } catch { + return { ok: false, reason: "read-error" }; + } + const verified = verifyBenchArtifactReference({ + root: input.root, + ref, + sha256: contentSha256, + kind: input.kind, + }); + return verified.ok ? { ok: true, ref, sha256: contentSha256 } : verified; +} + interface CapturedReviewEntry { provider: ProviderId; kind: "review" | "complete"; @@ -1232,15 +1730,31 @@ interface CapturedReviewEntry { request_sha256: string; response_sha256: string; outcome: "return" | "throw"; + throw_snapshot?: CapturedThrowableSnapshot; } +type CapturedPreflightEntry = + | { + request_sha256: string; + response_sha256: string; + outcome: "return"; + value: Preflight; + } + | { + request_sha256: string; + response_sha256: string; + outcome: "throw"; + throw_snapshot: CapturedThrowableSnapshot; + }; + interface ReviewCaptureState { entries: CapturedReviewEntry[]; + preflights: Map; responses: Map< number, { kind: "review"; value: ReviewResult } | { kind: "complete"; value: string } >; - errors: Map; + throws: Map; nextOrdinal: number; mismatch: string | null; } @@ -1314,6 +1828,10 @@ function completionRequestHash( ); } +function preflightRequestHash(provider: ProviderId, ordinal: number, cfg: ProviderConfig): string { + return sha256(stableJson({ provider, ordinal, config: cfg })); +} + function captureReviewerAdapters( adapters: Partial>, state: ReviewCaptureState, @@ -1326,7 +1844,34 @@ function captureReviewerAdapters( const complete = adapter.complete?.bind(adapter); out[provider] = { id: adapter.id, - preflight: (cfg) => adapter.preflight(cfg), + async preflight(cfg) { + const entries = state.preflights.get(provider) ?? []; + state.preflights.set(provider, entries); + const requestHash = preflightRequestHash(provider, entries.length, cfg); + try { + const value = structuredClone(await adapter.preflight(cfg)); + entries.push({ + request_sha256: requestHash, + response_sha256: sha256(stableJson(value)), + outcome: "return", + value, + }); + return structuredClone(value); + } catch (error) { + const captured = captureThrowableSnapshot(error); + if (!captured.ok) { + state.mismatch ??= `${provider} preflight throw ${entries.length} is not safely reconstructable: ${captured.reason}`; + throw error; + } + entries.push({ + request_sha256: requestHash, + response_sha256: captured.sha256, + outcome: "throw", + throw_snapshot: captured.snapshot, + }); + throw error; + } + }, async review(input) { const ordinal = state.nextOrdinal++; const requestHash = reviewRequestHash(provider, ordinal, input); @@ -1344,15 +1889,20 @@ function captureReviewerAdapters( }); return structuredClone(response); } catch (error) { - const message = error instanceof Error ? error.message : String(error); - state.errors.set(ordinal, message); + const captured = captureThrowableSnapshot(error); + if (!captured.ok) { + state.mismatch ??= `${provider} review throw ${ordinal} is not safely reconstructable: ${captured.reason}`; + throw error; + } + state.throws.set(ordinal, captured.snapshot); state.entries.push({ provider, kind: "review", ordinal, request_sha256: requestHash, - response_sha256: sha256(message), + response_sha256: captured.sha256, outcome: "throw", + throw_snapshot: captured.snapshot, }); throw error; } @@ -1375,15 +1925,20 @@ function captureReviewerAdapters( }); return response; } catch (error) { - const message = error instanceof Error ? error.message : String(error); - state.errors.set(ordinal, message); + const captured = captureThrowableSnapshot(error); + if (!captured.ok) { + state.mismatch ??= `${provider} complete throw ${ordinal} is not safely reconstructable: ${captured.reason}`; + throw error; + } + state.throws.set(ordinal, captured.snapshot); state.entries.push({ provider, kind: "complete", ordinal, request_sha256: requestHash, - response_sha256: sha256(message), + response_sha256: captured.sha256, outcome: "throw", + throw_snapshot: captured.snapshot, }); throw error; } @@ -1401,6 +1956,7 @@ function replayReviewerAdapters( ): { adapters: Partial>; consumed: () => boolean } { const out: Partial> = {}; let cursor = 0; + const preflightCursors = new Map(); const mismatch = (message: string): void => { capture.mismatch ??= message; }; @@ -1410,7 +1966,28 @@ function replayReviewerAdapters( if (!adapter) continue; out[provider] = { id: adapter.id, - preflight: (cfg) => adapter.preflight(cfg), + async preflight(cfg) { + const ordinal = preflightCursors.get(provider) ?? 0; + preflightCursors.set(provider, ordinal + 1); + const expected = capture.preflights.get(provider)?.[ordinal]; + const requestHash = preflightRequestHash(provider, ordinal, cfg); + if (expected === undefined || expected.request_sha256 !== requestHash) { + mismatch(`${provider} preflight request ${ordinal} did not match baseline identity`); + throw new Error(capture.mismatch ?? "preflight replay mismatch"); + } + if (expected.outcome === "throw") { + if (sha256(canonicalJson(expected.throw_snapshot)) !== expected.response_sha256) { + mismatch(`${provider} preflight throw ${ordinal} failed snapshot hash validation`); + throw new Error(capture.mismatch ?? "preflight replay mismatch"); + } + throw replayThrowableSnapshot(expected.throw_snapshot); + } + if (sha256(stableJson(expected.value)) !== expected.response_sha256) { + mismatch(`${provider} preflight response ${ordinal} failed baseline hash validation`); + throw new Error(capture.mismatch ?? "preflight replay mismatch"); + } + return structuredClone(expected.value); + }, async review(input) { const ordinal = cursor++; const requestHash = reviewRequestHash(provider, ordinal, input); @@ -1435,7 +2012,19 @@ function replayReviewerAdapters( statusDetail: capture.mismatch ?? "replay mismatch", }; } - if (expected.outcome === "throw") throw new Error(capture.errors.get(ordinal)); + if (expected.outcome === "throw") { + const snapshot = capture.throws.get(ordinal); + if ( + snapshot === undefined || + expected.throw_snapshot === undefined || + sha256(canonicalJson(snapshot)) !== expected.response_sha256 || + canonicalJson(snapshot) !== canonicalJson(expected.throw_snapshot) + ) { + mismatch(`${provider} review throw ${ordinal} failed snapshot hash validation`); + throw new Error(capture.mismatch ?? "replay throw mismatch"); + } + throw replayThrowableSnapshot(snapshot); + } if ( stored?.kind !== "review" || sha256(stableJson(stored.value)) !== expected.response_sha256 @@ -1473,7 +2062,19 @@ function replayReviewerAdapters( ); throw new Error(capture.mismatch ?? "replay mismatch"); } - if (expected.outcome === "throw") throw new Error(capture.errors.get(ordinal)); + if (expected.outcome === "throw") { + const snapshot = capture.throws.get(ordinal); + if ( + snapshot === undefined || + expected.throw_snapshot === undefined || + sha256(canonicalJson(snapshot)) !== expected.response_sha256 || + canonicalJson(snapshot) !== canonicalJson(expected.throw_snapshot) + ) { + mismatch(`${provider} complete throw ${ordinal} failed snapshot hash validation`); + throw new Error(capture.mismatch ?? "replay throw mismatch"); + } + throw replayThrowableSnapshot(snapshot); + } if ( stored?.kind !== "complete" || sha256(stored.value) !== expected.response_sha256 @@ -1492,17 +2093,24 @@ function replayReviewerAdapters( return { adapters: out, consumed: () => { - if (cursor === capture.entries.length) return true; - mismatch(`replay consumed ${cursor}/${capture.entries.length} captured provider responses`); - return false; + if (cursor !== capture.entries.length) { + mismatch(`replay consumed ${cursor}/${capture.entries.length} captured provider responses`); + return false; + } + for (const [provider, preflights] of capture.preflights) { + const consumed = preflightCursors.get(provider) ?? 0; + if (consumed !== preflights.length) { + mismatch( + `replay consumed ${consumed}/${preflights.length} captured ${provider} preflights`, + ); + return false; + } + } + return true; }, }; } -function relativeArtifact(fromDir: string, path: string): string { - return relative(fromDir, path).split("\\").join("/"); -} - function matrixVariantProvenanceMismatch(baseline: BenchResult, variant: BenchResult): string[] { const reasons: string[] = []; const baseIntegrity = baseline.provenance.integrity; @@ -1575,19 +2183,25 @@ function validateBenchResultTracePairs( return { ok: true }; } -function matrixPolicyProvenance(result: BenchResult, ablatedPassId: PolicyPassId | null) { +function matrixPolicyProvenance( + result: BenchResult, + ablatedPassId: PolicyPassId | null, + traceSet: { ref: string; sha256: string } | null, +) { const policyRows = result.cases.map((caseResult) => caseResult.policy_trace); - const complete = policyRows.length > 0 && policyRows.every((row) => row?.authoritative === true); + const complete = + traceSet !== null && + policyRows.length > 0 && + policyRows.every((row) => row?.authoritative === true); const rawResponseSha256 = policyRows.flatMap((row) => row?.trace?.raw_response_sha256 ?? []); - const traceSha256 = sha256(canonicalJson(policyRows)); return { catalog_version: POLICY_CATALOG_VERSION, ablated_pass_id: ablatedPassId, trace_status: complete ? ("complete" as const) : ("not-run" as const), ...(complete ? { - trace_ref: `inline-policy-trace-set/${traceSha256}.json`, - trace_sha256: traceSha256, + trace_ref: traceSet.ref, + trace_sha256: traceSet.sha256, } : {}), raw_response_sha256: rawResponseSha256, @@ -1596,6 +2210,114 @@ function matrixPolicyProvenance(result: BenchResult, ablatedPassId: PolicyPassId }; } +function verifyResultTraceArtifacts( + root: string, + result: BenchResult, +): { ok: true } | { ok: false; reason: string } { + for (const row of result.cases) { + const policy = row.policy_trace; + if ( + policy?.authoritative !== true || + policy.trace === undefined || + policy.trace_ref === undefined || + policy.trace_sha256 === undefined + ) { + return { ok: false, reason: `case ${row.id} has no authoritative persisted trace` }; + } + const verified = verifyBenchArtifactReference({ + root, + ref: policy.trace_ref, + sha256: policy.trace_sha256, + kind: "policy-trace", + }); + if (!verified.ok) { + return { ok: false, reason: `case ${row.id} trace ${verified.reason}` }; + } + if (canonicalJson(verified.value) !== canonicalJson(policy.trace)) { + return { ok: false, reason: `case ${row.id} trace embedded identity mismatch` }; + } + } + return { ok: true }; +} + +function publishResultTraces( + stagingRoot: string, + outputRoot: string, + results: readonly BenchResult[], +): { ok: true } | { ok: false; reason: string } { + if (!ensureDirectoryWithoutSymlinks(join(outputRoot, "artifacts"))) { + return { ok: false, reason: "trace output path is unsafe" }; + } + for (const result of results) { + const staged = verifyResultTraceArtifacts(stagingRoot, result); + if (!staged.ok) return staged; + for (const row of result.cases) { + const policy = row.policy_trace; + if ( + policy?.trace === undefined || + policy.trace_ref === undefined || + policy.trace_sha256 === undefined + ) { + return { ok: false, reason: `case ${row.id} trace missing before publish` }; + } + const dateMatch = policy.trace_ref.match( + /^artifacts\/policy-traces\/(\d{4})\/(\d{2})\/(\d{2})\//, + ); + if (dateMatch === null) return { ok: false, reason: `case ${row.id} trace ref invalid` }; + const stored = writePolicyTrace({ + auditDir: join(outputRoot, "artifacts", "policy-traces"), + trace: policy.trace, + now: new Date(`${dateMatch[1]}-${dateMatch[2]}-${dateMatch[3]}T00:00:00.000Z`), + }); + if ( + stored.status !== "complete" || + `artifacts/policy-traces/${stored.ref}` !== policy.trace_ref || + stored.sha256 !== policy.trace_sha256 + ) { + return { ok: false, reason: `case ${row.id} trace publish failed` }; + } + } + } + for (const result of results) { + const published = verifyResultTraceArtifacts(outputRoot, result); + if (!published.ok) return published; + } + return { ok: true }; +} + +function traceSetRun( + label: string, + ablatedPassId: PolicyPassId | null, + result: BenchResult, + artifact: { ref: string; sha256: string }, +) { + return { + label, + ablated_pass_id: ablatedPassId, + result: { path: artifact.ref, sha256: artifact.sha256 }, + traces: result.cases.map((row) => { + const policy = row.policy_trace; + if ( + policy?.trace === undefined || + policy.trace_ref === undefined || + policy.trace_sha256 === undefined + ) { + throw new Error(`missing persisted policy trace for ${row.id}`); + } + return { + case_id: row.id, + repeat: row.repeat ?? 1, + trace_ref: policy.trace_ref, + trace_sha256: policy.trace_sha256, + effective_config_sha256: policy.effective_config_sha256, + request_identity_sha256: policy.request_identity_sha256, + final_identity_sha256: policy.final_identity_sha256, + raw_response_sha256: policy.trace.raw_response_sha256, + }; + }), + }; +} + /** * Ablation matrix (spec §8): run the corpus once as a BASELINE (full suppression) * and once per `--ablate` layer with that ONE layer turned off, then report the @@ -1637,19 +2359,12 @@ export async function runBenchMatrix(input: BenchMatrixInput): Promise [passId, join(artifactDir, `no-${passId}.result.json`)]), - ); - for (const path of [matrixPath, baselinePath, responseManifestPath, ...variantPaths.values()]) { - if (existsSync(path)) { - return { - exitCode: 2, - stdout: "", - stderr: `bench matrix: output already exists (immutable): ${path}\n`, - }; - } + if (existsSync(matrixPath)) { + return { + exitCode: 2, + stdout: "", + stderr: `bench matrix: output already exists (immutable): ${matrixPath}\n`, + }; } let baselineConfig: ReviewgateConfig; @@ -1708,8 +2423,9 @@ export async function runBenchMatrix(input: BenchMatrixInput): Promise left.ordinal - right.ordinal); - - const manifest = { - schema: "reviewgate.bench.reviewer-response-hashes.v1", + if (capture.mismatch !== null) { + return { + exitCode: 4, + stdout: "", + stderr: `bench matrix: benchmark-invalid — ${capture.mismatch}\n`, + }; + } + const manifest = BenchResponseManifestSchema.parse({ + schema: "reviewgate.bench.provider-response-hashes.v2", entries: [...capture.entries], - }; - const responseManifestTempPath = join(work, "reviewer-responses.sha256.json"); - writeFileSync(responseManifestTempPath, `${JSON.stringify(manifest, null, 2)}\n`); - - const dv = (b: Metric, v: Metric): number => (b.value ?? 0) - (v.value ?? 0); - const baselineHash = sha256File(baselineRun.tempPath); - const variants: MatrixVariant[] = [ - { - label: "baseline", - ablation: "", - class: "baseline", - precision: baseline.aggregate.precision, - recall: baseline.aggregate.recall, - clean_fp_rate: baseline.aggregate.clean_fp_rate, - delta: null, - authoritative: isAuthoritative(baseline).ok, - result_ref: relativeArtifact(artifactDir, baselinePath), - result_sha256: baselineHash, - policy: matrixPolicyProvenance(baseline, null), - }, - ]; - const variantArtifactRefs: Array<{ path: string; sha256: string }> = []; - const completedVariantArtifacts: Array<{ tempPath: string; finalPath: string }> = []; + }); + const executed: Array<{ + label: string; + passId: PolicyPassId | null; + result: BenchResult; + }> = [{ label: "baseline", passId: null, result: baseline }]; for (const passId of ablatedPassIds) { const replay = replayReviewerAdapters(underlying, capture); const variantRun = await runVariant(`no-${passId}`, [passId], replay.adapters, false); replay.consumed(); - const finalPath = variantPaths.get(passId); - if (!finalPath) throw new Error(`missing artifact path for ${passId}`); if (variantRun.output.exitCode !== 0 || !variantRun.result || capture.mismatch) { - mkdirSync(artifactDir, { recursive: true }); - writeFileSync(baselinePath, readFileSync(baselineRun.tempPath)); - writeFileSync(responseManifestPath, readFileSync(responseManifestTempPath)); - if (existsSync(variantRun.tempPath)) { - writeFileSync(finalPath, readFileSync(variantRun.tempPath)); - } return { exitCode: variantRun.output.exitCode === 0 ? 4 : variantRun.output.exitCode, stdout: variantRun.output.stdout, @@ -1837,12 +2543,6 @@ export async function runBenchMatrix(input: BenchMatrixInput): Promise 0) { - mkdirSync(artifactDir, { recursive: true }); - writeFileSync(baselinePath, readFileSync(baselineRun.tempPath)); - writeFileSync(responseManifestPath, readFileSync(responseManifestTempPath)); - if (existsSync(variantRun.tempPath)) { - writeFileSync(finalPath, readFileSync(variantRun.tempPath)); - } return { exitCode: 4, stdout: "", @@ -1851,44 +2551,179 @@ export async function runBenchMatrix(input: BenchMatrixInput): Promise run.result), + ); + if (!publishedTraces.ok) { + return { + exitCode: 4, + stdout: "", + stderr: `bench matrix: benchmark-invalid — policy trace artifact ${publishedTraces.reason}\n`, + }; + } + const responseArtifact = persistBenchArtifact({ + root: artifactDir, + kind: "response-manifest", + value: manifest, + }); + if (!responseArtifact.ok) { + return { + exitCode: 4, + stdout: "", + stderr: `bench matrix: benchmark-invalid — response manifest ${responseArtifact.reason}\n`, + }; + } + const resultArtifacts = new Map(); + for (const run of executed) { + const artifact = persistBenchArtifact({ + root: artifactDir, + kind: "bench-result", + value: run.result, }); - variants.push({ - label: `-${passId}`, - ablation: passId, - class: "A", - precision: r.aggregate.precision, - recall: r.aggregate.recall, - clean_fp_rate: r.aggregate.clean_fp_rate, - delta: { - precision: dv(baseline.aggregate.precision, r.aggregate.precision), - recall: dv(baseline.aggregate.recall, r.aggregate.recall), - clean_fp_rate: dv(baseline.aggregate.clean_fp_rate, r.aggregate.clean_fp_rate), - }, - authoritative: isAuthoritative(r).ok, - result_ref: relativeArtifact(artifactDir, finalPath), - result_sha256: resultHash, - policy: matrixPolicyProvenance(r, passId), + if (!artifact.ok) { + return { + exitCode: 4, + stdout: "", + stderr: `bench matrix: benchmark-invalid — result ${run.label} ${artifact.reason}\n`, + }; + } + const traceVerification = verifyResultTraceArtifacts(artifactDir, run.result); + if (!traceVerification.ok) { + return { + exitCode: 4, + stdout: "", + stderr: `bench matrix: benchmark-invalid — ${traceVerification.reason}\n`, + }; + } + resultArtifacts.set(run.label, { ref: artifact.ref, sha256: artifact.sha256 }); + } + const traceSet = BenchPolicyTraceSetSchema.parse({ + schema: "reviewgate.bench.policy-trace-set.v1", + catalog_version: POLICY_CATALOG_VERSION, + response_manifest: { + path: responseArtifact.ref, + sha256: responseArtifact.sha256, + }, + runs: executed.map((run) => { + const artifact = resultArtifacts.get(run.label); + if (artifact === undefined) throw new Error(`missing result artifact for ${run.label}`); + return traceSetRun(run.label, run.passId, run.result, artifact); + }), + }); + const traceSetArtifact = persistBenchArtifact({ + root: artifactDir, + kind: "policy-trace-set", + value: traceSet, + }); + if (!traceSetArtifact.ok) { + return { + exitCode: 4, + stdout: "", + stderr: `bench matrix: benchmark-invalid — policy trace set ${traceSetArtifact.reason}\n`, + }; + } + const finalResponseVerification = verifyBenchArtifactReference({ + root: artifactDir, + ref: responseArtifact.ref, + sha256: responseArtifact.sha256, + kind: "response-manifest", + }); + if ( + !finalResponseVerification.ok || + canonicalJson(finalResponseVerification.value) !== canonicalJson(manifest) + ) { + return { + exitCode: 4, + stdout: "", + stderr: `bench matrix: benchmark-invalid — response manifest final verification ${finalResponseVerification.ok ? "identity-mismatch" : finalResponseVerification.reason}\n`, + }; + } + for (const run of executed) { + const artifact = resultArtifacts.get(run.label); + if (artifact === undefined) throw new Error(`missing result artifact for ${run.label}`); + const finalResultVerification = verifyBenchArtifactReference({ + root: artifactDir, + ref: artifact.ref, + sha256: artifact.sha256, + kind: "bench-result", }); + const finalTraceVerification = verifyResultTraceArtifacts(artifactDir, run.result); + if ( + !finalResultVerification.ok || + canonicalJson(finalResultVerification.value) !== canonicalJson(run.result) || + !finalTraceVerification.ok + ) { + const reason = !finalResultVerification.ok + ? finalResultVerification.reason + : !finalTraceVerification.ok + ? finalTraceVerification.reason + : "identity-mismatch"; + return { + exitCode: 4, + stdout: "", + stderr: `bench matrix: benchmark-invalid — result ${run.label} final verification ${reason}\n`, + }; + } } - + const finalTraceSetVerification = verifyBenchArtifactReference({ + root: artifactDir, + ref: traceSetArtifact.ref, + sha256: traceSetArtifact.sha256, + kind: "policy-trace-set", + }); + if ( + !finalTraceSetVerification.ok || + canonicalJson(finalTraceSetVerification.value) !== canonicalJson(traceSet) + ) { + return { + exitCode: 4, + stdout: "", + stderr: `bench matrix: benchmark-invalid — policy trace set final verification ${finalTraceSetVerification.ok ? "identity-mismatch" : finalTraceSetVerification.reason}\n`, + }; + } + const traceSetIdentity = { ref: traceSetArtifact.ref, sha256: traceSetArtifact.sha256 }; + const dv = (b: Metric, v: Metric): number => (b.value ?? 0) - (v.value ?? 0); + const variants: MatrixVariant[] = executed.map((run) => { + const artifact = resultArtifacts.get(run.label); + if (artifact === undefined) throw new Error(`missing result artifact for ${run.label}`); + const baselineRow = run.passId === null; + return { + label: run.label, + ablation: run.passId ?? "", + class: baselineRow ? "baseline" : "A", + precision: run.result.aggregate.precision, + recall: run.result.aggregate.recall, + clean_fp_rate: run.result.aggregate.clean_fp_rate, + delta: baselineRow + ? null + : { + precision: dv(baseline.aggregate.precision, run.result.aggregate.precision), + recall: dv(baseline.aggregate.recall, run.result.aggregate.recall), + clean_fp_rate: dv( + baseline.aggregate.clean_fp_rate, + run.result.aggregate.clean_fp_rate, + ), + }, + authoritative: isAuthoritative(run.result).ok, + result_ref: artifact.ref, + result_sha256: artifact.sha256, + policy: matrixPolicyProvenance(run.result, run.passId, traceSetIdentity), + }; + }); const allAuthoritative = variants.every((variant) => variant.authoritative === true); + const baselineArtifact = resultArtifacts.get("baseline"); + if (baselineArtifact === undefined) throw new Error("missing baseline result artifact"); const matrix: BenchMatrix = { schema: "reviewgate.bench.matrix.v1", provenance: baseline.provenance, @@ -1896,24 +2731,40 @@ export async function runBenchMatrix(input: BenchMatrixInput): Promise { + const artifact = resultArtifacts.get(run.label); + if (artifact === undefined) throw new Error(`missing result artifact for ${run.label}`); + return { path: artifact.ref, sha256: artifact.sha256 }; + }), reviewer_responses: { - path: relativeArtifact(artifactDir, responseManifestPath), - sha256: sha256File(responseManifestTempPath), + path: responseArtifact.ref, + sha256: responseArtifact.sha256, + }, + policy_trace_set: { + path: traceSetArtifact.ref, + sha256: traceSetArtifact.sha256, }, }, }; BenchMatrixSchema.parse(matrix); - mkdirSync(artifactDir, { recursive: true }); - writeFileSync(baselinePath, readFileSync(baselineRun.tempPath)); - writeFileSync(responseManifestPath, readFileSync(responseManifestTempPath)); - for (const artifact of completedVariantArtifacts) { - writeFileSync(artifact.finalPath, readFileSync(artifact.tempPath)); + if (!ensureDirectoryWithoutSymlinks(artifactDir)) { + return { + exitCode: 4, + stdout: "", + stderr: "bench matrix: benchmark-invalid — output path is not a safe directory\n", + }; + } + const matrixCreated = writeFileIfAbsent(matrixPath, canonicalJson(matrix), { mode: 0o600 }); + if (!matrixCreated) { + return { + exitCode: 2, + stdout: "", + stderr: `bench matrix: output already exists (immutable): ${matrixPath}\n`, + }; } - writeFileSync(matrixPath, `${JSON.stringify(matrix, null, 2)}\n`); if (input.authoritative && !allAuthoritative) { return { exitCode: 4, diff --git a/src/schemas/bench-result.ts b/src/schemas/bench-result.ts index 0eb9019..2194fa1 100644 --- a/src/schemas/bench-result.ts +++ b/src/schemas/bench-result.ts @@ -190,6 +190,192 @@ export const CaseCriticSchema = z const Sha256Schema = z.string().regex(/^[0-9a-f]{64}$/); const PolicyPassIdSchema = z.enum(POLICY_PASS_IDS); +const PolicyTraceArtifactRefSchema = z + .string() + .regex( + /^artifacts\/policy-traces\/\d{4}\/\d{2}\/\d{2}\/policy\/[0-9a-f]{12}-i(?:0|[1-9]\d*)-[0-9a-f]{12}\.json$/, + ); +const PolicyTraceSetArtifactRefSchema = z + .string() + .regex(/^artifacts\/policy-trace-sets\/[0-9a-f]{64}\.json$/); +const ResultArtifactRefSchema = z.string().regex(/^artifacts\/results\/[0-9a-f]{64}\.json$/); +const ResponseManifestArtifactRefSchema = z + .string() + .regex(/^artifacts\/responses\/[0-9a-f]{64}\.json$/); + +export type ThrowableSafeValue = + | null + | string + | number + | boolean + | ThrowableSafeValue[] + | { [key: string]: ThrowableSafeValue }; + +const SensitiveThrowableFieldSchema = z + .string() + .min(1) + .refine( + (value) => + !/(?:authorization|cookie|credential|password|passwd|secret|token|api[_-]?key|private[_-]?key)/i.test( + value, + ), + "throwable field contains sensitive key", + ); +function isSafeThrowableString(value: string): boolean { + const hasUnsafeControlCharacter = Array.from(value).some((character) => { + const codePoint = character.codePointAt(0) ?? 0; + return ( + codePoint <= 8 || + codePoint === 11 || + codePoint === 12 || + (codePoint >= 14 && codePoint <= 31) || + codePoint === 127 + ); + }); + return ( + !/(?:\/Users\/|\/home\/|\/private\/|\/tmp\/|[A-Za-z]:[\\/]|\bBearer\s+|\bsk-[A-Za-z0-9_-]{8,})/i.test( + value, + ) && !hasUnsafeControlCharacter + ); +} +const SafeThrowableStringSchema = z + .string() + .refine(isSafeThrowableString, "throwable string contains unsafe data"); +const SafeThrowableNameSchema = z + .string() + .min(1) + .refine(isSafeThrowableString, "throwable string contains unsafe data"); + +export const ThrowableSafeValueSchema: z.ZodType = z.lazy(() => + z.union([ + z.null(), + SafeThrowableStringSchema, + z.number().finite(), + z.boolean(), + z.array(ThrowableSafeValueSchema), + z.record(SensitiveThrowableFieldSchema, ThrowableSafeValueSchema), + ]), +); + +export type CapturedThrowableSnapshot = + | { kind: "primitive"; primitive_type: "string"; value: string } + | { kind: "primitive"; primitive_type: "undefined" | "null" } + | { + kind: "error"; + error_type: "Error" | "SandboxUnavailableError"; + name: string; + message: string; + cause?: CapturedThrowableSnapshot | undefined; + fields: Array<{ key: string; value: ThrowableSafeValue; enumerable: boolean }>; + }; + +export const CAPTURED_THROWABLE_FIELD_KEYS = [ + "code", + "context", + "errno", + "exitCode", + "killed", + "retryable", + "signal", + "status", + "statusCode", + "syscall", + "timedOut", +] as const; +const CapturedThrowableFieldKeySchema = z.enum(CAPTURED_THROWABLE_FIELD_KEYS); + +export const CapturedThrowableSnapshotSchema: z.ZodType = z.lazy(() => + z.union([ + z + .object({ + kind: z.literal("primitive"), + primitive_type: z.literal("string"), + value: SafeThrowableStringSchema, + }) + .strict(), + z.object({ kind: z.literal("primitive"), primitive_type: z.literal("undefined") }).strict(), + z.object({ kind: z.literal("primitive"), primitive_type: z.literal("null") }).strict(), + z + .object({ + kind: z.literal("error"), + error_type: z.enum(["Error", "SandboxUnavailableError"]), + name: SafeThrowableNameSchema, + message: SafeThrowableStringSchema, + cause: CapturedThrowableSnapshotSchema.optional(), + fields: z.array( + z + .object({ + key: CapturedThrowableFieldKeySchema, + value: ThrowableSafeValueSchema, + enumerable: z.boolean(), + }) + .strict(), + ), + }) + .strict() + .superRefine((value, ctx) => { + const keys = value.fields.map((field) => field.key); + if ( + keys.some((key, index) => index > 0 && (keys[index - 1] ?? "").localeCompare(key) >= 0) + ) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + path: ["fields"], + message: "throwable fields must be uniquely sorted", + }); + } + }), + ]), +); + +export const BenchResponseManifestSchema = z + .object({ + schema: z.literal("reviewgate.bench.provider-response-hashes.v2"), + entries: z.array( + z + .object({ + provider: z.string().min(1), + kind: z.enum(["review", "complete"]), + ordinal: z.number().int().nonnegative(), + request_sha256: Sha256Schema, + response_sha256: Sha256Schema, + outcome: z.enum(["return", "throw"]), + throw_snapshot: CapturedThrowableSnapshotSchema.optional(), + }) + .strict() + .superRefine((value, ctx) => { + if ((value.outcome === "throw") !== (value.throw_snapshot !== undefined)) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + path: ["throw_snapshot"], + message: "throw outcomes require exactly one typed snapshot", + }); + } + if (value.throw_snapshot !== undefined) { + const actual = createHash("sha256") + .update(Buffer.from(canonicalJson(value.throw_snapshot), "utf8")) + .digest("hex"); + if (actual !== value.response_sha256) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + path: ["response_sha256"], + message: "throw snapshot hash mismatch", + }); + } + } + }), + ), + }) + .strict() + .superRefine((value, ctx) => { + if (value.entries.some((entry, index) => entry.ordinal !== index)) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + path: ["entries"], + message: "response ordinals must be globally contiguous", + }); + } + }); export const BenchPolicyTraceRunSchema = z .object({ @@ -198,7 +384,7 @@ export const BenchPolicyTraceRunSchema = z catalog_version: z.string().min(1), requested_ablations: z.array(PolicyPassIdSchema), trace: PolicyTraceSchema.optional(), - trace_ref: z.string().min(1).optional(), + trace_ref: PolicyTraceArtifactRefSchema.optional(), trace_sha256: Sha256Schema.optional(), request_identity_sha256: Sha256Schema, effective_config_sha256: Sha256Schema, @@ -227,11 +413,21 @@ export const BenchPolicyTraceRunSchema = z message: "embedded policy trace hash mismatch", }); } - if (value.trace_ref !== `inline-policy-trace/${actualSha256}.json`) { + const traceRefMatch = value.trace_ref?.match( + /^artifacts\/policy-traces\/\d{4}\/\d{2}\/\d{2}\/policy\/([0-9a-f]{12})-i(0|[1-9]\d*)-([0-9a-f]{12})\.json$/, + ); + const runSha12 = createHash("sha256").update(value.trace.run_id).digest("hex").slice(0, 12); + if ( + traceRefMatch === null || + traceRefMatch === undefined || + traceRefMatch[1] !== runSha12 || + Number(traceRefMatch[2]) !== value.trace.iter || + traceRefMatch[3] !== actualSha256.slice(0, 12) + ) { ctx.addIssue({ code: z.ZodIssueCode.custom, path: ["trace_ref"], - message: "embedded policy trace reference mismatch", + message: "persisted policy trace reference mismatch", }); } const finalSha256 = createHash("sha256") @@ -474,6 +670,124 @@ export const BenchResultSchema = z }) .strict(); +const BenchPolicyTraceSetRunSchema = z + .object({ + label: z.string().min(1), + ablated_pass_id: PolicyPassIdSchema.nullable(), + result: z.object({ path: ResultArtifactRefSchema, sha256: Sha256Schema }).strict(), + traces: z.array( + z + .object({ + case_id: z.string().min(1), + repeat: z.number().int().positive(), + trace_ref: PolicyTraceArtifactRefSchema, + trace_sha256: Sha256Schema, + effective_config_sha256: Sha256Schema, + request_identity_sha256: Sha256Schema, + final_identity_sha256: Sha256Schema, + raw_response_sha256: z.array(Sha256Schema), + }) + .strict(), + ), + }) + .strict(); + +export const BenchPolicyTraceSetSchema = z + .object({ + schema: z.literal("reviewgate.bench.policy-trace-set.v1"), + catalog_version: z.literal(POLICY_CATALOG_VERSION), + response_manifest: z + .object({ path: ResponseManifestArtifactRefSchema, sha256: Sha256Schema }) + .strict(), + runs: z.array(BenchPolicyTraceSetRunSchema).min(2), + }) + .strict() + .superRefine((value, ctx) => { + if ( + value.response_manifest.path !== `artifacts/responses/${value.response_manifest.sha256}.json` + ) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + path: ["response_manifest", "path"], + message: "response manifest path/hash identity mismatch", + }); + } + const baseline = value.runs[0]; + if (baseline?.label !== "baseline" || baseline.ablated_pass_id !== null) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + path: ["runs", 0], + message: "trace set must start with the unabated baseline", + }); + return; + } + const labels = new Set(); + const passIds = new Set(); + for (const [runIndex, run] of value.runs.entries()) { + if (run.result.path !== `artifacts/results/${run.result.sha256}.json`) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + path: ["runs", runIndex, "result", "path"], + message: "result path/hash identity mismatch", + }); + } + if (labels.has(run.label)) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + path: ["runs", runIndex, "label"], + message: "trace-set run labels must be unique", + }); + } + labels.add(run.label); + if (runIndex > 0) { + if (run.ablated_pass_id === null || passIds.has(run.ablated_pass_id)) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + path: ["runs", runIndex, "ablated_pass_id"], + message: "counterfactual trace sets require one unique pass ID", + }); + } else { + passIds.add(run.ablated_pass_id); + } + } + if (baseline === undefined || run.traces.length !== baseline.traces.length) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + path: ["runs", runIndex, "traces"], + message: "trace-set runs must have identical case cardinality", + }); + continue; + } + for (const [traceIndex, trace] of run.traces.entries()) { + if (!trace.trace_ref.endsWith(`-${trace.trace_sha256.slice(0, 12)}.json`)) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + path: ["runs", runIndex, "traces", traceIndex, "trace_ref"], + message: "policy trace path/hash identity mismatch", + }); + } + const base = baseline.traces[traceIndex]; + if ( + base === undefined || + trace.case_id !== base.case_id || + trace.repeat !== base.repeat || + trace.effective_config_sha256 !== base.effective_config_sha256 || + trace.request_identity_sha256 !== base.request_identity_sha256 || + trace.raw_response_sha256.length !== base.raw_response_sha256.length || + trace.raw_response_sha256.some( + (hash, hashIndex) => hash !== base.raw_response_sha256[hashIndex], + ) + ) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + path: ["runs", runIndex, "traces", traceIndex], + message: "trace-set pair identity mismatch", + }); + } + } + } + }); + // reviewgate bench matrix (spec §8) — the ablation Δ table. One variant per row: // the baseline (full suppression) plus one row per ablated layer, each carrying // its point metrics and the signed delta vs. baseline. @@ -504,7 +818,7 @@ export const MatrixVariantSchema = z catalog_version: z.string().min(1), ablated_pass_id: PolicyPassIdSchema.nullable(), trace_status: z.enum(["complete", "not-run", "error", "overflow"]), - trace_ref: z.string().min(1).optional(), + trace_ref: PolicyTraceSetArtifactRefSchema.optional(), trace_sha256: Sha256Schema.optional(), raw_response_sha256: z.array(Sha256Schema), authoritative: z.boolean(), @@ -560,6 +874,7 @@ export const BenchMatrixSchema = z baseline: MatrixArtifactRefSchema, variants: z.array(MatrixArtifactRefSchema), reviewer_responses: MatrixArtifactRefSchema, + policy_trace_set: MatrixArtifactRefSchema.optional(), }) .strict() .optional(), @@ -573,6 +888,8 @@ export type PhasesSnapshot = z.infer; export type Provenance = z.infer; export type CaseResult = z.infer; export type BenchPolicyTraceRun = z.infer; +export type BenchResponseManifest = z.infer; +export type BenchPolicyTraceSet = z.infer; export type SpreadStat = z.infer; export type Stability = z.infer; export type ProviderResult = z.infer; diff --git a/tests/unit/bench-matrix.test.ts b/tests/unit/bench-matrix.test.ts index 701f2c9..a530cb8 100644 --- a/tests/unit/bench-matrix.test.ts +++ b/tests/unit/bench-matrix.test.ts @@ -6,7 +6,15 @@ import { describe, expect, it } from "bun:test"; import { execFileSync } from "node:child_process"; import { createHash } from "node:crypto"; -import { existsSync, mkdirSync, mkdtempSync, readFileSync, writeFileSync } from "node:fs"; +import { + existsSync, + mkdirSync, + mkdtempSync, + readFileSync, + symlinkSync, + unlinkSync, + writeFileSync, +} from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { canonicalJson } from "../../src/audit/canonical.ts"; @@ -15,10 +23,21 @@ import { type AuthoritativeTraceRun, validateAuthoritativeTracePair, } from "../../src/bench/runner.ts"; -import { runBenchMatrix } from "../../src/cli/commands/bench.ts"; +import { + captureThrowableSnapshot, + replayThrowableSnapshot, + runBenchMatrix, + verifyBenchArtifactReference, +} from "../../src/cli/commands/bench.ts"; import { POLICY_CATALOG_VERSION, POLICY_PASS_IDS } from "../../src/core/policy/catalog.ts"; import type { ProviderAdapter, ReviewResult } from "../../src/providers/adapter-base.ts"; -import { BenchMatrixSchema, BenchResultSchema } from "../../src/schemas/bench-result.ts"; +import { SandboxUnavailableError } from "../../src/sandbox/errors.ts"; +import { + BenchMatrixSchema, + BenchPolicyTraceSetSchema, + BenchResponseManifestSchema, + BenchResultSchema, +} from "../../src/schemas/bench-result.ts"; import type { Finding } from "../../src/schemas/finding.ts"; import type { PolicyTrace } from "../../src/schemas/policy-trace.ts"; @@ -218,7 +237,7 @@ function traceRun(ablated: PolicyTrace["ablated"]): AuthoritativeTraceRun { catalogVersion: POLICY_CATALOG_VERSION, requestedAblations: ablated, trace, - traceRef: `inline-policy-trace/${traceSha256}.json`, + traceRef: `artifacts/policy-traces/2026/07/01/policy/${sha256(trace.run_id).slice(0, 12)}-i${trace.iter}-${traceSha256.slice(0, 12)}.json`, traceSha256, requestIdentitySha256: "c".repeat(64), effectiveConfigSha256: "d".repeat(64), @@ -257,10 +276,10 @@ describe("runBenchMatrix", () => { expect(ablated?.delta?.precision).toBeCloseTo(0.5, 10); expect(ablated?.delta?.clean_fp_rate).toBeCloseTo(-1, 10); const baselineResult = BenchResultSchema.parse( - JSON.parse(readFileSync(join(corpus, "baseline.result.json"), "utf8")), + JSON.parse(readFileSync(join(corpus, m.artifacts?.baseline.path ?? "missing"), "utf8")), ); const ablatedResult = BenchResultSchema.parse( - JSON.parse(readFileSync(join(corpus, "no-judgment.confidence.result.json"), "utf8")), + JSON.parse(readFileSync(join(corpus, m.artifacts?.variants[0]?.path ?? "missing"), "utf8")), ); expect(baselineResult.provenance.config_hash).toBe(ablatedResult.provenance.config_hash); expect(baselineResult.provenance.phases.confidence_floor).toBeGreaterThan(0); @@ -271,11 +290,387 @@ describe("runBenchMatrix", () => { expect(baseline?.policy?.authoritative).toBe(true); expect(ablated?.policy?.authoritative).toBe(true); expect(ablated?.policy?.ablated_pass_id).toBe("judgment.confidence"); + const resultRefs = [m.artifacts?.baseline, ...(m.artifacts?.variants ?? [])]; + for (const resultRef of resultRefs) { + expect(resultRef).toBeDefined(); + if (!resultRef) continue; + const persisted = BenchResultSchema.parse( + JSON.parse(readFileSync(join(corpus, resultRef.path), "utf8")), + ); + for (const row of persisted.cases) { + expect(row.policy_trace?.trace_ref).toStartWith("artifacts/policy-traces/"); + expect(existsSync(join(corpus, row.policy_trace?.trace_ref ?? "missing"))).toBe(true); + if ( + row.policy_trace?.trace_ref !== undefined && + row.policy_trace.trace_sha256 !== undefined + ) { + expect( + verifyBenchArtifactReference({ + root: corpus, + ref: row.policy_trace.trace_ref, + sha256: row.policy_trace.trace_sha256, + kind: "policy-trace", + }), + ).toMatchObject({ ok: true }); + expect(readFileSync(join(corpus, row.policy_trace.trace_ref), "utf8")).toBe( + canonicalJson(row.policy_trace.trace), + ); + } + } + } + const traceSetRef = ( + m.artifacts as typeof m.artifacts & { + policy_trace_set?: { path: string; sha256: string }; + } + )?.policy_trace_set; + expect(traceSetRef).toBeDefined(); + expect(existsSync(join(corpus, traceSetRef?.path ?? "missing"))).toBe(true); + if (traceSetRef) { + const traceSetBytes = readFileSync(join(corpus, traceSetRef.path), "utf8"); + const traceSet = BenchPolicyTraceSetSchema.parse(JSON.parse(traceSetBytes)); + expect(traceSetBytes).toBe(canonicalJson(traceSet)); + expect( + verifyBenchArtifactReference({ + root: corpus, + ref: traceSetRef.path, + sha256: traceSetRef.sha256, + kind: "policy-trace-set", + }), + ).toMatchObject({ ok: true }); + expect(traceSet.runs.map((run) => run.ablated_pass_id)).toEqual([ + null, + "judgment.confidence", + ]); + expect(traceSet.response_manifest.sha256).toBe( + m.artifacts?.reviewer_responses.sha256 ?? "missing", + ); + const tamperedTraceSet = { + ...traceSet, + runs: traceSet.runs.map((run, index) => + index === 1 ? { ...run, label: `${run.label}-tampered` } : run, + ), + }; + writeFileSync(join(corpus, traceSetRef.path), canonicalJson(tamperedTraceSet)); + expect( + verifyBenchArtifactReference({ + root: corpus, + ref: traceSetRef.path, + sha256: traceSetRef.sha256, + kind: "policy-trace-set", + }), + ).toEqual({ ok: false, reason: "hash-mismatch" }); + } // The Δ table renders. expect(res.stdout).toContain("ablation"); expect(res.stdout.toLowerCase()).toContain("baseline"); }); + it("fails closed when a published trace path is replaced by a symlink", async () => { + const corpus = newCorpus(); + const artifactDir = join(corpus, "symlink-attack"); + const firstOut = join(artifactDir, "first.json"); + const first = await runBenchMatrix({ + repoRoot: corpus, + corpus, + out: firstOut, + ablate: ["confidence-floor"], + adapters: { codex: stub() }, + now: () => new Date("2026-07-01T00:00:00Z"), + }); + expect(first.exitCode).toBe(0); + const firstMatrix = BenchMatrixSchema.parse(JSON.parse(readFileSync(firstOut, "utf8"))); + const baselineResult = BenchResultSchema.parse( + JSON.parse( + readFileSync(join(artifactDir, firstMatrix.artifacts?.baseline.path ?? "missing"), "utf8"), + ), + ); + const traceRef = baselineResult.cases[0]?.policy_trace?.trace_ref; + expect(traceRef).toBeDefined(); + if (traceRef === undefined) return; + const tracePath = join(artifactDir, traceRef); + const outside = join(corpus, "outside-trace.json"); + writeFileSync(outside, readFileSync(tracePath)); + unlinkSync(tracePath); + symlinkSync(outside, tracePath); + + const attacked = await runBenchMatrix({ + repoRoot: corpus, + corpus, + out: join(artifactDir, "attacked.json"), + ablate: ["confidence-floor"], + adapters: { codex: stub() }, + now: () => new Date("2026-07-01T00:00:00Z"), + }); + expect(attacked.exitCode).toBe(4); + expect(attacked.stdout).toBe(""); + expect(attacked.stderr).toContain("policy trace artifact"); + expect(existsSync(join(artifactDir, "attacked.json"))).toBe(false); + }); + + it("rejects missing, tampered, non-canonical, wrong-hash, traversing and symlink artifacts", () => { + const root = mkdtempSync(join(tmpdir(), "rg-bench-artifact-verifier-")); + const manifest = BenchResponseManifestSchema.parse({ + schema: "reviewgate.bench.provider-response-hashes.v2", + entries: [], + }); + const canonical = canonicalJson(manifest); + const canonicalSha = sha256(canonical); + const canonicalRef = `artifacts/responses/${canonicalSha}.json`; + mkdirSync(join(root, "artifacts", "responses"), { recursive: true }); + writeFileSync(join(root, canonicalRef), canonical, { mode: 0o600 }); + + expect( + verifyBenchArtifactReference({ + root, + ref: canonicalRef, + sha256: canonicalSha, + kind: "response-manifest", + }), + ).toMatchObject({ ok: true }); + + const missingSha = "1".repeat(64); + expect( + verifyBenchArtifactReference({ + root, + ref: `artifacts/responses/${missingSha}.json`, + sha256: missingSha, + kind: "response-manifest", + }), + ).toEqual({ ok: false, reason: "missing" }); + + const tamperedSha = "2".repeat(64); + const tamperedRef = `artifacts/responses/${tamperedSha}.json`; + writeFileSync(join(root, tamperedRef), canonical, { mode: 0o600 }); + expect( + verifyBenchArtifactReference({ + root, + ref: tamperedRef, + sha256: tamperedSha, + kind: "response-manifest", + }), + ).toEqual({ ok: false, reason: "hash-mismatch" }); + + const pretty = JSON.stringify(manifest, null, 2); + const prettySha = sha256(pretty); + const prettyRef = `artifacts/responses/${prettySha}.json`; + writeFileSync(join(root, prettyRef), pretty, { mode: 0o600 }); + expect( + verifyBenchArtifactReference({ + root, + ref: prettyRef, + sha256: prettySha, + kind: "response-manifest", + }), + ).toEqual({ ok: false, reason: "non-canonical" }); + + expect( + verifyBenchArtifactReference({ + root, + ref: canonicalRef, + sha256: "3".repeat(64), + kind: "response-manifest", + }), + ).toEqual({ ok: false, reason: "identity-mismatch" }); + expect( + verifyBenchArtifactReference({ + root, + ref: "../outside.json", + sha256: canonicalSha, + kind: "response-manifest", + }), + ).toEqual({ ok: false, reason: "invalid-reference" }); + + const symlinkManifest = BenchResponseManifestSchema.parse({ + schema: "reviewgate.bench.provider-response-hashes.v2", + entries: [ + { + provider: "codex", + kind: "review", + ordinal: 0, + request_sha256: "5".repeat(64), + response_sha256: "6".repeat(64), + outcome: "return", + }, + ], + }); + const symlinkBytes = canonicalJson(symlinkManifest); + const symlinkSha = sha256(symlinkBytes); + const symlinkRef = `artifacts/responses/${symlinkSha}.json`; + const symlinkTarget = join(root, "symlink-target.json"); + writeFileSync(symlinkTarget, symlinkBytes, { mode: 0o600 }); + symlinkSync(symlinkTarget, join(root, symlinkRef)); + expect( + verifyBenchArtifactReference({ + root, + ref: symlinkRef, + sha256: symlinkSha, + kind: "response-manifest", + }), + ).toEqual({ ok: false, reason: "not-a-file" }); + }); + + it("captures and reconstructs exact immutable throwable snapshots without cross-variant aliasing", () => { + const sandbox = captureThrowableSnapshot(new SandboxUnavailableError("sandbox unavailable")); + expect(sandbox.ok).toBe(true); + if (!sandbox.ok) return; + const sandboxA = replayThrowableSnapshot(sandbox.snapshot); + const sandboxB = replayThrowableSnapshot(sandbox.snapshot); + expect(sandboxA).toBeInstanceOf(SandboxUnavailableError); + expect(sandboxB).toBeInstanceOf(SandboxUnavailableError); + expect(sandboxA).not.toBe(sandboxB); + expect((sandboxA as Error).message).toBe("sandbox unavailable"); + expect(Object.isFrozen(sandbox.snapshot)).toBe(true); + + const cause = new Error("root cause"); + const ordinary = new Error("outer failure", { cause }) as Error & { + code: string; + retryable: boolean; + context: { attempt: number; labels: string[] }; + }; + ordinary.code = "E_RETRY"; + ordinary.retryable = true; + ordinary.context = { attempt: 2, labels: ["critic", "retry"] }; + const captured = captureThrowableSnapshot(ordinary); + expect(captured.ok).toBe(true); + if (!captured.ok) return; + const first = replayThrowableSnapshot(captured.snapshot) as typeof ordinary; + const second = replayThrowableSnapshot(captured.snapshot) as typeof ordinary; + expect(first).toBeInstanceOf(Error); + expect(first).not.toBe(second); + expect(first.cause).toBeInstanceOf(Error); + expect(first.cause).not.toBe(second.cause); + expect(first.code).toBe("E_RETRY"); + expect(first.retryable).toBe(true); + expect(first.context).toEqual({ attempt: 2, labels: ["critic", "retry"] }); + + const changedCode = new Error("outer failure") as Error & { code: string }; + changedCode.code = "E_OTHER"; + const changed = captureThrowableSnapshot(changedCode); + expect(changed.ok).toBe(true); + if (changed.ok) expect(changed.sha256).not.toBe(captured.sha256); + + for (const primitive of ["plain throw", undefined, null] as const) { + const primitiveCapture = captureThrowableSnapshot(primitive); + expect(primitiveCapture.ok).toBe(true); + if (primitiveCapture.ok) { + expect(replayThrowableSnapshot(primitiveCapture.snapshot)).toBe(primitive); + } + } + }); + + it("fails closed for non-reconstructable or sensitive thrown values", () => { + class CustomError extends Error {} + expect(captureThrowableSnapshot(new CustomError("custom"))).toMatchObject({ + ok: false, + reason: "unsupported-error-type", + }); + const secret = new Error("failed") as Error & { apiToken: string }; + secret.apiToken = "secret-value"; + expect(captureThrowableSnapshot(secret)).toMatchObject({ + ok: false, + reason: "sensitive-field", + }); + expect( + captureThrowableSnapshot(new Error("failed at /Users/alice/private/file")), + ).toMatchObject({ + ok: false, + reason: "unsafe-string", + }); + const unsupported = new Error("failed") as Error & { debugPayload: { requestId: string } }; + unsupported.debugPayload = { requestId: "request-1" }; + expect(captureThrowableSnapshot(unsupported)).toMatchObject({ + ok: false, + reason: "unsupported-field", + }); + }); + + it("replays typed review and complete throws in full order across multiple variants", async () => { + const corpus = newCorpus(); + const artifactDir = join(corpus, "typed-throws"); + const out = join(artifactDir, "matrix.json"); + let primaryCalls = 0; + let fallbackCalls = 0; + let criticCalls = 0; + const primary: ProviderAdapter = { + ...stub(), + async review() { + primaryCalls++; + throw new SandboxUnavailableError("sandbox unavailable"); + }, + }; + const fallbackBase = stub(); + const fallback: ProviderAdapter = { + ...fallbackBase, + id: "gemini", + async review(input) { + fallbackCalls++; + return fallbackBase.review(input); + }, + }; + const critic: ProviderAdapter = { + id: "openrouter", + async preflight() { + return { available: true, version: "stub-1", authMode: "openrouter", error: null }; + }, + async review() { + throw new Error("critic must use complete"); + }, + async complete() { + criticCalls++; + const error = new Error("critic failed", { + cause: new Error("upstream failed"), + }) as Error & { + code: string; + retryable: boolean; + }; + error.code = "E_CRITIC"; + error.retryable = true; + throw error; + }, + }; + + const result = await runBenchMatrix({ + repoRoot: corpus, + corpus, + out, + ablate: ["confidence-floor", "judgment.hypothetical"], + criticProvider: "openrouter", + maxProviderCalls: 10, + adapters: { codex: primary, gemini: fallback, openrouter: critic }, + providerAvailable: () => true, + now: () => new Date("2026-07-01T00:00:00Z"), + }); + + expect(result.stderr).toBe(""); + expect(result.exitCode).toBe(0); + expect(primaryCalls).toBe(2); + expect(fallbackCalls).toBe(2); + expect(criticCalls).toBe(2); + const matrix = BenchMatrixSchema.parse(JSON.parse(readFileSync(out, "utf8"))); + expect(matrix.variants).toHaveLength(3); + const manifestPath = join(artifactDir, matrix.artifacts?.reviewer_responses.path ?? "missing"); + const manifestBytes = readFileSync(manifestPath, "utf8"); + const manifest = BenchResponseManifestSchema.parse(JSON.parse(manifestBytes)); + expect(manifest.entries.map((entry) => [entry.kind, entry.outcome])).toEqual([ + ["review", "throw"], + ["review", "return"], + ["complete", "throw"], + ["review", "throw"], + ["review", "return"], + ["complete", "throw"], + ]); + expect( + manifest.entries + .filter((entry) => entry.throw_snapshot?.kind === "error") + .map((entry) => + entry.throw_snapshot?.kind === "error" ? entry.throw_snapshot.error_type : null, + ), + ).toEqual(["SandboxUnavailableError", "Error", "SandboxUnavailableError", "Error"]); + expect(manifestBytes).not.toContain("stack"); + expect(manifestBytes).not.toContain("sourceURL"); + expect(manifestBytes).not.toContain("/Users/"); + }); + it("rejects every non-authoritative trace-pair boundary with a precise closed reason", () => { const baseline = traceRun([]); const counterfactual = traceRun(["judgment.confidence"]); @@ -494,18 +889,27 @@ describe("runBenchMatrix", () => { expect(res.exitCode).toBe(0); expect(reviewCalls).toBe(2); // baseline only; the variant is deterministic replay expect(criticCalls).toBe(2); - expect(existsSync(join(artifactDir, "baseline.result.json"))).toBe(true); - expect(existsSync(join(artifactDir, "no-judgment.critic.result.json"))).toBe(true); - const manifest = JSON.parse( - readFileSync(join(artifactDir, "reviewer-responses.sha256.json"), "utf8"), - ) as { entries: Array<{ request_sha256: string; response_sha256: string }> }; + const matrix = BenchMatrixSchema.parse(JSON.parse(readFileSync(out, "utf8"))); + expect(existsSync(join(artifactDir, matrix.artifacts?.baseline.path ?? "missing"))).toBe(true); + expect(existsSync(join(artifactDir, matrix.artifacts?.variants[0]?.path ?? "missing"))).toBe( + true, + ); + const manifest = BenchResponseManifestSchema.parse( + JSON.parse( + readFileSync( + join(artifactDir, matrix.artifacts?.reviewer_responses.path ?? "missing"), + "utf8", + ), + ), + ); expect(manifest.entries).toHaveLength(4); expect(manifest.entries.every((e) => e.request_sha256.length === 64)).toBe(true); expect(manifest.entries.every((e) => e.response_sha256.length === 64)).toBe(true); - const matrix = BenchMatrixSchema.parse(JSON.parse(readFileSync(out, "utf8"))); - expect(matrix.artifacts?.baseline.path).toBe("baseline.result.json"); - expect(matrix.artifacts?.variants[0]?.path).toBe("no-judgment.critic.result.json"); - expect(matrix.artifacts?.reviewer_responses.path).toBe("reviewer-responses.sha256.json"); + expect(matrix.artifacts?.baseline.path).toMatch(/^artifacts\/results\/[0-9a-f]{64}\.json$/); + expect(matrix.artifacts?.variants[0]?.path).toMatch(/^artifacts\/results\/[0-9a-f]{64}\.json$/); + expect(matrix.artifacts?.reviewer_responses.path).toMatch( + /^artifacts\/responses\/[0-9a-f]{64}\.json$/, + ); }); it("replays reviewer retry attempts in the same order as the captured baseline", async () => { @@ -549,19 +953,29 @@ describe("runBenchMatrix", () => { expect(res.exitCode).toBe(0); expect(reviewCalls).toBe(3); + const matrix = BenchMatrixSchema.parse(JSON.parse(readFileSync(out, "utf8"))); const baseline = BenchResultSchema.parse( - JSON.parse(readFileSync(join(artifactDir, "baseline.result.json"), "utf8")), + JSON.parse( + readFileSync(join(artifactDir, matrix.artifacts?.baseline.path ?? "missing"), "utf8"), + ), ); const variant = BenchResultSchema.parse( - JSON.parse(readFileSync(join(artifactDir, "no-judgment.confidence.result.json"), "utf8")), + JSON.parse( + readFileSync(join(artifactDir, matrix.artifacts?.variants[0]?.path ?? "missing"), "utf8"), + ), ); expect(baseline.providers[0]?.coverage.value).toBe(1); expect(variant.providers[0]?.coverage.value).toBe(1); expect(baseline.provenance.integrity?.provider_calls_used).toBe(3); expect(baseline.provenance.integrity?.reviewer_max_attempts).toBe(2); - const manifest = JSON.parse( - readFileSync(join(artifactDir, "reviewer-responses.sha256.json"), "utf8"), - ) as { entries: unknown[] }; + const manifest = BenchResponseManifestSchema.parse( + JSON.parse( + readFileSync( + join(artifactDir, matrix.artifacts?.reviewer_responses.path ?? "missing"), + "utf8", + ), + ), + ); expect(manifest.entries).toHaveLength(3); }); @@ -613,8 +1027,8 @@ describe("runBenchMatrix", () => { expect(res.exitCode).toBe(4); expect(res.stderr).toContain("variant corpus commit differs from baseline"); expect(res.stderr).toContain("variant source commit differs from baseline"); - expect(existsSync(join(artifactDir, "baseline.result.json"))).toBe(true); - expect(existsSync(join(artifactDir, "no-judgment.critic.result.json"))).toBe(true); + expect(existsSync(join(artifactDir, "baseline.result.json"))).toBe(false); + expect(existsSync(join(artifactDir, "no-judgment.critic.result.json"))).toBe(false); expect(existsSync(join(artifactDir, "matrix.json"))).toBe(false); }); @@ -683,11 +1097,16 @@ describe("runBenchMatrix", () => { it("never makes live critic completions in a replay variant", async () => { const corpus = newCorpus(); + let preflightCalls = 0; let reviewCalls = 0; let criticCalls = 0; const reviewer = stub(); const countedReviewer: ProviderAdapter = { ...reviewer, + async preflight(config) { + preflightCalls++; + return reviewer.preflight(config); + }, async review(input) { reviewCalls++; return reviewer.review(input); @@ -729,6 +1148,7 @@ describe("runBenchMatrix", () => { }); expect(res.exitCode).toBe(0); + expect(preflightCalls).toBe(1); expect(reviewCalls).toBe(2); expect(criticCalls).toBe(2); }); @@ -783,9 +1203,17 @@ describe("runBenchMatrix", () => { expect(res.exitCode).toBe(0); expect(primaryCalls).toBe(2); expect(fallbackCalls).toBe(2); - const manifest = JSON.parse( - readFileSync(join(artifactDir, "reviewer-responses.sha256.json"), "utf8"), - ) as { entries: Array<{ provider: string }> }; + const matrix = BenchMatrixSchema.parse( + JSON.parse(readFileSync(join(artifactDir, "matrix.json"), "utf8")), + ); + const manifest = BenchResponseManifestSchema.parse( + JSON.parse( + readFileSync( + join(artifactDir, matrix.artifacts?.reviewer_responses.path ?? "missing"), + "utf8", + ), + ), + ); expect(manifest.entries.filter((entry) => entry.provider === "codex")).toHaveLength(2); expect(manifest.entries.filter((entry) => entry.provider === "gemini")).toHaveLength(2); }); diff --git a/tests/unit/bench-result-schema.test.ts b/tests/unit/bench-result-schema.test.ts index 1cc83ca..9eb04b9 100644 --- a/tests/unit/bench-result-schema.test.ts +++ b/tests/unit/bench-result-schema.test.ts @@ -2,7 +2,12 @@ import { describe, expect, it } from "bun:test"; import { createHash } from "node:crypto"; import { canonicalJson } from "../../src/audit/canonical.ts"; import { POLICY_CATALOG_VERSION, POLICY_PASS_IDS } from "../../src/core/policy/catalog.ts"; -import { BenchMatrixSchema, BenchResultSchema } from "../../src/schemas/bench-result.ts"; +import { + BenchMatrixSchema, + BenchPolicyTraceSetSchema, + BenchResponseManifestSchema, + BenchResultSchema, +} from "../../src/schemas/bench-result.ts"; function emptyPolicyTrace(ablated: string[] = []) { return { @@ -53,7 +58,7 @@ function completePolicyRecord() { catalog_version: POLICY_CATALOG_VERSION, requested_ablations: [], trace, - trace_ref: `inline-policy-trace/${traceSha256}.json`, + trace_ref: `artifacts/policy-traces/2026/07/01/policy/${createHash("sha256").update(trace.run_id).digest("hex").slice(0, 12)}-i${trace.iter}-${traceSha256.slice(0, 12)}.json`, trace_sha256: traceSha256, request_identity_sha256: "c".repeat(64), effective_config_sha256: "d".repeat(64), @@ -62,6 +67,47 @@ function completePolicyRecord() { }; } +function validPolicyTraceSet() { + const trace = (caseId: string, traceSha256: string) => ({ + case_id: caseId, + repeat: 1, + trace_ref: `artifacts/policy-traces/2026/07/01/policy/${"9".repeat(12)}-i1-${traceSha256.slice(0, 12)}.json`, + trace_sha256: traceSha256, + effective_config_sha256: "d".repeat(64), + request_identity_sha256: "c".repeat(64), + final_identity_sha256: "e".repeat(64), + raw_response_sha256: ["a".repeat(64)], + }); + return { + schema: "reviewgate.bench.policy-trace-set.v1", + catalog_version: POLICY_CATALOG_VERSION, + response_manifest: { + path: `artifacts/responses/${"b".repeat(64)}.json`, + sha256: "b".repeat(64), + }, + runs: [ + { + label: "baseline", + ablated_pass_id: null, + result: { + path: `artifacts/results/${"1".repeat(64)}.json`, + sha256: "1".repeat(64), + }, + traces: [trace("case-1", "2".repeat(64))], + }, + { + label: "-judgment.confidence", + ablated_pass_id: "judgment.confidence", + result: { + path: `artifacts/results/${"3".repeat(64)}.json`, + sha256: "3".repeat(64), + }, + traces: [trace("case-1", "4".repeat(64))], + }, + ], + }; +} + const validResult = { schema: "reviewgate.bench.result.v1", provenance: { @@ -184,6 +230,77 @@ describe("BenchResultSchema", () => { ).toBe(false); }); + it("binds every trace-set artifact path to its declared immutable hash", () => { + const traceSet = validPolicyTraceSet(); + expect(BenchPolicyTraceSetSchema.safeParse(traceSet).success).toBe(true); + + expect( + BenchPolicyTraceSetSchema.safeParse({ + ...traceSet, + response_manifest: { ...traceSet.response_manifest, sha256: "5".repeat(64) }, + }).success, + ).toBe(false); + expect( + BenchPolicyTraceSetSchema.safeParse({ + ...traceSet, + runs: traceSet.runs.map((run, index) => + index === 0 ? { ...run, result: { ...run.result, sha256: "6".repeat(64) } } : run, + ), + }).success, + ).toBe(false); + expect( + BenchPolicyTraceSetSchema.safeParse({ + ...traceSet, + runs: traceSet.runs.map((run, index) => + index === 1 + ? { + ...run, + traces: run.traces.map((row) => ({ + ...row, + trace_sha256: "7".repeat(64), + })), + } + : run, + ), + }).success, + ).toBe(false); + }); + + it("rejects credential fields and host paths in persisted throw snapshots", () => { + const parseSnapshot = (snapshot: unknown) => + BenchResponseManifestSchema.safeParse({ + schema: "reviewgate.bench.provider-response-hashes.v2", + entries: [ + { + provider: "codex", + kind: "review", + ordinal: 0, + request_sha256: "a".repeat(64), + response_sha256: createHash("sha256").update(canonicalJson(snapshot)).digest("hex"), + outcome: "throw", + throw_snapshot: snapshot, + }, + ], + }); + + expect( + parseSnapshot({ + kind: "error", + error_type: "Error", + name: "Error", + message: "failed", + fields: [{ key: "apiToken", value: "secret", enumerable: true }], + }).success, + ).toBe(false); + expect( + parseSnapshot({ + kind: "primitive", + primitive_type: "string", + value: "failed at /Users/alice/private/file", + }).success, + ).toBe(false); + }); + it("strictly validates normalized matrix policy provenance", () => { const metric = validResult.aggregate.precision; const matrix = { @@ -202,7 +319,7 @@ describe("BenchResultSchema", () => { catalog_version: POLICY_CATALOG_VERSION, ablated_pass_id: "judgment.confidence", trace_status: "complete", - trace_ref: `inline-policy-trace-set/${"f".repeat(64)}.json`, + trace_ref: `artifacts/policy-trace-sets/${"f".repeat(64)}.json`, trace_sha256: "f".repeat(64), raw_response_sha256: ["a".repeat(64)], authoritative: true, @@ -255,6 +372,23 @@ describe("BenchResultSchema", () => { ], }).success, ).toBe(false); + + expect( + BenchMatrixSchema.safeParse({ + ...matrix, + artifacts: { + baseline: { path: `artifacts/results/${"a".repeat(64)}.json`, sha256: "a".repeat(64) }, + variants: [], + reviewer_responses: { + path: `artifacts/responses/${"b".repeat(64)}.json`, + sha256: "b".repeat(64), + }, + policy_trace_set: { + path: `artifacts/policy-trace-sets/${"c".repeat(64)}.json`, + }, + }, + }).success, + ).toBe(false); }); it("accepts Alpha.12 integrity, critic coverage and honest unknown costs additively", () => { From 12c40685f44f63ef9b7e6bf7086b27c331ce3b23 Mon Sep 17 00:00:00 2001 From: Codevena Date: Tue, 11 Aug 2026 13:52:11 +0200 Subject: [PATCH 40/55] fix(bench): harden authoritative replay artifacts --- src/audit/policy-trace-store.ts | 8 +- src/cli/commands/bench.ts | 90 +++++++++++++------- src/schemas/bench-result.ts | 26 ++++-- tests/unit/bench-matrix.test.ts | 109 +++++++++++++++++++++++++ tests/unit/bench-result-schema.test.ts | 56 +++++++++++++ tests/unit/policy-trace-store.test.ts | 21 ++++- 6 files changed, 269 insertions(+), 41 deletions(-) diff --git a/src/audit/policy-trace-store.ts b/src/audit/policy-trace-store.ts index 2a18fe3..f449784 100644 --- a/src/audit/policy-trace-store.ts +++ b/src/audit/policy-trace-store.ts @@ -122,7 +122,7 @@ function readBoundedRegularArtifact( if (pathBefore.isSymbolicLink() || !pathBefore.isFile() || pathBefore.nlink !== 1) { return { ok: false, reason: "not-a-file" }; } - if (requiredMode !== undefined && (pathBefore.mode & 0o777) !== requiredMode) { + if (requiredMode !== undefined && (pathBefore.mode & 0o7777) !== requiredMode) { return { ok: false, reason: "not-a-file" }; } if (pathBefore.size > POLICY_TRACE_MAX_BYTES) return { ok: false, reason: "too-large" }; @@ -135,7 +135,7 @@ function readBoundedRegularArtifact( if (!openedBefore.isFile() || openedBefore.nlink !== 1) { return { ok: false, reason: "not-a-file" }; } - if (requiredMode !== undefined && (openedBefore.mode & 0o777) !== requiredMode) { + if (requiredMode !== undefined && (openedBefore.mode & 0o7777) !== requiredMode) { return { ok: false, reason: "not-a-file" }; } if (openedBefore.size > POLICY_TRACE_MAX_BYTES) { @@ -162,7 +162,7 @@ function readBoundedRegularArtifact( pathAfter.isSymbolicLink() || !pathAfter.isFile() || pathAfter.nlink !== 1 || - (requiredMode !== undefined && (pathAfter.mode & 0o777) !== requiredMode) || + (requiredMode !== undefined && (pathAfter.mode & 0o7777) !== requiredMode) || pathAfter.dev !== openedAfter.dev || pathAfter.ino !== openedAfter.ino || realpathSync(path) !== realPath @@ -264,7 +264,7 @@ export function verifyPolicyTraceReference( return { ok: false, reason: "path-escape" }; } } - const read = readBoundedRegularArtifact(candidate, realRoot); + const read = readBoundedRegularArtifact(candidate, realRoot, 0o600); if (!read.ok) return read; const { bytes } = read; const contentSha256 = sha256(bytes); diff --git a/src/cli/commands/bench.ts b/src/cli/commands/bench.ts index 2c4fa1e..c30bbae 100644 --- a/src/cli/commands/bench.ts +++ b/src/cli/commands/bench.ts @@ -77,6 +77,7 @@ import { type MatrixVariant, type Metric, type ProviderResult, + isAuthoritativeThrowableString, } from "../../schemas/bench-result.ts"; import { writeFileIfAbsent } from "../../utils/atomic-write.ts"; import { spawnCapture } from "../../utils/spawn-capture.ts"; @@ -1260,26 +1261,18 @@ export type ThrowableCaptureResult = const SENSITIVE_THROW_FIELD = /(?:authorization|cookie|credential|password|passwd|secret|token|api[_-]?key|private[_-]?key)/i; -const UNSAFE_THROW_STRING = - /(?:\/Users\/|\/home\/|\/private\/|\/tmp\/|[A-Za-z]:[\\/]|\bBearer\s+|\bsk-[A-Za-z0-9_-]{8,})/i; const CAPTURED_THROWABLE_FIELD_KEY_SET = new Set(CAPTURED_THROWABLE_FIELD_KEYS); - -function hasUnsafeControlCharacter(value: string): boolean { - return Array.from(value).some((character) => { - const codePoint = character.codePointAt(0) ?? 0; - return ( - codePoint <= 8 || - codePoint === 11 || - codePoint === 12 || - (codePoint >= 14 && codePoint <= 31) || - codePoint === 127 - ); - }); -} - -function safeThrowableString(value: string): boolean { - return !UNSAFE_THROW_STRING.test(value) && !hasUnsafeControlCharacter(value); -} +const OMITTED_STANDARD_THROWABLE_FIELD_KEYS = new Set([ + "cause", + "column", + "line", + "message", + "name", + "originalColumn", + "originalLine", + "sourceURL", + "stack", +]); type SafeValueCapture = | { ok: true; value: import("../../schemas/bench-result.ts").ThrowableSafeValue } @@ -1293,7 +1286,7 @@ function captureSafeThrowableValue( if (depth > 12) return { ok: false, reason: "unsupported-field" }; if (value === null || typeof value === "boolean") return { ok: true, value }; if (typeof value === "string") { - return safeThrowableString(value) + return isAuthoritativeThrowableString(value) ? { ok: true, value } : { ok: false, reason: "unsafe-string" }; } @@ -1303,9 +1296,26 @@ function captureSafeThrowableValue( seen.add(value); try { if (Array.isArray(value)) { + const ownKeys = Reflect.ownKeys(value); + if ( + ownKeys.some((key) => { + if (typeof key === "symbol") return true; + if (key === "length") return false; + const index = Number(key); + return ( + !Number.isInteger(index) || index < 0 || index >= value.length || String(index) !== key + ); + }) + ) { + return { ok: false, reason: "unsupported-field" }; + } const captured: import("../../schemas/bench-result.ts").ThrowableSafeValue[] = []; - for (const item of value) { - const next = captureSafeThrowableValue(item, seen, depth + 1); + for (let index = 0; index < value.length; index += 1) { + const descriptor = Object.getOwnPropertyDescriptor(value, String(index)); + if (descriptor === undefined || !("value" in descriptor)) { + return { ok: false, reason: "unsupported-field" }; + } + const next = captureSafeThrowableValue(descriptor.value, seen, depth + 1); if (!next.ok) return next; captured.push(next.value); } @@ -1316,7 +1326,11 @@ function captureSafeThrowableValue( return { ok: false, reason: "unsupported-field" }; } const captured: Record = {}; - for (const key of Object.keys(value).sort()) { + const ownKeys = Reflect.ownKeys(value); + if (ownKeys.some((key) => typeof key === "symbol")) { + return { ok: false, reason: "unsupported-field" }; + } + for (const key of (ownKeys as string[]).sort()) { if (SENSITIVE_THROW_FIELD.test(key)) return { ok: false, reason: "sensitive-field" }; const descriptor = Object.getOwnPropertyDescriptor(value, key); if (descriptor === undefined || !("value" in descriptor)) { @@ -1338,7 +1352,9 @@ function captureThrowableSnapshotInner( depth: number, ): ThrowableCaptureResult { if (typeof thrown === "string") { - if (!safeThrowableString(thrown)) return { ok: false, reason: "unsafe-string" }; + if (!isAuthoritativeThrowableString(thrown)) { + return { ok: false, reason: "unsafe-string" }; + } const snapshot = { kind: "primitive" as const, primitive_type: "string" as const, @@ -1362,7 +1378,10 @@ function captureThrowableSnapshotInner( ? "SandboxUnavailableError" : null; if (errorType === null) return { ok: false, reason: "unsupported-error-type" }; - if (!safeThrowableString(thrown.name) || !safeThrowableString(thrown.message)) { + if ( + !isAuthoritativeThrowableString(thrown.name) || + !isAuthoritativeThrowableString(thrown.message) + ) { return { ok: false, reason: "unsafe-string" }; } seen.add(thrown); @@ -1382,16 +1401,19 @@ function captureThrowableSnapshotInner( value: import("../../schemas/bench-result.ts").ThrowableSafeValue; enumerable: boolean; }> = []; - for (const key of Object.getOwnPropertyNames(thrown).sort()) { - if (["cause", "message", "name", "stack"].includes(key)) continue; + const ownKeys = Reflect.ownKeys(thrown); + if (ownKeys.some((key) => typeof key === "symbol")) { + return { ok: false, reason: "unsupported-field" }; + } + for (const key of (ownKeys as string[]).sort()) { + if (OMITTED_STANDARD_THROWABLE_FIELD_KEYS.has(key)) continue; if (SENSITIVE_THROW_FIELD.test(key)) return { ok: false, reason: "sensitive-field" }; const descriptor = Object.getOwnPropertyDescriptor(thrown, key); if (descriptor === undefined || !("value" in descriptor)) { return { ok: false, reason: "unsupported-field" }; } if (!CAPTURED_THROWABLE_FIELD_KEY_SET.has(key)) { - if (descriptor.enumerable) return { ok: false, reason: "unsupported-field" }; - continue; + return { ok: false, reason: "unsupported-field" }; } const captured = captureSafeThrowableValue(descriptor.value, seen, depth + 1); if (!captured.ok) return captured; @@ -1563,7 +1585,12 @@ export function verifyBenchArtifactReference(input: { } } const before = lstatSync(candidate); - if (before.isSymbolicLink() || !before.isFile() || before.nlink !== 1) { + if ( + before.isSymbolicLink() || + !before.isFile() || + before.nlink !== 1 || + (before.mode & 0o7777) !== 0o600 + ) { return { ok: false, reason: "not-a-file" }; } if (before.size > BENCH_ARTIFACT_MAX_BYTES) return { ok: false, reason: "too-large" }; @@ -1575,6 +1602,7 @@ export function verifyBenchArtifactReference(input: { if ( !opened.isFile() || opened.nlink !== 1 || + (opened.mode & 0o7777) !== 0o600 || opened.dev !== before.dev || opened.ino !== before.ino ) { @@ -1594,6 +1622,8 @@ export function verifyBenchArtifactReference(input: { pathAfter.isSymbolicLink() || !pathAfter.isFile() || pathAfter.nlink !== 1 || + (after.mode & 0o7777) !== 0o600 || + (pathAfter.mode & 0o7777) !== 0o600 || pathAfter.dev !== after.dev || pathAfter.ino !== after.ino ) { diff --git a/src/schemas/bench-result.ts b/src/schemas/bench-result.ts index 2194fa1..00ffde6 100644 --- a/src/schemas/bench-result.ts +++ b/src/schemas/bench-result.ts @@ -2,6 +2,7 @@ import { createHash } from "node:crypto"; import { z } from "zod"; import { canonicalJson } from "../audit/canonical.ts"; import { POLICY_CATALOG_VERSION, POLICY_PASS_IDS } from "../core/policy/catalog.ts"; +import { redactHighEntropy } from "../diff/sanitizer.ts"; import { PolicyTraceSchema } from "./policy-trace.ts"; // reviewgate bench — result schema (spec §5, §7.2). What `bench run` writes and @@ -221,7 +222,16 @@ const SensitiveThrowableFieldSchema = z ), "throwable field contains sensitive key", ); -function isSafeThrowableString(value: string): boolean { +const HOST_ABSOLUTE_PATH = + /(?:^|[^A-Za-z0-9])\/(?:Users|home|root|var|private|Volumes|tmp|etc|opt|mnt|proc|sys|dev)(?=\/|\\|$|[\s"'`)\],;:])/i; +const WINDOWS_DRIVE_PATH = /(?:^|[^A-Za-z0-9])[A-Za-z]:[\\/]/; +const WINDOWS_UNC_PATH = /\\\\[^\\\s]+\\[^\\\s]+/; +const FILE_URL = /\bfile:\/\//i; +const CREDENTIAL_VALUE = + /(?:\bBearer\s+\S+|\bgh[pousr]_[A-Za-z0-9]{20,}\b|\bgithub_pat_[A-Za-z0-9_]{20,}\b|\bsk-(?:proj-)?[A-Za-z0-9_-]{16,}\b|\b(?:AKIA|ASIA)[A-Z0-9]{16}\b|\bxox[baprs]-[A-Za-z0-9-]{10,}\b)/i; + +/** Shared at-rest boundary for every string in a captured provider throw. */ +export function isAuthoritativeThrowableString(value: string): boolean { const hasUnsafeControlCharacter = Array.from(value).some((character) => { const codePoint = character.codePointAt(0) ?? 0; return ( @@ -233,18 +243,22 @@ function isSafeThrowableString(value: string): boolean { ); }); return ( - !/(?:\/Users\/|\/home\/|\/private\/|\/tmp\/|[A-Za-z]:[\\/]|\bBearer\s+|\bsk-[A-Za-z0-9_-]{8,})/i.test( - value, - ) && !hasUnsafeControlCharacter + !hasUnsafeControlCharacter && + !HOST_ABSOLUTE_PATH.test(value) && + !WINDOWS_DRIVE_PATH.test(value) && + !WINDOWS_UNC_PATH.test(value) && + !FILE_URL.test(value) && + !CREDENTIAL_VALUE.test(value) && + redactHighEntropy(value).count === 0 ); } const SafeThrowableStringSchema = z .string() - .refine(isSafeThrowableString, "throwable string contains unsafe data"); + .refine(isAuthoritativeThrowableString, "throwable string contains unsafe data"); const SafeThrowableNameSchema = z .string() .min(1) - .refine(isSafeThrowableString, "throwable string contains unsafe data"); + .refine(isAuthoritativeThrowableString, "throwable string contains unsafe data"); export const ThrowableSafeValueSchema: z.ZodType = z.lazy(() => z.union([ diff --git a/tests/unit/bench-matrix.test.ts b/tests/unit/bench-matrix.test.ts index a530cb8..f7d4aa9 100644 --- a/tests/unit/bench-matrix.test.ts +++ b/tests/unit/bench-matrix.test.ts @@ -8,6 +8,7 @@ import { execFileSync } from "node:child_process"; import { createHash } from "node:crypto"; import { existsSync, + lstatSync, mkdirSync, mkdtempSync, readFileSync, @@ -509,6 +510,32 @@ describe("runBenchMatrix", () => { ).toEqual({ ok: false, reason: "not-a-file" }); }); + it("rejects a hash-valid Bench artifact unless its mode remains exactly 0600", () => { + const root = mkdtempSync(join(tmpdir(), "rg-bench-artifact-mode-")); + const manifest = BenchResponseManifestSchema.parse({ + schema: "reviewgate.bench.provider-response-hashes.v2", + entries: [], + }); + const canonical = canonicalJson(manifest); + const canonicalSha = sha256(canonical); + const canonicalRef = `artifacts/responses/${canonicalSha}.json`; + mkdirSync(join(root, "artifacts", "responses"), { recursive: true }); + writeFileSync(join(root, canonicalRef), canonical, { mode: 0o600 }); + + for (const unsafeMode of [0o644, 0o4600]) { + execFileSync("/bin/chmod", [unsafeMode.toString(8), join(root, canonicalRef)]); + expect(lstatSync(join(root, canonicalRef)).mode & 0o7777).toBe(unsafeMode); + expect( + verifyBenchArtifactReference({ + root, + ref: canonicalRef, + sha256: canonicalSha, + kind: "response-manifest", + }), + ).toEqual({ ok: false, reason: "not-a-file" }); + } + }); + it("captures and reconstructs exact immutable throwable snapshots without cross-variant aliasing", () => { const sandbox = captureThrowableSnapshot(new SandboxUnavailableError("sandbox unavailable")); expect(sandbox.ok).toBe(true); @@ -582,6 +609,88 @@ describe("runBenchMatrix", () => { ok: false, reason: "unsupported-field", }); + + const hiddenUnknown = new Error("failed"); + Object.defineProperty(hiddenUnknown, "retryAfterMs", { + value: 250, + enumerable: false, + }); + expect(captureThrowableSnapshot(hiddenUnknown)).toMatchObject({ + ok: false, + reason: "unsupported-field", + }); + + const symbolUnknown = new Error("failed"); + Object.defineProperty(symbolUnknown, Symbol("debug"), { + value: "internal", + enumerable: false, + }); + expect(captureThrowableSnapshot(symbolUnknown)).toMatchObject({ + ok: false, + reason: "unsupported-field", + }); + }); + + it("uses the persisted-schema string boundary for host paths and credentials", () => { + const unsafe = [ + ...[ + "/Users", + "/home", + "/root", + "/var", + "/private", + "/Volumes", + "/tmp", + "/etc", + "/opt", + "/mnt", + "/proc", + "/sys", + "/dev", + ].map((directory) => `open ${directory}/reviewgate/private.json`), + String.raw`open C:\Users\alice\secrets.json`, + String.raw`open \\server\share\credentials.json`, + "open file:///etc/passwd", + "request used ghp_abcdefghijklmnopqrstuvwxyz123456", + "request used sk-proj-AbCdEfGhIjKlMnOpQrStUvWxYz012345", + "request used AKIAIOSFODNN7EXAMPLE", + "request used xoxb-123456789012-123456789012-AbCdEfGhIjKlMnOpQrStUvWx", + "request used AbCd3fGh1jKlMnOpQrSt7vWxYz09+/=AbCd", + ]; + for (const value of unsafe) { + expect(captureThrowableSnapshot(new Error(value))).toMatchObject({ + ok: false, + reason: "unsafe-string", + }); + } + + const nested = new Error("provider failed") as Error & { + context: { attempts: Array<{ detail: string }> }; + }; + nested.context = { attempts: [{ detail: "safe" }, { detail: "open /var/folders/token" }] }; + expect(captureThrowableSnapshot(nested)).toMatchObject({ + ok: false, + reason: "unsafe-string", + }); + + const hiddenArrayField = ["safe"] as string[] & { debugPayload?: string }; + Object.defineProperty(hiddenArrayField, "debugPayload", { + value: "internal", + enumerable: false, + }); + const nestedArray = new Error("provider failed") as Error & { context: unknown }; + nestedArray.context = hiddenArrayField; + expect(captureThrowableSnapshot(nestedArray)).toMatchObject({ + ok: false, + reason: "unsupported-field", + }); + + for (const value of [ + "request failed in src/core/orchestrator.ts", + "see https://example.com/root/replay?case=safe-value", + ]) { + expect(captureThrowableSnapshot(new Error(value))).toMatchObject({ ok: true }); + } }); it("replays typed review and complete throws in full order across multiple variants", async () => { diff --git a/tests/unit/bench-result-schema.test.ts b/tests/unit/bench-result-schema.test.ts index 9eb04b9..d0a4ff4 100644 --- a/tests/unit/bench-result-schema.test.ts +++ b/tests/unit/bench-result-schema.test.ts @@ -299,6 +299,62 @@ describe("BenchResultSchema", () => { value: "failed at /Users/alice/private/file", }).success, ).toBe(false); + + const unsafeStrings = [ + ...[ + "/Users", + "/home", + "/root", + "/var", + "/private", + "/Volumes", + "/tmp", + "/etc", + "/opt", + "/mnt", + "/proc", + "/sys", + "/dev", + ].map((directory) => `open ${directory}/reviewgate/private.json`), + String.raw`open C:\Users\alice\secrets.json`, + String.raw`open \\server\share\credentials.json`, + "open file:///etc/passwd", + "request used ghp_abcdefghijklmnopqrstuvwxyz123456", + "request used sk-proj-AbCdEfGhIjKlMnOpQrStUvWxYz012345", + "request used AKIAIOSFODNN7EXAMPLE", + "request used xoxb-123456789012-123456789012-AbCdEfGhIjKlMnOpQrStUvWx", + "request used AbCd3fGh1jKlMnOpQrSt7vWxYz09+/=AbCd", + ]; + for (const value of unsafeStrings) { + expect(parseSnapshot({ kind: "primitive", primitive_type: "string", value }).success).toBe( + false, + ); + } + + expect( + parseSnapshot({ + kind: "error", + error_type: "Error", + name: "Error", + message: "provider failed", + fields: [ + { + key: "context", + value: { attempts: [{ detail: "safe" }, { detail: "open /var/folders/token" }] }, + enumerable: true, + }, + ], + }).success, + ).toBe(false); + + for (const value of [ + "request failed in src/core/orchestrator.ts", + "see https://example.com/root/replay?case=safe-value", + ]) { + expect(parseSnapshot({ kind: "primitive", primitive_type: "string", value }).success).toBe( + true, + ); + } }); it("strictly validates normalized matrix policy provenance", () => { diff --git a/tests/unit/policy-trace-store.test.ts b/tests/unit/policy-trace-store.test.ts index 9512206..48113a5 100644 --- a/tests/unit/policy-trace-store.test.ts +++ b/tests/unit/policy-trace-store.test.ts @@ -1,4 +1,5 @@ import { describe, expect, it } from "bun:test"; +import { execFileSync } from "node:child_process"; import { createHash } from "node:crypto"; import { chmodSync, @@ -162,7 +163,7 @@ function writeUncheckedArtifact( const ref = `2026/08/10/policy/${sha256(trace.run_id).slice(0, 12)}-i${trace.iter}-${contentSha256.slice(0, 12)}.json`; const path = join(auditDir, ...ref.split("/")); mkdirSync(dirname(path), { recursive: true }); - writeFileSync(path, bytes); + writeFileSync(path, bytes, { mode: 0o600 }); return { ref, sha256: contentSha256, path }; } @@ -528,6 +529,24 @@ describe("canonical policy trace storage", () => { }); describe("policy trace reference security", () => { + it("rejects a hash-valid policy trace unless its mode remains exactly 0600", () => { + const auditDir = join(tmp(), "audit"); + const stored = writePolicyTrace({ auditDir, trace: emptyTrace(), now: NOW }); + if (stored.status !== "complete") throw new Error("fixture trace did not persist"); + const artifact = join(auditDir, ...stored.ref.split("/")); + + for (const unsafeMode of [0o644, 0o4600]) { + execFileSync("/bin/chmod", [unsafeMode.toString(8), artifact]); + expect(lstatSync(artifact).mode & 0o7777).toBe(unsafeMode); + const verified = verifyPolicyTraceReference({ + auditDir, + ref: stored.ref, + sha256: stored.sha256, + }); + expect(verified).toEqual({ ok: false, reason: "not-a-file" }); + } + }); + it("rejects missing, absolute, traversing, wrong-hash, tampered, and symlink-escaping refs", () => { const root = tmp(); const auditDir = join(root, "audit"); From 575ec617f327eb3628b8f0ee70fc8020fac99272 Mon Sep 17 00:00:00 2001 From: Codevena Date: Tue, 11 Aug 2026 14:16:57 +0200 Subject: [PATCH 41/55] fix(bench): classify replay strings precisely --- src/schemas/bench-result.ts | 57 ++++++-- tests/unit/bench-matrix.test.ts | 112 ++++++++++----- tests/unit/bench-result-schema.test.ts | 189 ++++++++++++++++--------- 3 files changed, 239 insertions(+), 119 deletions(-) diff --git a/src/schemas/bench-result.ts b/src/schemas/bench-result.ts index 00ffde6..9dcb635 100644 --- a/src/schemas/bench-result.ts +++ b/src/schemas/bench-result.ts @@ -1,4 +1,5 @@ import { createHash } from "node:crypto"; +import { URL } from "node:url"; import { z } from "zod"; import { canonicalJson } from "../audit/canonical.ts"; import { POLICY_CATALOG_VERSION, POLICY_PASS_IDS } from "../core/policy/catalog.ts"; @@ -222,14 +223,47 @@ const SensitiveThrowableFieldSchema = z ), "throwable field contains sensitive key", ); -const HOST_ABSOLUTE_PATH = - /(?:^|[^A-Za-z0-9])\/(?:Users|home|root|var|private|Volumes|tmp|etc|opt|mnt|proc|sys|dev)(?=\/|\\|$|[\s"'`)\],;:])/i; -const WINDOWS_DRIVE_PATH = /(?:^|[^A-Za-z0-9])[A-Za-z]:[\\/]/; -const WINDOWS_UNC_PATH = /\\\\[^\\\s]+\\[^\\\s]+/; -const FILE_URL = /\bfile:\/\//i; +const ABSOLUTE_POSIX_PATH = /(?:^|[\s"\x27\x60([{=,:;])\/(?!\/)(?=[^\s/])/; +const FORWARD_UNC_PATH = /(?:^|[\s"\x27\x60([{=,:;])\/\/[^/\s\\]+\/[^/\s\\]+/; +const WINDOWS_DRIVE_PATH = /(?:^|[\s"\x27\x60([{=,:;])[A-Za-z]:[\\/]/; +const WINDOWS_UNC_PATH = /(?:^|[\s"\x27\x60([{=,:;])\\\\[^\\\s]+\\[^\\\s]+/; +const URL_LIKE = /\b[A-Za-z][A-Za-z0-9+.-]*:\/\/[^\s<>"'`)\]}]+/gi; +const SECRET_ASSIGNMENT = + /(?:^|[^A-Za-z0-9_])(?:authorization|cookie|credential|password|passwd|secret|token|api[_-]?key|private[_-]?key)\s*[:=]\s*\S+/i; const CREDENTIAL_VALUE = /(?:\bBearer\s+\S+|\bgh[pousr]_[A-Za-z0-9]{20,}\b|\bgithub_pat_[A-Za-z0-9_]{20,}\b|\bsk-(?:proj-)?[A-Za-z0-9_-]{16,}\b|\b(?:AKIA|ASIA)[A-Z0-9]{16}\b|\bxox[baprs]-[A-Za-z0-9-]{10,}\b)/i; +function maskSafeHttpUrls(value: string): string | null { + let invalidUrl = false; + const nonUrlText = value.replace(URL_LIKE, (rawUrl) => { + try { + const parsed = new URL(rawUrl); + if ( + (parsed.protocol !== "http:" && parsed.protocol !== "https:") || + parsed.username !== "" || + parsed.password !== "" + ) { + invalidUrl = true; + return rawUrl; + } + const decodedUrlParts = [parsed.pathname, parsed.search, parsed.hash].map((part) => + decodeURIComponent(part), + ); + if ( + decodedUrlParts.some((part) => SECRET_ASSIGNMENT.test(part) || CREDENTIAL_VALUE.test(part)) + ) { + invalidUrl = true; + return rawUrl; + } + return " ".repeat(rawUrl.length); + } catch { + invalidUrl = true; + return rawUrl; + } + }); + return invalidUrl ? null : nonUrlText; +} + /** Shared at-rest boundary for every string in a captured provider throw. */ export function isAuthoritativeThrowableString(value: string): boolean { const hasUnsafeControlCharacter = Array.from(value).some((character) => { @@ -242,14 +276,17 @@ export function isAuthoritativeThrowableString(value: string): boolean { codePoint === 127 ); }); + const nonUrlText = maskSafeHttpUrls(value); return ( !hasUnsafeControlCharacter && - !HOST_ABSOLUTE_PATH.test(value) && - !WINDOWS_DRIVE_PATH.test(value) && - !WINDOWS_UNC_PATH.test(value) && - !FILE_URL.test(value) && + nonUrlText !== null && + !ABSOLUTE_POSIX_PATH.test(nonUrlText) && + !FORWARD_UNC_PATH.test(nonUrlText) && + !WINDOWS_DRIVE_PATH.test(nonUrlText) && + !WINDOWS_UNC_PATH.test(nonUrlText) && + !SECRET_ASSIGNMENT.test(value) && !CREDENTIAL_VALUE.test(value) && - redactHighEntropy(value).count === 0 + redactHighEntropy(nonUrlText).count === 0 ); } const SafeThrowableStringSchema = z diff --git a/tests/unit/bench-matrix.test.ts b/tests/unit/bench-matrix.test.ts index f7d4aa9..9605e74 100644 --- a/tests/unit/bench-matrix.test.ts +++ b/tests/unit/bench-matrix.test.ts @@ -91,6 +91,50 @@ const cleanJson = { source: "hand-written", }; +const UNSAFE_REPLAY_STRINGS = [ + ...[ + "/Users", + "/home", + "/root", + "/var", + "/private", + "/Volumes", + "/tmp", + "/etc", + "/opt", + "/mnt", + "/proc", + "/sys", + "/dev", + ].map((directory) => `open ${directory}/reviewgate/private.json`), + "open /usr/local/bin/reviewgate", + "open /Applications/ReviewGate.app/Contents/Info.plist", + "open /Library/Application Support/ReviewGate/config.json", + "open /System/Library/CoreServices/SystemVersion.plist", + "open /workspace/reviewgate/private.json", + "open //server/share/credentials.json", + String.raw`open C:\Users\alice\secrets.json`, + String.raw`open \\server\share\credentials.json`, + "open file:///etc/passwd", + "password=hunter2", + "api_key=notverysecret", + "Authorization: Basic YTpi", + "https://user:password@example.com/private", + "Authorization: Bearer short-but-secret", + "request used ghp_abcdefghijklmnopqrstuvwxyz123456", + "request used sk-proj-AbCdEfGhIjKlMnOpQrStUvWxYz012345", + "request used AKIAIOSFODNN7EXAMPLE", + "request used xoxb-123456789012-123456789012-AbCdEfGhIjKlMnOpQrStUvWx", + "request used AbCd3fGh1jKlMnOpQrSt7vWxYz09+/=AbCd", +]; + +const SAFE_REPLAY_STRINGS = [ + "request failed in src/core/orchestrator.ts", + "see https://example.com/root/replay?case=safe-value", + "see https://docs.example.com/guides/reviewgate/policy-traces", + "see https://docs.example.com/runs/550e8400-e29b-41d4-a716-446655440000/long-safe-policy-trace-slug", +]; + function sqlFinding(): Finding { return { id: "codex-1", @@ -631,48 +675,29 @@ describe("runBenchMatrix", () => { }); }); - it("uses the persisted-schema string boundary for host paths and credentials", () => { - const unsafe = [ - ...[ - "/Users", - "/home", - "/root", - "/var", - "/private", - "/Volumes", - "/tmp", - "/etc", - "/opt", - "/mnt", - "/proc", - "/sys", - "/dev", - ].map((directory) => `open ${directory}/reviewgate/private.json`), - String.raw`open C:\Users\alice\secrets.json`, - String.raw`open \\server\share\credentials.json`, - "open file:///etc/passwd", - "request used ghp_abcdefghijklmnopqrstuvwxyz123456", - "request used sk-proj-AbCdEfGhIjKlMnOpQrStUvWxYz012345", - "request used AKIAIOSFODNN7EXAMPLE", - "request used xoxb-123456789012-123456789012-AbCdEfGhIjKlMnOpQrStUvWx", - "request used AbCd3fGh1jKlMnOpQrSt7vWxYz09+/=AbCd", - ]; - for (const value of unsafe) { + it("rejects unsafe replay strings at runtime top level", () => { + for (const value of UNSAFE_REPLAY_STRINGS) { expect(captureThrowableSnapshot(new Error(value))).toMatchObject({ ok: false, reason: "unsafe-string", }); } + }); - const nested = new Error("provider failed") as Error & { - context: { attempts: Array<{ detail: string }> }; - }; - nested.context = { attempts: [{ detail: "safe" }, { detail: "open /var/folders/token" }] }; - expect(captureThrowableSnapshot(nested)).toMatchObject({ - ok: false, - reason: "unsafe-string", - }); + it("rejects unsafe replay strings recursively during runtime capture", () => { + for (const value of UNSAFE_REPLAY_STRINGS) { + const nested = new Error("provider failed") as Error & { + context: { attempts: Array<{ detail: string }> }; + }; + nested.context = { attempts: [{ detail: "safe" }, { detail: value }] }; + expect(captureThrowableSnapshot(nested)).toMatchObject({ + ok: false, + reason: "unsafe-string", + }); + } + }); + it("rejects extension fields on nested throwable arrays", () => { const hiddenArrayField = ["safe"] as string[] & { debugPayload?: string }; Object.defineProperty(hiddenArrayField, "debugPayload", { value: "internal", @@ -684,15 +709,24 @@ describe("runBenchMatrix", () => { ok: false, reason: "unsupported-field", }); + }); - for (const value of [ - "request failed in src/core/orchestrator.ts", - "see https://example.com/root/replay?case=safe-value", - ]) { + it("allows safe replay strings at runtime top level", () => { + for (const value of SAFE_REPLAY_STRINGS) { expect(captureThrowableSnapshot(new Error(value))).toMatchObject({ ok: true }); } }); + it("allows safe replay strings recursively during runtime capture", () => { + for (const value of SAFE_REPLAY_STRINGS) { + const nested = new Error("provider failed") as Error & { + context: { attempts: Array<{ detail: string }> }; + }; + nested.context = { attempts: [{ detail: value }] }; + expect(captureThrowableSnapshot(nested)).toMatchObject({ ok: true }); + } + }); + it("replays typed review and complete throws in full order across multiple variants", async () => { const corpus = newCorpus(); const artifactDir = join(corpus, "typed-throws"); diff --git a/tests/unit/bench-result-schema.test.ts b/tests/unit/bench-result-schema.test.ts index d0a4ff4..4c8ba13 100644 --- a/tests/unit/bench-result-schema.test.ts +++ b/tests/unit/bench-result-schema.test.ts @@ -176,6 +176,67 @@ const validResult = { }, }; +const UNSAFE_REPLAY_STRINGS = [ + ...[ + "/Users", + "/home", + "/root", + "/var", + "/private", + "/Volumes", + "/tmp", + "/etc", + "/opt", + "/mnt", + "/proc", + "/sys", + "/dev", + ].map((directory) => `open ${directory}/reviewgate/private.json`), + "open /usr/local/bin/reviewgate", + "open /Applications/ReviewGate.app/Contents/Info.plist", + "open /Library/Application Support/ReviewGate/config.json", + "open /System/Library/CoreServices/SystemVersion.plist", + "open /workspace/reviewgate/private.json", + "open //server/share/credentials.json", + String.raw`open C:\Users\alice\secrets.json`, + String.raw`open \\server\share\credentials.json`, + "open file:///etc/passwd", + "password=hunter2", + "api_key=notverysecret", + "Authorization: Basic YTpi", + "https://user:password@example.com/private", + "Authorization: Bearer short-but-secret", + "request used ghp_abcdefghijklmnopqrstuvwxyz123456", + "request used sk-proj-AbCdEfGhIjKlMnOpQrStUvWxYz012345", + "request used AKIAIOSFODNN7EXAMPLE", + "request used xoxb-123456789012-123456789012-AbCdEfGhIjKlMnOpQrStUvWx", + "request used AbCd3fGh1jKlMnOpQrSt7vWxYz09+/=AbCd", +]; + +const SAFE_REPLAY_STRINGS = [ + "request failed in src/core/orchestrator.ts", + "see https://example.com/root/replay?case=safe-value", + "see https://docs.example.com/guides/reviewgate/policy-traces", + "see https://docs.example.com/runs/550e8400-e29b-41d4-a716-446655440000/long-safe-policy-trace-slug", +]; + +function parseThrowableSnapshot(snapshot: unknown) { + return BenchResponseManifestSchema.safeParse({ + schema: "reviewgate.bench.provider-response-hashes.v2", + entries: [ + { + provider: "codex", + kind: "review", + ordinal: 0, + request_sha256: "a".repeat(64), + response_sha256: createHash("sha256").update(canonicalJson(snapshot)).digest("hex"), + outcome: "throw", + throw_snapshot: snapshot, + }, + ], + }); +} + describe("BenchResultSchema", () => { it("parses a valid result", () => { const r = BenchResultSchema.safeParse(validResult); @@ -266,25 +327,9 @@ describe("BenchResultSchema", () => { ).toBe(false); }); - it("rejects credential fields and host paths in persisted throw snapshots", () => { - const parseSnapshot = (snapshot: unknown) => - BenchResponseManifestSchema.safeParse({ - schema: "reviewgate.bench.provider-response-hashes.v2", - entries: [ - { - provider: "codex", - kind: "review", - ordinal: 0, - request_sha256: "a".repeat(64), - response_sha256: createHash("sha256").update(canonicalJson(snapshot)).digest("hex"), - outcome: "throw", - throw_snapshot: snapshot, - }, - ], - }); - + it("rejects unsafe replay strings at the persisted-schema top level", () => { expect( - parseSnapshot({ + parseThrowableSnapshot({ kind: "error", error_type: "Error", name: "Error", @@ -293,67 +338,71 @@ describe("BenchResultSchema", () => { }).success, ).toBe(false); expect( - parseSnapshot({ + parseThrowableSnapshot({ kind: "primitive", primitive_type: "string", value: "failed at /Users/alice/private/file", }).success, ).toBe(false); - const unsafeStrings = [ - ...[ - "/Users", - "/home", - "/root", - "/var", - "/private", - "/Volumes", - "/tmp", - "/etc", - "/opt", - "/mnt", - "/proc", - "/sys", - "/dev", - ].map((directory) => `open ${directory}/reviewgate/private.json`), - String.raw`open C:\Users\alice\secrets.json`, - String.raw`open \\server\share\credentials.json`, - "open file:///etc/passwd", - "request used ghp_abcdefghijklmnopqrstuvwxyz123456", - "request used sk-proj-AbCdEfGhIjKlMnOpQrStUvWxYz012345", - "request used AKIAIOSFODNN7EXAMPLE", - "request used xoxb-123456789012-123456789012-AbCdEfGhIjKlMnOpQrStUvWx", - "request used AbCd3fGh1jKlMnOpQrSt7vWxYz09+/=AbCd", - ]; - for (const value of unsafeStrings) { - expect(parseSnapshot({ kind: "primitive", primitive_type: "string", value }).success).toBe( - false, - ); + for (const value of UNSAFE_REPLAY_STRINGS) { + expect( + parseThrowableSnapshot({ + kind: "error", + error_type: "Error", + name: "Error", + message: value, + fields: [], + }).success, + ).toBe(false); } + }); - expect( - parseSnapshot({ - kind: "error", - error_type: "Error", - name: "Error", - message: "provider failed", - fields: [ - { - key: "context", - value: { attempts: [{ detail: "safe" }, { detail: "open /var/folders/token" }] }, - enumerable: true, - }, - ], - }).success, - ).toBe(false); + it("rejects unsafe replay strings recursively in the persisted schema", () => { + for (const value of UNSAFE_REPLAY_STRINGS) { + expect( + parseThrowableSnapshot({ + kind: "error", + error_type: "Error", + name: "Error", + message: "provider failed", + fields: [ + { + key: "context", + value: { attempts: [{ detail: "safe" }, { detail: value }] }, + enumerable: true, + }, + ], + }).success, + ).toBe(false); + } + }); + + it("allows safe replay strings at the persisted-schema top level", () => { + for (const value of SAFE_REPLAY_STRINGS) { + expect( + parseThrowableSnapshot({ + kind: "error", + error_type: "Error", + name: "Error", + message: value, + fields: [], + }).success, + ).toBe(true); + } + }); - for (const value of [ - "request failed in src/core/orchestrator.ts", - "see https://example.com/root/replay?case=safe-value", - ]) { - expect(parseSnapshot({ kind: "primitive", primitive_type: "string", value }).success).toBe( - true, - ); + it("allows safe replay strings recursively in the persisted schema", () => { + for (const value of SAFE_REPLAY_STRINGS) { + expect( + parseThrowableSnapshot({ + kind: "error", + error_type: "Error", + name: "Error", + message: "provider failed", + fields: [{ key: "context", value: { detail: value }, enumerable: true }], + }).success, + ).toBe(true); } }); From d8606fbea4c889e29d01ef1d08893074a25ed151 Mon Sep 17 00:00:00 2001 From: Codevena Date: Tue, 11 Aug 2026 14:31:52 +0200 Subject: [PATCH 42/55] fix(bench): close encoded replay secrets --- src/schemas/bench-result.ts | 80 ++++++++++++++++++++------ tests/unit/bench-matrix.test.ts | 19 ++++++ tests/unit/bench-result-schema.test.ts | 19 ++++++ 3 files changed, 99 insertions(+), 19 deletions(-) diff --git a/src/schemas/bench-result.ts b/src/schemas/bench-result.ts index 9dcb635..441d2e1 100644 --- a/src/schemas/bench-result.ts +++ b/src/schemas/bench-result.ts @@ -213,29 +213,66 @@ export type ThrowableSafeValue = | ThrowableSafeValue[] | { [key: string]: ThrowableSafeValue }; +const MAX_PERCENT_DECODE_PASSES = 3; +const SENSITIVE_KEY_PARTS = [ + "authorization", + "cookie", + "credential", + "password", + "passwd", + "secret", + "token", + "apikey", + "privatekey", + "signature", +] as const; + +function isSensitiveThrowableKey(value: string): boolean { + const normalized = value + .normalize("NFKC") + .toLowerCase() + .replace(/[^a-z0-9]/g, ""); + return SENSITIVE_KEY_PARTS.some((part) => normalized.includes(part)); +} + const SensitiveThrowableFieldSchema = z .string() .min(1) - .refine( - (value) => - !/(?:authorization|cookie|credential|password|passwd|secret|token|api[_-]?key|private[_-]?key)/i.test( - value, - ), - "throwable field contains sensitive key", - ); + .refine((value) => !isSensitiveThrowableKey(value), "throwable field contains sensitive key"); const ABSOLUTE_POSIX_PATH = /(?:^|[\s"\x27\x60([{=,:;])\/(?!\/)(?=[^\s/])/; const FORWARD_UNC_PATH = /(?:^|[\s"\x27\x60([{=,:;])\/\/[^/\s\\]+\/[^/\s\\]+/; const WINDOWS_DRIVE_PATH = /(?:^|[\s"\x27\x60([{=,:;])[A-Za-z]:[\\/]/; const WINDOWS_UNC_PATH = /(?:^|[\s"\x27\x60([{=,:;])\\\\[^\\\s]+\\[^\\\s]+/; -const URL_LIKE = /\b[A-Za-z][A-Za-z0-9+.-]*:\/\/[^\s<>"'`)\]}]+/gi; -const SECRET_ASSIGNMENT = - /(?:^|[^A-Za-z0-9_])(?:authorization|cookie|credential|password|passwd|secret|token|api[_-]?key|private[_-]?key)\s*[:=]\s*\S+/i; +const URL_SEGMENT = /\b[A-Za-z][A-Za-z0-9+.-]*:\/\/[^\s<>"'`)}]+/gi; +const KEY_VALUE_PAIR = + /(?:"([^"\r\n]{1,128})"|'([^'\r\n]{1,128})'|([A-Za-z][A-Za-z0-9_.-]{0,127}))\s*[:=]\s*(?:"[^"\r\n]*"|'[^'\r\n]*'|[^\s,}\]]+)/g; const CREDENTIAL_VALUE = /(?:\bBearer\s+\S+|\bgh[pousr]_[A-Za-z0-9]{20,}\b|\bgithub_pat_[A-Za-z0-9_]{20,}\b|\bsk-(?:proj-)?[A-Za-z0-9_-]{16,}\b|\b(?:AKIA|ASIA)[A-Z0-9]{16}\b|\bxox[baprs]-[A-Za-z0-9-]{10,}\b)/i; +function decodePercentToFixedPoint(value: string): string | null { + let decoded = value; + for (let pass = 0; pass < MAX_PERCENT_DECODE_PASSES; pass += 1) { + if (!decoded.includes("%")) return decoded; + try { + decoded = decodeURIComponent(decoded); + } catch { + return null; + } + } + return decoded.includes("%") ? null : decoded; +} + +function hasSensitiveKeyValuePair(value: string): boolean { + for (const match of value.matchAll(KEY_VALUE_PAIR)) { + const key = match[1] ?? match[2] ?? match[3]; + if (key !== undefined && isSensitiveThrowableKey(key)) return true; + } + return false; +} + function maskSafeHttpUrls(value: string): string | null { let invalidUrl = false; - const nonUrlText = value.replace(URL_LIKE, (rawUrl) => { + const nonUrlText = value.replace(URL_SEGMENT, (rawUrl) => { try { const parsed = new URL(rawUrl); if ( @@ -246,11 +283,14 @@ function maskSafeHttpUrls(value: string): string | null { invalidUrl = true; return rawUrl; } - const decodedUrlParts = [parsed.pathname, parsed.search, parsed.hash].map((part) => - decodeURIComponent(part), - ); if ( - decodedUrlParts.some((part) => SECRET_ASSIGNMENT.test(part) || CREDENTIAL_VALUE.test(part)) + hasSensitiveKeyValuePair(`${parsed.pathname} ${parsed.hash}`) || + Array.from(parsed.searchParams).some( + ([key, partValue]) => + isSensitiveThrowableKey(key) || + hasSensitiveKeyValuePair(partValue) || + CREDENTIAL_VALUE.test(partValue), + ) ) { invalidUrl = true; return rawUrl; @@ -266,7 +306,9 @@ function maskSafeHttpUrls(value: string): string | null { /** Shared at-rest boundary for every string in a captured provider throw. */ export function isAuthoritativeThrowableString(value: string): boolean { - const hasUnsafeControlCharacter = Array.from(value).some((character) => { + const decodedValue = decodePercentToFixedPoint(value); + if (decodedValue === null) return false; + const hasUnsafeControlCharacter = Array.from(decodedValue).some((character) => { const codePoint = character.codePointAt(0) ?? 0; return ( codePoint <= 8 || @@ -276,7 +318,7 @@ export function isAuthoritativeThrowableString(value: string): boolean { codePoint === 127 ); }); - const nonUrlText = maskSafeHttpUrls(value); + const nonUrlText = maskSafeHttpUrls(decodedValue); return ( !hasUnsafeControlCharacter && nonUrlText !== null && @@ -284,8 +326,8 @@ export function isAuthoritativeThrowableString(value: string): boolean { !FORWARD_UNC_PATH.test(nonUrlText) && !WINDOWS_DRIVE_PATH.test(nonUrlText) && !WINDOWS_UNC_PATH.test(nonUrlText) && - !SECRET_ASSIGNMENT.test(value) && - !CREDENTIAL_VALUE.test(value) && + !hasSensitiveKeyValuePair(nonUrlText) && + !CREDENTIAL_VALUE.test(decodedValue) && redactHighEntropy(nonUrlText).count === 0 ); } diff --git a/tests/unit/bench-matrix.test.ts b/tests/unit/bench-matrix.test.ts index 9605e74..86bfd6a 100644 --- a/tests/unit/bench-matrix.test.ts +++ b/tests/unit/bench-matrix.test.ts @@ -121,6 +121,22 @@ const UNSAFE_REPLAY_STRINGS = [ "Authorization: Basic YTpi", "https://user:password@example.com/private", "Authorization: Bearer short-but-secret", + "open %2Fusr%2Flocal%2Fbin%2Freviewgate", + "open file%3A%2F%2F%2Fetc%2Fpasswd", + "open file%253A%252F%252F%252Fetc%252Fpasswd", + "query %2570assword%253Dhunter2", + 'payload {"password":"hunter2"}', + 'payload {"authorization":"Basic YTpi"}', + "client_secret=notverysecret", + "ACCESS-TOKEN=notverysecret", + "https://example.com/download?X-Amz-Signature=aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "https://example.com/download?token=notverysecret", + "https://example.com/download?signature=notverysecret", + "https://example.com/download?password=hunter2", + "https://example.com/download?api-key=notverysecret", + "invalid percent %ZZ", + "invalid percent %2", + "open %2525252Fusr%2525252Flocal%2525252Fbin", "request used ghp_abcdefghijklmnopqrstuvwxyz123456", "request used sk-proj-AbCdEfGhIjKlMnOpQrStUvWxYz012345", "request used AKIAIOSFODNN7EXAMPLE", @@ -133,6 +149,9 @@ const SAFE_REPLAY_STRINGS = [ "see https://example.com/root/replay?case=safe-value", "see https://docs.example.com/guides/reviewgate/policy-traces", "see https://docs.example.com/runs/550e8400-e29b-41d4-a716-446655440000/long-safe-policy-trace-slug", + "probe http://[::1]:3000/api/health/check", + "probe https://[2001:db8::1]/reviewgate/status", + "see https://docs.example.com/search?topic=policy&case=safe-value", ]; function sqlFinding(): Finding { diff --git a/tests/unit/bench-result-schema.test.ts b/tests/unit/bench-result-schema.test.ts index 4c8ba13..6a09775 100644 --- a/tests/unit/bench-result-schema.test.ts +++ b/tests/unit/bench-result-schema.test.ts @@ -206,6 +206,22 @@ const UNSAFE_REPLAY_STRINGS = [ "Authorization: Basic YTpi", "https://user:password@example.com/private", "Authorization: Bearer short-but-secret", + "open %2Fusr%2Flocal%2Fbin%2Freviewgate", + "open file%3A%2F%2F%2Fetc%2Fpasswd", + "open file%253A%252F%252F%252Fetc%252Fpasswd", + "query %2570assword%253Dhunter2", + 'payload {"password":"hunter2"}', + 'payload {"authorization":"Basic YTpi"}', + "client_secret=notverysecret", + "ACCESS-TOKEN=notverysecret", + "https://example.com/download?X-Amz-Signature=aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "https://example.com/download?token=notverysecret", + "https://example.com/download?signature=notverysecret", + "https://example.com/download?password=hunter2", + "https://example.com/download?api-key=notverysecret", + "invalid percent %ZZ", + "invalid percent %2", + "open %2525252Fusr%2525252Flocal%2525252Fbin", "request used ghp_abcdefghijklmnopqrstuvwxyz123456", "request used sk-proj-AbCdEfGhIjKlMnOpQrStUvWxYz012345", "request used AKIAIOSFODNN7EXAMPLE", @@ -218,6 +234,9 @@ const SAFE_REPLAY_STRINGS = [ "see https://example.com/root/replay?case=safe-value", "see https://docs.example.com/guides/reviewgate/policy-traces", "see https://docs.example.com/runs/550e8400-e29b-41d4-a716-446655440000/long-safe-policy-trace-slug", + "probe http://[::1]:3000/api/health/check", + "probe https://[2001:db8::1]/reviewgate/status", + "see https://docs.example.com/search?topic=policy&case=safe-value", ]; function parseThrowableSnapshot(snapshot: unknown) { From 02e59a513489272e9d6bea30f226c4bad95d91c3 Mon Sep 17 00:00:00 2001 From: Codevena Date: Tue, 11 Aug 2026 14:48:56 +0200 Subject: [PATCH 43/55] fix(bench): normalize replay guard inputs --- src/schemas/bench-result.ts | 35 ++++++++++++++++++-------- tests/unit/bench-matrix.test.ts | 13 ++++++++-- tests/unit/bench-result-schema.test.ts | 13 ++++++++-- 3 files changed, 46 insertions(+), 15 deletions(-) diff --git a/src/schemas/bench-result.ts b/src/schemas/bench-result.ts index 441d2e1..dbd962d 100644 --- a/src/schemas/bench-result.ts +++ b/src/schemas/bench-result.ts @@ -214,7 +214,7 @@ export type ThrowableSafeValue = | { [key: string]: ThrowableSafeValue }; const MAX_PERCENT_DECODE_PASSES = 3; -const SENSITIVE_KEY_PARTS = [ +const SENSITIVE_THROWABLE_KEYS = new Set([ "authorization", "cookie", "credential", @@ -222,17 +222,23 @@ const SENSITIVE_KEY_PARTS = [ "passwd", "secret", "token", + "apitoken", "apikey", "privatekey", + "clientsecret", + "accesstoken", "signature", -] as const; + "xamzsignature", + "xamzcredential", + "xamzsecuritytoken", +]); function isSensitiveThrowableKey(value: string): boolean { const normalized = value .normalize("NFKC") .toLowerCase() .replace(/[^a-z0-9]/g, ""); - return SENSITIVE_KEY_PARTS.some((part) => normalized.includes(part)); + return SENSITIVE_THROWABLE_KEYS.has(normalized); } const SensitiveThrowableFieldSchema = z @@ -248,18 +254,25 @@ const KEY_VALUE_PAIR = /(?:"([^"\r\n]{1,128})"|'([^'\r\n]{1,128})'|([A-Za-z][A-Za-z0-9_.-]{0,127}))\s*[:=]\s*(?:"[^"\r\n]*"|'[^'\r\n]*'|[^\s,}\]]+)/g; const CREDENTIAL_VALUE = /(?:\bBearer\s+\S+|\bgh[pousr]_[A-Za-z0-9]{20,}\b|\bgithub_pat_[A-Za-z0-9_]{20,}\b|\bsk-(?:proj-)?[A-Za-z0-9_-]{16,}\b|\b(?:AKIA|ASIA)[A-Z0-9]{16}\b|\bxox[baprs]-[A-Za-z0-9-]{10,}\b)/i; +const PERCENT_ENCODED_BYTE = /%[0-9A-Fa-f]{2}/; +const PERCENT_ENCODED_RUN = /(?:%[0-9A-Fa-f]{2})+/g; function decodePercentToFixedPoint(value: string): string | null { let decoded = value; for (let pass = 0; pass < MAX_PERCENT_DECODE_PASSES; pass += 1) { - if (!decoded.includes("%")) return decoded; - try { - decoded = decodeURIComponent(decoded); - } catch { - return null; - } + if (!PERCENT_ENCODED_BYTE.test(decoded)) return decoded; + let invalidRun = false; + decoded = decoded.replace(PERCENT_ENCODED_RUN, (run) => { + try { + return decodeURIComponent(run); + } catch { + invalidRun = true; + return run; + } + }); + if (invalidRun) return null; } - return decoded.includes("%") ? null : decoded; + return PERCENT_ENCODED_BYTE.test(decoded) ? null : decoded; } function hasSensitiveKeyValuePair(value: string): boolean { @@ -306,7 +319,7 @@ function maskSafeHttpUrls(value: string): string | null { /** Shared at-rest boundary for every string in a captured provider throw. */ export function isAuthoritativeThrowableString(value: string): boolean { - const decodedValue = decodePercentToFixedPoint(value); + const decodedValue = decodePercentToFixedPoint(value.normalize("NFKC")); if (decodedValue === null) return false; const hasUnsafeControlCharacter = Array.from(decodedValue).some((character) => { const codePoint = character.codePointAt(0) ?? 0; diff --git a/tests/unit/bench-matrix.test.ts b/tests/unit/bench-matrix.test.ts index 86bfd6a..4f50dae 100644 --- a/tests/unit/bench-matrix.test.ts +++ b/tests/unit/bench-matrix.test.ts @@ -121,6 +121,11 @@ const UNSAFE_REPLAY_STRINGS = [ "Authorization: Basic YTpi", "https://user:password@example.com/private", "Authorization: Bearer short-but-secret", + "ｐａｓｓｗｏｒｄ=hunter2", + "password：hunter2", + "ａｐｉ＿ｋｅｙ＝notverysecret", + 'payload ｛"ｐａｓｓｗｏｒｄ"："hunter2"｝', + "https://example.com/download？ａｐｉ＿ｋｅｙ＝notverysecret", "open %2Fusr%2Flocal%2Fbin%2Freviewgate", "open file%3A%2F%2F%2Fetc%2Fpasswd", "open file%253A%252F%252F%252Fetc%252Fpasswd", @@ -134,8 +139,6 @@ const UNSAFE_REPLAY_STRINGS = [ "https://example.com/download?signature=notverysecret", "https://example.com/download?password=hunter2", "https://example.com/download?api-key=notverysecret", - "invalid percent %ZZ", - "invalid percent %2", "open %2525252Fusr%2525252Flocal%2525252Fbin", "request used ghp_abcdefghijklmnopqrstuvwxyz123456", "request used sk-proj-AbCdEfGhIjKlMnOpQrStUvWxYz012345", @@ -152,6 +155,12 @@ const SAFE_REPLAY_STRINGS = [ "probe http://[::1]:3000/api/health/check", "probe https://[2001:db8::1]/reviewgate/status", "see https://docs.example.com/search?topic=policy&case=safe-value", + "quota is 50% remaining", + "100% utilization", + "see https://example.com/search?q=100%25+coverage", + "see https://example.com/search?signatureAlgorithm=ed25519", + "see https://example.com/search?token_count=128", + "see https://example.com/search?passwordPolicy=strict", ]; function sqlFinding(): Finding { diff --git a/tests/unit/bench-result-schema.test.ts b/tests/unit/bench-result-schema.test.ts index 6a09775..8064c2c 100644 --- a/tests/unit/bench-result-schema.test.ts +++ b/tests/unit/bench-result-schema.test.ts @@ -206,6 +206,11 @@ const UNSAFE_REPLAY_STRINGS = [ "Authorization: Basic YTpi", "https://user:password@example.com/private", "Authorization: Bearer short-but-secret", + "ｐａｓｓｗｏｒｄ=hunter2", + "password：hunter2", + "ａｐｉ＿ｋｅｙ＝notverysecret", + 'payload ｛"ｐａｓｓｗｏｒｄ"："hunter2"｝', + "https://example.com/download？ａｐｉ＿ｋｅｙ＝notverysecret", "open %2Fusr%2Flocal%2Fbin%2Freviewgate", "open file%3A%2F%2F%2Fetc%2Fpasswd", "open file%253A%252F%252F%252Fetc%252Fpasswd", @@ -219,8 +224,6 @@ const UNSAFE_REPLAY_STRINGS = [ "https://example.com/download?signature=notverysecret", "https://example.com/download?password=hunter2", "https://example.com/download?api-key=notverysecret", - "invalid percent %ZZ", - "invalid percent %2", "open %2525252Fusr%2525252Flocal%2525252Fbin", "request used ghp_abcdefghijklmnopqrstuvwxyz123456", "request used sk-proj-AbCdEfGhIjKlMnOpQrStUvWxYz012345", @@ -237,6 +240,12 @@ const SAFE_REPLAY_STRINGS = [ "probe http://[::1]:3000/api/health/check", "probe https://[2001:db8::1]/reviewgate/status", "see https://docs.example.com/search?topic=policy&case=safe-value", + "quota is 50% remaining", + "100% utilization", + "see https://example.com/search?q=100%25+coverage", + "see https://example.com/search?signatureAlgorithm=ed25519", + "see https://example.com/search?token_count=128", + "see https://example.com/search?passwordPolicy=strict", ]; function parseThrowableSnapshot(snapshot: unknown) { From 60857a5cc88c6f8e42f64178bd60af5e57508800 Mon Sep 17 00:00:00 2001 From: Codevena Date: Tue, 11 Aug 2026 14:56:02 +0200 Subject: [PATCH 44/55] fix(bench): renormalize decoded replay strings --- src/schemas/bench-result.ts | 3 ++- tests/unit/bench-matrix.test.ts | 3 +++ tests/unit/bench-result-schema.test.ts | 3 +++ 3 files changed, 8 insertions(+), 1 deletion(-) diff --git a/src/schemas/bench-result.ts b/src/schemas/bench-result.ts index dbd962d..45a50b0 100644 --- a/src/schemas/bench-result.ts +++ b/src/schemas/bench-result.ts @@ -262,7 +262,7 @@ function decodePercentToFixedPoint(value: string): string | null { for (let pass = 0; pass < MAX_PERCENT_DECODE_PASSES; pass += 1) { if (!PERCENT_ENCODED_BYTE.test(decoded)) return decoded; let invalidRun = false; - decoded = decoded.replace(PERCENT_ENCODED_RUN, (run) => { + const nextDecoded = decoded.replace(PERCENT_ENCODED_RUN, (run) => { try { return decodeURIComponent(run); } catch { @@ -271,6 +271,7 @@ function decodePercentToFixedPoint(value: string): string | null { } }); if (invalidRun) return null; + decoded = nextDecoded.normalize("NFKC"); } return PERCENT_ENCODED_BYTE.test(decoded) ? null : decoded; } diff --git a/tests/unit/bench-matrix.test.ts b/tests/unit/bench-matrix.test.ts index 4f50dae..2982ba9 100644 --- a/tests/unit/bench-matrix.test.ts +++ b/tests/unit/bench-matrix.test.ts @@ -126,6 +126,9 @@ const UNSAFE_REPLAY_STRINGS = [ "ａｐｉ＿ｋｅｙ＝notverysecret", 'payload ｛"ｐａｓｓｗｏｒｄ"："hunter2"｝', "https://example.com/download？ａｐｉ＿ｋｅｙ＝notverysecret", + "password%EF%BC%9Ahunter2", + "%EF%BD%90%EF%BD%81%EF%BD%93%EF%BD%93%EF%BD%97%EF%BD%8F%EF%BD%92%EF%BD%84%EF%BC%9Dhunter2", + "https://example.com/download%EF%BC%9Fapi_key%EF%BC%9Dnotverysecret", "open %2Fusr%2Flocal%2Fbin%2Freviewgate", "open file%3A%2F%2F%2Fetc%2Fpasswd", "open file%253A%252F%252F%252Fetc%252Fpasswd", diff --git a/tests/unit/bench-result-schema.test.ts b/tests/unit/bench-result-schema.test.ts index 8064c2c..9817bda 100644 --- a/tests/unit/bench-result-schema.test.ts +++ b/tests/unit/bench-result-schema.test.ts @@ -211,6 +211,9 @@ const UNSAFE_REPLAY_STRINGS = [ "ａｐｉ＿ｋｅｙ＝notverysecret", 'payload ｛"ｐａｓｓｗｏｒｄ"："hunter2"｝', "https://example.com/download？ａｐｉ＿ｋｅｙ＝notverysecret", + "password%EF%BC%9Ahunter2", + "%EF%BD%90%EF%BD%81%EF%BD%93%EF%BD%93%EF%BD%97%EF%BD%8F%EF%BD%92%EF%BD%84%EF%BC%9Dhunter2", + "https://example.com/download%EF%BC%9Fapi_key%EF%BC%9Dnotverysecret", "open %2Fusr%2Flocal%2Fbin%2Freviewgate", "open file%3A%2F%2F%2Fetc%2Fpasswd", "open file%253A%252F%252F%252Fetc%252Fpasswd", From 05c0038b017c5ca0dea6c3e79504398917d200ca Mon Sep 17 00:00:00 2001 From: Codevena Date: Tue, 11 Aug 2026 16:13:46 +0200 Subject: [PATCH 45/55] feat(rig): validate and replay policy traces --- src/cli/commands/gate.ts | 12 + src/cli/commands/rig.ts | 108 ++- src/cli/index.ts | 59 +- src/core/orchestrator.ts | 80 +- src/core/policy/replay-capture.ts | 439 +++++++++ src/rig/ablate.ts | 15 +- src/rig/driver.ts | 93 +- src/rig/harvest.ts | 35 + src/rig/policy-replay-state.ts | 883 +++++++++++++++++++ src/rig/replay.ts | 525 ++++++++++- src/schemas/policy-replay.ts | 271 ++++++ src/schemas/rig-manifest.ts | 42 + src/schemas/rig-result.ts | 72 ++ tests/unit/cli-required-args.test.ts | 49 + tests/unit/orchestrator-policy-trace.test.ts | 10 + tests/unit/policy-replay-capture.test.ts | 419 +++++++++ tests/unit/rig-ablate.test.ts | 28 + tests/unit/rig-driver.test.ts | 111 +++ tests/unit/rig-harvest.test.ts | 21 + tests/unit/rig-replay.test.ts | 587 +++++++++++- 20 files changed, 3778 insertions(+), 81 deletions(-) create mode 100644 src/core/policy/replay-capture.ts create mode 100644 src/rig/policy-replay-state.ts create mode 100644 src/schemas/policy-replay.ts create mode 100644 tests/unit/policy-replay-capture.test.ts diff --git a/src/cli/commands/gate.ts b/src/cli/commands/gate.ts index 17c7224..c4983d9 100644 --- a/src/cli/commands/gate.ts +++ b/src/cli/commands/gate.ts @@ -16,6 +16,7 @@ import type { loadEffectiveConfig } from "../../config/global.ts"; import { buildSessionStartInjection } from "../../core/agent-lessons/inject.ts"; import { LoopDriver } from "../../core/loop-driver.ts"; import { Orchestrator } from "../../core/orchestrator.ts"; +import { resolvePolicyReplayCaptureSink } from "../../core/policy/replay-capture.ts"; import { type SnapshotFileEntry, snapshotReviewedFiles } from "../../core/reviewed-snapshot.ts"; import { computeForeignFiles } from "../../core/session-manifest.ts"; import { StateStore } from "../../core/state-store.ts"; @@ -1127,6 +1128,14 @@ async function runStopGate( // A corrupt dirty.flag (ctx.diffIncomplete) OR a collectDiff-truncation trailer // both mean the diff isn't a trustworthy complete picture. const diffIncomplete = ctx.diffIncomplete || diffMarkedIncomplete(diff); + const replaySinkEnv = process.env.REVIEWGATE_RIG_REPLAY_DIR; + const replayCapture = + replaySinkEnv === undefined + ? null + : resolvePolicyReplayCaptureSink({ + sinkDir: replaySinkEnv, + measuredRepoRoot: input.repoRoot, + }); const orchestrator = new Orchestrator({ repoRoot: input.repoRoot, config: cfg, @@ -1156,6 +1165,9 @@ async function runStopGate( ...(foreignFiles ? { foreignFiles } : {}), // S2: session_id + dirtyNow snapshot → orchestrator stamps session_attributable / whole_diff_attributable. ...(attribution ? { attribution } : {}), + ...(replayCapture === null + ? {} + : { policyReplayCapture: { sinkDir: replayCapture.sinkDir, sourceCommit: gitInfo.sha } }), }); const driver = new LoopDriver({ diff --git a/src/cli/commands/rig.ts b/src/cli/commands/rig.ts index 6470662..385d634 100644 --- a/src/cli/commands/rig.ts +++ b/src/cli/commands/rig.ts @@ -4,8 +4,18 @@ // layer (fp-ledger, reputation, region memory, lore, agent-lessons) inert. The rig measures // what that structurally cannot: the gate as an interactive loop, over a run whose history // accumulates. -import { constants, accessSync, existsSync, mkdirSync, readFileSync } from "node:fs"; -import { dirname, isAbsolute, resolve, sep } from "node:path"; +import { + constants, + accessSync, + existsSync, + lstatSync, + mkdirSync, + readFileSync, + realpathSync, +} from "node:fs"; +import { dirname, isAbsolute, relative, resolve, sep } from "node:path"; +import { POLICY_CATALOG_VERSION } from "../../core/policy/catalog.ts"; +import { resolvePolicyReplayCaptureSink } from "../../core/policy/replay-capture.ts"; import { SUPPRESSION_LAYERS, type SuppressionLayer, @@ -15,12 +25,18 @@ import { } from "../../rig/ablate.ts"; import { type DriverRunManifest, runDriver } from "../../rig/driver.ts"; import { harvest } from "../../rig/harvest.ts"; -import { renderReplayReport, replay } from "../../rig/replay.ts"; +import { createPolicyStateSnapshot } from "../../rig/policy-replay-state.ts"; +import { + renderPolicyAblationRows, + renderReplayReport, + replay, + replayPolicyAblations, +} from "../../rig/replay.ts"; import { renderRigReport } from "../../rig/report.ts"; import { loadTurnScript } from "../../rig/turn-script.ts"; import { type RigResult, RigResultSchema } from "../../schemas/rig-result.ts"; import { writeFileAtomic } from "../../utils/atomic-write.ts"; -import { workingTreeDirtyFiles } from "../../utils/git.ts"; +import { gitHeadSha, workingTreeDirtyFiles } from "../../utils/git.ts"; export interface RigRunInput { scriptPath: string; @@ -112,15 +128,62 @@ export async function runRigRun(input: RigRunInput): Promise `rig run: ${input.repoRoot} has ${dirty.length} uncommitted change(s). This run would let an agent edit that directory with acceptEdits, driven by prompts from ${input.scriptPath}. Point it at a throwaway repo, or pass allowDirtyRepo once you have read the script and accept what it will do.`, ); } + const repoReal = realpathSync(input.repoRoot); + const output = resolve(input.outDir); + mkdirSync(output, { recursive: true, mode: 0o700 }); + const outputStat = lstatSync(output); + if (outputStat.isSymbolicLink() || !outputStat.isDirectory()) { + throw new Error("rig run: the output root must be an ordinary directory"); + } + const outputReal = realpathSync(output); + const outputRelative = relative(repoReal, outputReal); + const repoRelative = relative(outputReal, repoReal); + const related = (value: string): boolean => + value === "" || (!value.startsWith(`..${sep}`) && value !== ".." && !isAbsolute(value)); + if (related(outputRelative) || related(repoRelative)) { + throw new Error( + "rig run: --out must be separate from the measured repository so replay artifacts cannot change the measured tree", + ); + } + const replaySinkDir = resolve(outputReal, "policy-replay"); + mkdirSync(replaySinkDir, { mode: 0o700 }); + const captureSink = resolvePolicyReplayCaptureSink({ + sinkDir: replaySinkDir, + measuredRepoRoot: repoReal, + }); + if (captureSink === null || relative(outputReal, captureSink.sinkDir).startsWith("..")) { + throw new Error("rig run: policy replay sink is not contained by the Rig output root"); + } + const initialState = createPolicyStateSnapshot({ + sourceRepoRoot: repoReal, + outputRoot: outputReal, + }); + const sourceCommit = await gitHeadSha(repoReal); + if (sourceCommit === null) throw new Error("rig run: could not resolve the source commit"); + const emptyCassetteSha256 = new Bun.CryptoHasher("sha256").update("").digest("hex"); process.stderr.write( `rig run: an agent will EDIT ${input.repoRoot} with acceptEdits, for ${script.turns.length} scripted turn(s).\n`, ); return await runDriver({ scriptPath: input.scriptPath, - outDir: input.outDir, - repoRoot: input.repoRoot, + outDir: outputReal, + repoRoot: repoReal, agentCmd: claudeAgentCmd, maxTurns: input.maxTurns ?? script.turns.length, + policyReplay: { + sinkDir: captureSink.sinkDir, + cassettePath, + metadata: { + catalogVersion: POLICY_CATALOG_VERSION, + sourceCommit, + initialStateRef: initialState.ref, + initialStateSha256: initialState.sha256, + initialStateDigest: initialState.stateSha256, + cassetteSha256: emptyCassetteSha256, + cassetteRef: "cassette.jsonl", + captureDir: "policy-replay", + }, + }, }); } @@ -144,36 +207,47 @@ export interface RigAblateInput { scriptPath: string; /** Omitted → every layer, as a matrix. */ layer?: SuppressionLayer | undefined; + sourceRepoRoot?: string | undefined; } /** - * Re-derive the metrics with one suppression layer switched off (or all four, as a matrix). - * - * Offline and pure — it reads two files and computes. No agent, no network, no `.reviewgate/`, - * so the Δ is attributable to the layer and to nothing else. + * Exact traced results replay every closed-catalog pass in isolated branches. Legacy results + * retain the old four-layer heuristic with a mandatory non-authoritative label. */ -export function runRigAblate(input: RigAblateInput): string { +export async function runRigAblate(input: RigAblateInput): Promise { const base = loadResult(input.resultPath); + if (base.policyReplay?.authoritative === true) { + const siblingManifest = resolve(dirname(input.resultPath), "manifest.json"); + const manifestPath = existsSync(siblingManifest) + ? siblingManifest + : base.provenance.manifest_path; + return renderPolicyAblationRows( + await replayPolicyAblations({ + manifestPath, + sourceRepoRoot: input.sourceRepoRoot ?? process.cwd(), + }), + ); + } const tags = seededTagsFromScript(input.scriptPath); const layers = input.layer === undefined ? [...SUPPRESSION_LAYERS] : [input.layer]; - return renderAblationMatrix( + return `NON-AUTHORITATIVE LEGACY ANALYSIS — exact policy opportunities were not captured.\n${renderAblationMatrix( base, layers.map((l) => ablate(base, l, tags)), - ); + )}`; } export interface RigReplayInput { manifestPath: string; scriptPath: string; cassettePath?: string | undefined; + sourceRepoRoot?: string | undefined; } /** - * Harness self-check. Returns the rendered report plus whether it PASSED, so the CLI can - * exit non-zero — a determinism check that always exits 0 cannot gate anything. + * Exact traced runs validate/counterfactually replay policy; legacy runs keep the harness check. */ -export function runRigReplay(input: RigReplayInput): { text: string; ok: boolean } { - const report = replay(input); +export async function runRigReplay(input: RigReplayInput): Promise<{ text: string; ok: boolean }> { + const report = await replay(input); return { text: renderReplayReport(report), ok: report.deterministic }; } diff --git a/src/cli/index.ts b/src/cli/index.ts index 2544ebe..42c18ae 100644 --- a/src/cli/index.ts +++ b/src/cli/index.ts @@ -7,6 +7,7 @@ import type { AgentHostSelection } from "../hosts/hooks.ts"; import { repoClaudeHookActive } from "../hosts/user-hooks.ts"; import type { ProviderId } from "../providers/registry.ts"; import { SUPPRESSION_LAYERS, type SuppressionLayer, isSuppressionLayer } from "../rig/ablate.ts"; +import { RigAuthorityError } from "../rig/policy-replay-state.ts"; import { RG_VERSION } from "../version.ts"; import { runAuditVerify } from "./commands/audit.ts"; import { @@ -59,6 +60,16 @@ function failArg(message: string): never { process.exit(1); } +async function runRigAuthorityCommand(run: () => T | Promise): Promise { + try { + return await run(); + } catch (error) { + if (!(error instanceof RigAuthorityError)) throw error; + process.stderr.write(`${error.message}\n`); + process.exit(error.exitCode); + } +} + // The activity query the user-scoped shims call. Exit 0 = a repo-local Claude hook for // this event will really fire here, so the user shim stands down; 1 = it will not; 2 = the // event is missing or unrecognised. Nothing is ever printed: the shims treat every @@ -1061,12 +1072,14 @@ const rig = defineCommand({ }, out: { type: "string", description: "Write the result JSON here (default: stdout only)" }, }, - run({ args }) { - const result = runRigHarvest({ - scriptPath: args.script as string, - manifestPath: args.manifest as string, - outPath: args.out as string | undefined, - }); + async run({ args }) { + const result = await runRigAuthorityCommand(() => + runRigHarvest({ + scriptPath: args.script as string, + manifestPath: args.manifest as string, + outPath: args.out as string | undefined, + }), + ); const p = result.provenance; const slope = result.metrics.fpBurdenSlope.slope === null @@ -1117,7 +1130,7 @@ const rig = defineCommand({ meta: { name: "ablate", description: - "Re-derive the metrics with one suppression layer switched off (default: all four, as a Δ matrix). Offline, free, and a pure function of the result — it never re-drives the agent.", + "Run exact closed-catalog policy ablations for traced runs; legacy four-layer results remain explicitly non-authoritative.", }, args: { result: { type: "string", required: true, description: "result.json from `rig harvest`" }, @@ -1131,7 +1144,7 @@ const rig = defineCommand({ description: `One of ${SUPPRESSION_LAYERS.join(" | ")} (default: all)`, }, }, - run({ args }) { + async run({ args }) { const layer = args.layer as string | undefined; if (layer !== undefined && !isSuppressionLayer(layer)) { console.error( @@ -1140,11 +1153,14 @@ const rig = defineCommand({ process.exit(2); } process.stdout.write( - runRigAblate({ - resultPath: args.result as string, - scriptPath: args.script as string, - ...(layer === undefined ? {} : { layer: layer as SuppressionLayer }), - }), + await runRigAuthorityCommand(() => + runRigAblate({ + resultPath: args.result as string, + scriptPath: args.script as string, + sourceRepoRoot: process.cwd(), + ...(layer === undefined ? {} : { layer: layer as SuppressionLayer }), + }), + ), ); }, }), @@ -1152,7 +1168,7 @@ const rig = defineCommand({ meta: { name: "replay", description: - "Self-check of the HARNESS: re-derive a recorded run's metrics twice and assert they match (the acceptance test an aggregator refactor needs). Never a counterfactual, never re-drives the agent.", + "Validate exact policy envelopes and replay production policy in isolated checkouts without live provider calls; legacy runs get only the deterministic harness check.", }, args: { manifest: { type: "string", required: true, description: "manifest.json from `rig run`" }, @@ -1162,12 +1178,15 @@ const rig = defineCommand({ description: "Also check the recording's integrity (entry count, FIFO keys, bodies)", }, }, - run({ args }) { - const report = runRigReplay({ - manifestPath: args.manifest as string, - scriptPath: args.script as string, - cassettePath: args.cassette as string | undefined, - }); + async run({ args }) { + const report = await runRigAuthorityCommand(() => + runRigReplay({ + manifestPath: args.manifest as string, + scriptPath: args.script as string, + cassettePath: args.cassette as string | undefined, + sourceRepoRoot: process.cwd(), + }), + ); process.stdout.write(`${report.text}\n`); // Non-zero on nondeterminism: this is a CHECK, and a check that always exits 0 is a // report. CI (or a refactor's acceptance step) must be able to gate on it. diff --git a/src/core/orchestrator.ts b/src/core/orchestrator.ts index c40b617..8a17848 100644 --- a/src/core/orchestrator.ts +++ b/src/core/orchestrator.ts @@ -10,7 +10,7 @@ import { writeFileSync, } from "node:fs"; import { tmpdir } from "node:os"; -import { isAbsolute, join, relative } from "node:path"; +import { dirname, isAbsolute, join, relative } from "node:path"; import type { AuditLogger } from "../audit/logger.ts"; import { computeBehaviorHash } from "../cache/behavior-hash.ts"; import { computeCacheKey, getCachedReview, putCachedReview } from "../cache/cache.ts"; @@ -42,6 +42,7 @@ import { collectReferencedFileContents } from "../research/plan-refs.ts"; import { researchPath, writeResearch } from "../research/research-writer.ts"; import { buildSymbolGraph, enclosingSymbol } from "../research/symbol-graph.ts"; import { analyzeUiFiles } from "../research/ui-analysis.ts"; +import { createPolicyStateSnapshot } from "../rig/policy-replay-state.ts"; import { sandboxRuntimeAvailable } from "../sandbox/availability.ts"; import { SandboxUnavailableError } from "../sandbox/errors.ts"; import { @@ -103,6 +104,10 @@ import { classifyEntry } from "./lore/staleness.ts"; import { type LoreEntryParsed, loadLore } from "./lore/store.ts"; import { PERSONA_REAFFIRM, reaffirmFor, resolvePersonas } from "./personas.ts"; import { POLICY_CATALOG_VERSION, POLICY_PASSES } from "./policy/catalog.ts"; +import { + capturePolicyReplayEnvelope, + serializePolicyReplayAggregateInputs, +} from "./policy/replay-capture.ts"; import type { PolicyExecutionOptions } from "./policy/replay.ts"; import { resolvePolicyExecutionOptions } from "./policy/replay.ts"; import { OrderedResponseHashes } from "./policy/response-hashes.ts"; @@ -246,6 +251,8 @@ export interface OrchestratorInput { // Internal-only policy instrumentation/ablation. Normal Gate construction omits // this and resolves to persist with an AuditLogger; direct/tests resolve off. policyExecution?: PolicyExecutionOptions; + /** Internal Rig sink only. It carries no pass/ablation controls. */ + policyReplayCapture?: { sinkDir: string; sourceCommit: string }; } // P1a (bench): a single reviewer's pre-aggregation output, captured per attempt. @@ -738,6 +745,18 @@ export class Orchestrator { this.input.policyExecution, this.input.audit !== undefined, ); + let policyReplayStateSha256: string | null = null; + if (this.input.policyReplayCapture !== undefined) { + try { + policyReplayStateSha256 = createPolicyStateSnapshot({ + sourceRepoRoot: repo, + outputRoot: dirname(this.input.policyReplayCapture.sinkDir), + }).stateSha256; + } catch { + // Telemetry only: an exact snapshot failure must never change the Gate verdict. + policyReplayStateSha256 = null; + } + } // S1: render prior adjudications ONCE — injected as trusted prompt context (before the // untrusted diff fence) AND hashed into the behavior cache key below. const adjudicationsText = renderAdjudications(opts.priorAdjudications ?? []); @@ -2313,13 +2332,19 @@ export class Orchestrator { // SEMANTICALLY fabricated (e.g. an invented `outerHTML` XSS sink where the code only sets // a React aria-label). Only fires when there is a CRITICAL to judge; any error → no demote. const groundingCfg = this.input.config.phases.grounding; + let groundingVerdicts = new Map(); + let groundingLlmStatus: "ran" | "not-run" | "error" = "not-run"; if (groundingCfg && groundedFindings.some((f) => f.severity === "CRITICAL")) { const gAdapter = this.input.adapters[groundingCfg.provider]; const gProviderCfg = this.input.config.providers[groundingCfg.provider] as | ProviderConfig | undefined; if (gAdapter && gProviderCfg) { - const { map, rawResponseSha256: groundingResponseSha256 } = await judgeGrounding( + const { + map, + status: groundingStatus, + rawResponseSha256: groundingResponseSha256, + } = await judgeGrounding( gAdapter, { model: groundingCfg.model ?? gProviderCfg.model, @@ -2334,6 +2359,9 @@ export class Orchestrator { groundedFindings, groundingCorpus, ); + groundingVerdicts = map; + groundingLlmStatus = + groundingStatus === "ran" ? "ran" : groundingStatus === "error" ? "error" : "not-run"; if (groundingResponseSha256 !== undefined) { rawResponseSha256.push(groundingResponseSha256); groundedFindings = applyGroundingJudgeVerdicts(groundedFindings, map, policyRuntime); @@ -2538,7 +2566,7 @@ export class Orchestrator { : "stage-precondition-miss"; } - const agg = aggregate({ + const aggregateInput: AggregateInput = { findings: groundedFindings, // Distinct reviewer identities, NOT raw slot count: collapsed fallbacks // (two slots → same provider:persona) must not satisfy the singleton- @@ -2578,7 +2606,8 @@ export class Orchestrator { ...(activeRegions ? { rejectedRegions: activeRegions } : {}), ...(policyRuntime === undefined ? {} : { policyRuntime }), ...(policyRuntime === undefined ? {} : { policyInactive }), - }); + }; + const agg = aggregate(aggregateInput); // Include critic-DROPPED likely_fp findings (INFO → drop): they never reach // dedupedFindings, so filtering it alone undercounts the critic's activity. @@ -2783,6 +2812,49 @@ export class Orchestrator { verdict: agg.verdict, finalFindings, }); + if ( + this.input.policyReplayCapture !== undefined && + policyReplayStateSha256 !== null && + policyTrace !== undefined && + policyTrace !== null + ) { + try { + capturePolicyReplayEnvelope({ + sinkDir: this.input.policyReplayCapture.sinkDir, + measuredRepoRoot: repo, + envelope: { + schema: "reviewgate.policy-replay-envelope.v1", + catalog_version: POLICY_CATALOG_VERSION, + run_id: opts.runId, + iter: opts.iter, + source_commit: this.input.policyReplayCapture.sourceCommit, + exact_diff: this.input.diff, + pre_policy_findings: structuredClone(symbolFindings), + grounding: { + corpus: groundingCorpus, + verdicts: [...groundingVerdicts] + .map(([signature, verdict]) => ({ signature, ...verdict })) + .sort((left, right) => + left.signature < right.signature ? -1 : left.signature > right.signature ? 1 : 0, + ), + llm_status: groundingLlmStatus, + }, + aggregate: serializePolicyReplayAggregateInputs(aggregateInput), + policy_final_findings: structuredClone(agg.dedupedFindings), + pre_policy: { + self_refutation_enabled: selfRefutationEnabled, + hypothetical_enabled: hypotheticalEnabled, + }, + state_sha256: policyReplayStateSha256, + raw_response_sha256: [...rawResponseSha256], + policy_trace: policyTrace, + lossless: true, + }, + }); + } catch { + // Capture and redaction are diagnostic telemetry. Never alter production behavior. + } + } let policySummary: PolicySummary | undefined; if (policyExecution.trace === "persist") { // Keep the gate's abort boundary ahead of all persistence. The complete diff --git a/src/core/policy/replay-capture.ts b/src/core/policy/replay-capture.ts new file mode 100644 index 0000000..219ee96 --- /dev/null +++ b/src/core/policy/replay-capture.ts @@ -0,0 +1,439 @@ +import { createHash } from "node:crypto"; +import { + constants, + closeSync, + existsSync, + fstatSync, + lstatSync, + openSync, + readFileSync, + realpathSync, +} from "node:fs"; +import { isAbsolute, join, relative, resolve } from "node:path"; +import { canonicalJson } from "../../audit/canonical.ts"; +import { redactHighEntropy } from "../../diff/sanitizer.ts"; +import { isAuthoritativeThrowableString } from "../../schemas/bench-result.ts"; +import { + type PolicyReplayEnvelope, + type PolicyReplayEnvelopeInput, + PolicyReplayEnvelopeInputSchema, + PolicyReplayEnvelopeSchema, +} from "../../schemas/policy-replay.ts"; +import { writeFileIfAbsent } from "../../utils/atomic-write.ts"; +import type { AggregateInput } from "../aggregator.ts"; + +export const POLICY_REPLAY_MAX_BYTES = 1_048_576; + +const FULL_SHA256 = /^[0-9a-f]{64}$/; +const POLICY_REPLAY_REF = /^([0-9a-f]{12})-i(0|[1-9]\d*)-([0-9a-f]{12})\.json$/; + +function persistCaptureStatus( + sinkReal: string, + envelope: PolicyReplayEnvelope, + status: "overflow", +): void { + const runSha12 = sha256(envelope.run_id).slice(0, 12); + const ref = `${runSha12}-i${envelope.iter}.${status}`; + const destination = resolve(sinkReal, ref); + if (!isContained(sinkReal, destination)) return; + const bytes = canonicalJson({ + schema: "reviewgate.policy-replay-status.v1", + run_sha256: sha256(envelope.run_id), + iter: envelope.iter, + status, + }); + try { + writeFileIfAbsent(destination, bytes, { mode: 0o600 }); + } catch { + // Best effort only. The caller still returns overflow and a missing marker becomes + // missing-trace, which remains fail-closed even if it is less specific. + } +} + +export type PolicyReplayCaptureResult = + | { status: "complete"; ref: string; sha256: string; envelope: PolicyReplayEnvelope } + | { status: "overflow"; reason: "too-large" } + | { + status: "error"; + reason: + | "invalid-envelope" + | "invalid-sink" + | "sink-inside-measured-repo" + | "artifact-collision" + | "write-error"; + }; + +export type PolicyReplayVerification = + | { ok: true; envelope: PolicyReplayEnvelope } + | { + ok: false; + reason: + | "invalid-reference" + | "path-escape" + | "missing" + | "not-a-file" + | "too-large" + | "hash-mismatch" + | "invalid-encoding" + | "invalid-json" + | "invalid-envelope" + | "non-canonical" + | "identity-mismatch" + | "lossy" + | "catalog-mismatch" + | "state-digest-mismatch" + | "response-hash-mismatch" + | "read-error"; + }; + +function sha256(value: string | Buffer): string { + return createHash("sha256").update(value).digest("hex"); +} + +function isContained(root: string, candidate: string): boolean { + const rel = relative(root, candidate); + return rel === "" || (!rel.startsWith("..") && !isAbsolute(rel)); +} + +function exactDirectory(path: string): string | null { + if (!existsSync(path)) return null; + const stat = lstatSync(path); + if (stat.isSymbolicLink() || !stat.isDirectory()) return null; + return realpathSync(path); +} + +/** Gate-side validation. The environment value is a sink only, never policy control. */ +export function resolvePolicyReplayCaptureSink(input: { + sinkDir: string; + measuredRepoRoot: string; +}): { sinkDir: string; measuredRepoRoot: string } | null { + try { + if (!isAbsolute(input.sinkDir)) return null; + const sinkReal = exactDirectory(input.sinkDir); + const repoReal = exactDirectory(input.measuredRepoRoot); + if (sinkReal === null || repoReal === null) return null; + if (isContained(repoReal, sinkReal) || isContained(sinkReal, repoReal)) return null; + return { sinkDir: sinkReal, measuredRepoRoot: repoReal }; + } catch { + return null; + } +} + +function sanitizeString(value: string): { value: string; changed: boolean } { + const redacted = redactHighEntropy(value); + if (redacted.count > 0) { + const safe = isAuthoritativeThrowableString(redacted.out) ? redacted.out : ""; + return { value: safe, changed: true }; + } + if (!isAuthoritativeThrowableString(value)) { + return { value: "", changed: true }; + } + return { value, changed: false }; +} + +function sanitizeStrings(value: unknown): { value: unknown; changed: boolean } { + if (typeof value === "string") return sanitizeString(value); + if (Array.isArray(value)) { + let changed = false; + const out = value.map((entry) => { + const sanitized = sanitizeStrings(entry); + changed ||= sanitized.changed; + return sanitized.value; + }); + return { value: out, changed }; + } + if (value !== null && typeof value === "object") { + let changed = false; + const out: Record = {}; + for (const [key, entry] of Object.entries(value as Record)) { + const sanitized = sanitizeStrings(entry); + changed ||= sanitized.changed; + out[key] = sanitized.value; + } + return { value: out, changed }; + } + return { value, changed: false }; +} + +function sortedStrings(values: Iterable): T[] { + return [...values].sort((left, right) => (left < right ? -1 : left > right ? 1 : 0)); +} + +/** Convert Map/Set policy inputs at the production call site into a stable persisted shape. */ +export function serializePolicyReplayAggregateInputs( + input: AggregateInput, +): PolicyReplayEnvelopeInput["aggregate"] { + return { + findings: structuredClone(input.findings), + reviewers_total: input.reviewersTotal, + changed_ranges: [...(input.changedRanges ?? [])] + .map(([file, ranges]) => ({ + file, + ranges: [...ranges] + .map(([start, end]) => ({ start, end })) + .sort((left, right) => left.start - right.start || left.end - right.end), + })) + .sort((left, right) => (left.file < right.file ? -1 : left.file > right.file ? 1 : 0)), + scope_to_diff: input.scopeToDiff !== false, + out_of_diff_blocking: sortedStrings(input.outOfDiffBlocking ?? []), + confidence_floor: input.confidenceFloor ?? 0, + demote_correctness: input.demoteCorrectness === true, + corroborate_critical: input.corroborateCritical === true, + demote_test_security: input.demoteTestSecurity === true, + cap_docs_severity: input.capDocsSeverity === true, + critic: [...(input.critic ?? [])] + .map(([signature, verdict]) => ({ signature, ...verdict })) + .sort((left, right) => + left.signature < right.signature ? -1 : left.signature > right.signature ? 1 : 0, + ), + fp_active: [...(input.fpActive ?? [])] + .map(([signature, value]) => ({ signature, id: value.id })) + .sort((left, right) => + left.signature < right.signature ? -1 : left.signature > right.signature ? 1 : 0, + ), + fp_active_clusters: [...(input.fpActiveClusters ?? [])] + .map(([key, value]) => ({ key, member_ids: sortedStrings(value.member_ids) })) + .sort((left, right) => (left.key < right.key ? -1 : left.key > right.key ? 1 : 0)), + rep_unreliable: sortedStrings(input.repUnreliable ?? []), + protected_reviewers: sortedStrings(input.protectedReviewers ?? []), + foreign_files: sortedStrings(input.foreignFiles ?? []), + cycle_rejected: sortedStrings(input.cycleRejected ?? []), + claimed_fixed: [...(input.claimedFixed ?? [])] + .map(([signature, iter]) => ({ signature, iter })) + .sort((left, right) => + left.signature < right.signature ? -1 : left.signature > right.signature ? 1 : 0, + ), + delta_scope: sortedStrings(input.deltaScope ?? []), + rejected_regions: [...(input.rejectedRegions ?? [])] + .map((region) => ({ ...region, categories: sortedStrings(region.categories) })) + .sort( + (left, right) => + (left.file < right.file ? -1 : left.file > right.file ? 1 : 0) || + left.start_line - right.start_line || + left.end_line - right.end_line, + ), + policy_inactive: Object.entries(input.policyInactive ?? {}) + .map(([pass_id, reason_code]) => ({ + pass_id: pass_id as "judgment.critic" | "scope.diff" | "scope.delta" | "scope.session", + reason_code, + })) + .sort((left, right) => + left.pass_id < right.pass_id ? -1 : left.pass_id > right.pass_id ? 1 : 0, + ), + }; +} + +export function sanitizePolicyReplayEnvelope( + input: PolicyReplayEnvelopeInput, +): PolicyReplayEnvelope { + const structural = PolicyReplayEnvelopeInputSchema.parse(input); + const sanitized = sanitizeStrings(structural); + const value = sanitized.value as PolicyReplayEnvelopeInput; + return PolicyReplayEnvelopeSchema.parse({ + ...value, + lossless: value.lossless && !sanitized.changed, + }); +} + +function readArtifact( + path: string, + realSink: string, +): + | { ok: true; bytes: Buffer } + | { ok: false; reason: "not-a-file" | "path-escape" | "too-large" | "read-error" } { + const before = lstatSync(path); + if ( + before.isSymbolicLink() || + !before.isFile() || + before.nlink !== 1 || + (before.mode & 0o7777) !== 0o600 + ) { + return { ok: false, reason: "not-a-file" }; + } + if (before.size > POLICY_REPLAY_MAX_BYTES) return { ok: false, reason: "too-large" }; + const realPath = realpathSync(path); + if (!isContained(realSink, realPath)) return { ok: false, reason: "path-escape" }; + + const fd = openSync(path, constants.O_RDONLY | constants.O_NOFOLLOW); + try { + const opened = fstatSync(fd); + if (!opened.isFile() || opened.nlink !== 1 || (opened.mode & 0o7777) !== 0o600) { + return { ok: false, reason: "not-a-file" }; + } + if (opened.size > POLICY_REPLAY_MAX_BYTES) return { ok: false, reason: "too-large" }; + if (opened.dev !== before.dev || opened.ino !== before.ino) { + return { ok: false, reason: "read-error" }; + } + const bytes = readFileSync(fd); + if (bytes.length > POLICY_REPLAY_MAX_BYTES) return { ok: false, reason: "too-large" }; + const after = fstatSync(fd); + if ( + after.dev !== opened.dev || + after.ino !== opened.ino || + after.size !== opened.size || + after.mtimeMs !== opened.mtimeMs || + after.ctimeMs !== opened.ctimeMs + ) { + return { ok: false, reason: "read-error" }; + } + const pathAfter = lstatSync(path); + if ( + pathAfter.isSymbolicLink() || + !pathAfter.isFile() || + pathAfter.nlink !== 1 || + (pathAfter.mode & 0o7777) !== 0o600 || + pathAfter.dev !== after.dev || + pathAfter.ino !== after.ino || + realpathSync(path) !== realPath + ) { + return { ok: false, reason: "read-error" }; + } + return { ok: true, bytes }; + } finally { + closeSync(fd); + } +} + +export function capturePolicyReplayEnvelope(input: { + sinkDir: string; + measuredRepoRoot: string; + envelope: PolicyReplayEnvelopeInput; + maxBytes?: number; +}): PolicyReplayCaptureResult { + let envelope: PolicyReplayEnvelope; + try { + envelope = sanitizePolicyReplayEnvelope(input.envelope); + } catch { + return { status: "error", reason: "invalid-envelope" }; + } + let sinkReal: string; + try { + if (!isAbsolute(input.sinkDir) || lstatSync(input.sinkDir).isSymbolicLink()) { + return { status: "error", reason: "invalid-sink" }; + } + sinkReal = exactDirectory(input.sinkDir) ?? ""; + if (sinkReal.length === 0) return { status: "error", reason: "invalid-sink" }; + const repoReal = realpathSync(input.measuredRepoRoot); + if (isContained(repoReal, sinkReal) || isContained(sinkReal, repoReal)) { + return { status: "error", reason: "sink-inside-measured-repo" }; + } + } catch { + return { status: "error", reason: "invalid-sink" }; + } + + const bytes = Buffer.from(canonicalJson(envelope), "utf8"); + const maxBytes = input.maxBytes ?? POLICY_REPLAY_MAX_BYTES; + if (!Number.isSafeInteger(maxBytes) || maxBytes < 0) { + return { status: "error", reason: "invalid-envelope" }; + } + if (bytes.length > maxBytes) { + persistCaptureStatus(sinkReal, envelope, "overflow"); + return { status: "overflow", reason: "too-large" }; + } + const contentSha256 = sha256(bytes); + const runSha12 = sha256(envelope.run_id).slice(0, 12); + const ref = `${runSha12}-i${envelope.iter}-${contentSha256.slice(0, 12)}.json`; + const destination = resolve(sinkReal, ref); + if (!isContained(sinkReal, destination)) return { status: "error", reason: "invalid-sink" }; + try { + if (existsSync(destination)) { + const existing = readArtifact(destination, sinkReal); + return existing.ok && existing.bytes.equals(bytes) + ? { status: "complete", ref, sha256: contentSha256, envelope } + : { status: "error", reason: "artifact-collision" }; + } + const created = writeFileIfAbsent(destination, bytes.toString("utf8"), { mode: 0o600 }); + if (!created && !existsSync(destination)) return { status: "error", reason: "write-error" }; + const published = readArtifact(destination, sinkReal); + if (!published.ok || !published.bytes.equals(bytes)) { + return { status: "error", reason: "artifact-collision" }; + } + return { status: "complete", ref, sha256: contentSha256, envelope }; + } catch { + return { status: "error", reason: "write-error" }; + } +} + +export function verifyPolicyReplayEnvelope(input: { + sinkDir: string; + ref: string; + sha256: string; + authoritative?: boolean; + expectedCatalogVersion?: string; + expectedStateSha256?: string; + expectedResponseSha256?: string[]; +}): PolicyReplayVerification { + if ( + !FULL_SHA256.test(input.sha256) || + isAbsolute(input.ref) || + input.ref.includes("\\") || + !POLICY_REPLAY_REF.test(input.ref) + ) { + return { ok: false, reason: "invalid-reference" }; + } + try { + const sinkReal = exactDirectory(input.sinkDir); + if (sinkReal === null) return { ok: false, reason: "path-escape" }; + const candidate = resolve(input.sinkDir, input.ref); + if (!isContained(resolve(input.sinkDir), candidate)) + return { ok: false, reason: "path-escape" }; + if (!existsSync(candidate)) return { ok: false, reason: "missing" }; + const read = readArtifact(candidate, sinkReal); + if (!read.ok) return read; + const contentSha256 = sha256(read.bytes); + if (contentSha256 !== input.sha256) return { ok: false, reason: "hash-mismatch" }; + const match = POLICY_REPLAY_REF.exec(input.ref); + if (match === null || match[3] !== contentSha256.slice(0, 12)) { + return { ok: false, reason: "identity-mismatch" }; + } + let text: string; + try { + text = new TextDecoder("utf-8", { fatal: true }).decode(read.bytes); + } catch { + return { ok: false, reason: "invalid-encoding" }; + } + let decoded: unknown; + try { + decoded = JSON.parse(text); + } catch { + return { ok: false, reason: "invalid-json" }; + } + const parsed = PolicyReplayEnvelopeSchema.safeParse(decoded); + if (!parsed.success) return { ok: false, reason: "invalid-envelope" }; + const canonical = Buffer.from(canonicalJson(parsed.data), "utf8"); + if (!canonical.equals(read.bytes)) return { ok: false, reason: "non-canonical" }; + if ( + match[1] !== sha256(parsed.data.run_id).slice(0, 12) || + Number(match[2]) !== parsed.data.iter + ) { + return { ok: false, reason: "identity-mismatch" }; + } + if (input.authoritative === true && !parsed.data.lossless) { + return { ok: false, reason: "lossy" }; + } + if ( + input.expectedCatalogVersion !== undefined && + parsed.data.catalog_version !== input.expectedCatalogVersion + ) { + return { ok: false, reason: "catalog-mismatch" }; + } + if ( + input.expectedStateSha256 !== undefined && + parsed.data.state_sha256 !== input.expectedStateSha256 + ) { + return { ok: false, reason: "state-digest-mismatch" }; + } + if ( + input.expectedResponseSha256 !== undefined && + (input.expectedResponseSha256.length !== parsed.data.raw_response_sha256.length || + input.expectedResponseSha256.some( + (hash, index) => hash !== parsed.data.raw_response_sha256[index], + )) + ) { + return { ok: false, reason: "response-hash-mismatch" }; + } + return { ok: true, envelope: parsed.data }; + } catch { + return { ok: false, reason: "read-error" }; + } +} diff --git a/src/rig/ablate.ts b/src/rig/ablate.ts index 8ef63f0..c3c5386 100644 --- a/src/rig/ablate.ts +++ b/src/rig/ablate.ts @@ -49,6 +49,8 @@ export function isSuppressionLayer(v: string): v is SuppressionLayer { } export interface RigAblation { + /** Historical post-hoc reconstruction is diagnostic only; exact rows come from traces. */ + authoritative: false; layer: SuppressionLayer; /** true when the layer's effect is exactly recoverable, i.e. `lower` and `upper` agree */ exact: boolean; @@ -281,7 +283,15 @@ export function ablate( `${unrecoverable} of ${touched} finding(s) carry a SECOND suppressor, so switching off ${layer} alone may not have changed them. They widen the interval instead of being guessed.`, ); } - return { layer, exact, lower, upper, counts: { touched, recovered, unrecoverable }, notes }; + return { + authoritative: false, + layer, + exact, + lower, + upper, + counts: { touched, recovered, unrecoverable }, + notes, + }; } /** Seeded tags by turn index, read from the turn script that produced the run. */ @@ -300,7 +310,8 @@ export function seededTagsFromScript(scriptPath: string): Map */ export function renderAblationMatrix(base: RigResult, ablations: RigAblation[]): string { const L: string[] = []; - L.push("Reviewgate rig — ablation matrix (baseline = full suppression)"); + L.push("Reviewgate rig — NON-AUTHORITATIVE LEGACY ablation matrix"); + L.push("(baseline = full suppression; missing policy opportunities are unknown, never zero)"); L.push(""); const fmtRange = (lo: number, hi: number): string => lo === hi diff --git a/src/rig/driver.ts b/src/rig/driver.ts index 1afd47a..28d1e89 100644 --- a/src/rig/driver.ts +++ b/src/rig/driver.ts @@ -11,6 +11,7 @@ import { mkdirSync, openSync, readFileSync, + readdirSync, statSync, } from "node:fs"; import { join } from "node:path"; @@ -30,6 +31,12 @@ export interface DriverOpts { maxTurns?: number; /** How long to wait for the workspace to go quiescent after the agent exits. */ quiesceTimeoutMs?: number; + /** Rig-owned, prevalidated sink and immutable source identity. Omitted by legacy callers. */ + policyReplay?: { + sinkDir: string; + metadata: NonNullable; + cassettePath: string; + }; } // The manifest shape lives in `src/schemas/rig-manifest.ts` — the harvester parses this file @@ -275,7 +282,12 @@ function startReportArchiver(repoRoot: string, destDir: string): () => void { * wants, and it keeps a multi-megabyte agent turn out of the parent's memory, which the * pipe version would have accumulated there for the whole run. */ -async function runAgent(argv: string[], cwd: string, logPath: string): Promise { +async function runAgent( + argv: string[], + cwd: string, + logPath: string, + replaySinkDir?: string, +): Promise { const fd = openSync(logPath, "a"); try { const proc = Bun.spawn(argv, { @@ -284,6 +296,9 @@ async function runAgent(argv: string[], cwd: string, logPath: string): Promise { + const inventory = new Map(); + const traceRef = /^[0-9a-f]{12}-i(?:0|[1-9]\d*)-[0-9a-f]{12}\.json$/; + const statusRef = /^[0-9a-f]{12}-i(?:0|[1-9]\d*)\.(?:overflow|error)$/; + const names = readdirSync(sinkDir).filter((name) => traceRef.test(name) || statusRef.test(name)); + names.sort((left, right) => { + const leftMatch = /^([0-9a-f]{12})-i(0|[1-9]\d*)-([0-9a-f]{12})\.json$/.exec(left); + const rightMatch = /^([0-9a-f]{12})-i(0|[1-9]\d*)-([0-9a-f]{12})\.json$/.exec(right); + if (leftMatch === null || rightMatch === null) return left.localeCompare(right); + return ( + (leftMatch[1] ?? "").localeCompare(rightMatch[1] ?? "") || + Number(leftMatch[2] ?? "0") - Number(rightMatch[2] ?? "0") || + (leftMatch[3] ?? "").localeCompare(rightMatch[3] ?? "") + ); + }); + for (const name of names) { + try { + inventory.set(name, sha256FileOrEmpty(join(sinkDir, name))); + } catch { + // A hostile/racing entry is left unrecorded. A reviewed turn with no complete + // new artifact becomes `missing`, which authoritative harvest rejects. + } + } + return inventory; +} + /** * Persist WHAT THE AGENT ACTUALLY WROTE this turn, as `/diff.patch`. * @@ -334,6 +383,7 @@ export async function runDriver(opts: DriverOpts): Promise { scriptId: script.id, outDir: opts.outDir, cassettePath: recordingCassettePath(), + ...(opts.policyReplay === undefined ? {} : { policyReplay: opts.policyReplay.metadata }), turns: [], }; mkdirSync(opts.outDir, { recursive: true }); @@ -345,6 +395,8 @@ export async function runDriver(opts: DriverOpts): Promise { // this turn's reviewer traffic, which is what makes the entries addressable per turn. const cassetteBefore = cassetteSize(manifest.cassettePath ?? null); const auditBytesBefore = auditBytes(opts.repoRoot); + const replayBefore = + opts.policyReplay === undefined ? null : policyReplayInventory(opts.policyReplay.sinkDir); // The turn directory is created BEFORE the agent runs, because the agent's transcript // is written into it live (see runAgent). The .reviewgate/ snapshot still happens after. const snapshotDir = join(opts.outDir, "turns", String(turn.index)); @@ -357,6 +409,7 @@ export async function runDriver(opts: DriverOpts): Promise { opts.agentCmd(turn.prompt), opts.repoRoot, join(snapshotDir, "agent.log"), + opts.policyReplay?.sinkDir, ); await awaitQuiescent(opts.repoRoot, quiesceTimeoutMs); } finally { @@ -378,6 +431,30 @@ export async function runDriver(opts: DriverOpts): Promise { // Checked BEFORE the snapshot is declared good: an unreviewed turn is not a slow turn, it // is a turn that produced no measurement, and the run must not quietly accumulate them. const gateReviewed = gateReviewedTurn(opts.repoRoot, auditBytesBefore); + const replayAfter = + opts.policyReplay === undefined ? null : policyReplayInventory(opts.policyReplay.sinkDir); + const changedReplayArtifacts = + replayBefore === null || replayAfter === null + ? [] + : [...replayAfter].filter(([ref, hash]) => replayBefore.get(ref) !== hash); + const replayOverflowed = changedReplayArtifacts.some(([ref]) => ref.endsWith(".overflow")); + const replayErrored = changedReplayArtifacts.some(([ref]) => ref.endsWith(".error")); + const replayTraces = changedReplayArtifacts + .filter(([ref]) => ref.endsWith(".json")) + .map(([ref, sha256]) => ({ ref, sha256 })); + if (manifest.policyReplay !== undefined && opts.policyReplay !== undefined) { + manifest.policyReplay.cassetteSha256 = sha256FileOrEmpty(opts.policyReplay.cassettePath); + try { + writeFileAtomic( + join(opts.outDir, manifest.policyReplay.cassetteRef), + readFileSync(opts.policyReplay.cassettePath, "utf8"), + { mode: 0o600 }, + ); + } catch { + // Missing/unreadable copy leaves the immutable hash bound but no artifact; + // authoritative replay reports the exact missing-cassette reason. + } + } manifest.turns.push({ index: turn.index, snapshotDir, @@ -389,6 +466,20 @@ export async function runDriver(opts: DriverOpts): Promise { ? null : { before: cassetteBefore, after: cassetteAfter }, diffBytes, + ...(opts.policyReplay === undefined + ? {} + : { + policyReplay: { + status: replayOverflowed + ? ("overflow" as const) + : replayErrored + ? ("error" as const) + : replayTraces.length > 0 + ? ("complete" as const) + : ("missing" as const), + traces: replayOverflowed || replayErrored ? [] : replayTraces, + }, + }), }); if (gateReviewed) { consecutiveUnreviewed = 0; diff --git a/src/rig/harvest.ts b/src/rig/harvest.ts index 73820bd..8913c0c 100644 --- a/src/rig/harvest.ts +++ b/src/rig/harvest.ts @@ -33,6 +33,7 @@ import { platform, release } from "node:os"; import { dirname, join } from "node:path"; import { matchesAnyTag } from "../bench/matcher.ts"; import { makeMetric, summarizeSpread } from "../bench/metrics.ts"; +import { POLICY_CATALOG_VERSION, POLICY_PASS_IDS } from "../core/policy/catalog.ts"; import type { DecisionOutcome } from "../schemas/audit-event.ts"; import type { Metric } from "../schemas/bench-result.ts"; import type { Finding } from "../schemas/finding.ts"; @@ -53,6 +54,7 @@ import type { RigTurn } from "../schemas/rig-turn-script.ts"; import type { LoadedRun } from "../stats/load.ts"; import { loadAuditWindow } from "../stats/load.ts"; import { RG_VERSION } from "../version.ts"; +import { validateRigPolicyReplayArtifacts } from "./policy-replay-state.ts"; import { loadTurnScript } from "./turn-script.ts"; /** Below this many defined FP-burden points the slope is not reported at all. */ @@ -523,6 +525,7 @@ export function harvest(manifestPath: string, scriptPath: string): RigResult { const manifest = RigManifestSchema.parse( JSON.parse(readFileSync(manifestPath, "utf8")) as unknown, ); + const policyReplay = validateRigPolicyReplayArtifacts({ manifest, manifestPath }); const script = loadTurnScript(scriptPath); if (manifest.scriptId !== script.id) { throw new Error( @@ -551,6 +554,21 @@ export function harvest(manifestPath: string, scriptPath: string): RigResult { previous, warnings, ); + const replayTraces = policyReplay?.turns.get(manifestTurn.index); + if (replayTraces !== undefined) { + turn.record.policyReplay = { + status: "complete", + reason: null, + traces: replayTraces.map(({ ref, sha256, envelope }) => ({ + ref, + sha256, + runId: envelope.run_id, + iter: envelope.iter, + stateSha256: envelope.state_sha256, + lossless: envelope.lossless, + })), + }; + } turns.push(turn); panelSlots.push(...panel); previous = cumulative; @@ -659,6 +677,23 @@ export function harvest(manifestPath: string, scriptPath: string): RigResult { }, suppression, }, + policyReplay: + policyReplay === null + ? { + authoritative: false, + catalogVersion: null, + sourceCommit: null, + passIds: [], + reason: + "legacy run: no exact policy replay metadata; four-layer counts are non-authoritative", + } + : { + authoritative: true, + catalogVersion: POLICY_CATALOG_VERSION, + sourceCommit: policyReplay.sourceCommit, + passIds: [...POLICY_PASS_IDS], + reason: null, + }, warnings, }; // Validate what we are about to hand out: the null contracts in RigTurnRecordSchema are the diff --git a/src/rig/policy-replay-state.ts b/src/rig/policy-replay-state.ts new file mode 100644 index 0000000..18a78d0 --- /dev/null +++ b/src/rig/policy-replay-state.ts @@ -0,0 +1,883 @@ +import { execFileSync } from "node:child_process"; +import { createHash } from "node:crypto"; +import { + constants, + closeSync, + existsSync, + fstatSync, + lstatSync, + mkdirSync, + mkdtempSync, + openSync, + readFileSync, + readdirSync, + realpathSync, + rmSync, + statSync, + writeFileSync, +} from "node:fs"; +import { tmpdir } from "node:os"; +import { basename, dirname, isAbsolute, join, relative, resolve, sep } from "node:path"; +import { z } from "zod"; +import { canonicalJson } from "../audit/canonical.ts"; +import { POLICY_CATALOG_VERSION } from "../core/policy/catalog.ts"; +import { verifyPolicyReplayEnvelope } from "../core/policy/replay-capture.ts"; +import { CassetteEntrySchema } from "../schemas/cassette.ts"; +import type { PolicyReplayEnvelope } from "../schemas/policy-replay.ts"; +import type { RigManifest } from "../schemas/rig-manifest.ts"; +import { writeFileIfAbsent } from "../utils/atomic-write.ts"; + +const STATE_MAX_FILE_BYTES = 8 * 1024 * 1024; +const STATE_MAX_TOTAL_BYTES = 64 * 1024 * 1024; +const STATE_MAX_FILES = 10_000; +const CASSETTE_MAX_BYTES = 64 * 1024 * 1024; +const STATE_MANIFEST_REF = /^policy-state\/[0-9a-f]{64}\.json$/; +const STATE_TREE_REF = /^policy-state\/[0-9a-f]{64}\/\.reviewgate$/; +const GIT_OBJECT_ID = /^(?:[0-9a-f]{40}|[0-9a-f]{64})$/; + +interface StateEntry { + path: string; + size: number; + sha256: string; + bytes: Buffer; +} + +const PolicyStateManifestSchema = z + .object({ + schema: z.literal("reviewgate.policy-state-snapshot.v1"), + catalog_version: z.literal(POLICY_CATALOG_VERSION), + state_sha256: z.string().regex(/^[0-9a-f]{64}$/), + files: z.array( + z + .object({ + path: z.string().min(1), + size: z.number().int().nonnegative(), + sha256: z.string().regex(/^[0-9a-f]{64}$/), + }) + .strict(), + ), + }) + .strict() + .superRefine((value, ctx) => { + for (let index = 0; index < value.files.length; index += 1) { + const entry = value.files[index]; + try { + validateRelativeStatePath(entry?.path ?? ""); + } catch { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + path: ["files", index, "path"], + message: "invalid state path", + }); + } + if (index > 0 && (value.files[index - 1]?.path ?? "") >= (entry?.path ?? "")) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + path: ["files", index, "path"], + message: "state paths must be uniquely sorted", + }); + } + } + }); + +type PolicyStateManifest = z.infer; + +export interface PolicyStateSnapshot { + ref: string; + sha256: string; + stateRef: string; + stateSha256: string; +} + +export interface ReplayBranch { + checkoutRoot: string; + startingStateSha256: string; +} + +export interface ReplayBranches { + root: string; + baseline: ReplayBranch; + counterfactual: ReplayBranch; +} + +export type RigAuthorityInvalidity = + | "missing-trace" + | "trace-status" + | "trace-overflow" + | "non-canonical-trace" + | "invalid-trace" + | "lossy-trace" + | "catalog-mismatch" + | "source-commit-mismatch" + | "state-digest-mismatch" + | "response-hash-mismatch" + | "missing-cassette" + | "cassette-hash-mismatch" + | "invalid-cassette" + | "source-state-alias" + | "live-provider-call"; + +export class RigAuthorityError extends Error { + readonly exitCode = 4; + + constructor( + readonly code: RigAuthorityInvalidity, + message: string, + ) { + super(`rig policy replay invalid (${code}): ${message}`); + this.name = "RigAuthorityError"; + } +} + +export interface ValidatedRigPolicyReplay { + sourceCommit: string; + initialStateRoot: string; + initialStateSha256: string; + cassettePath: string; + turns: Map< + number, + Array<{ ref: string; sha256: string; envelope: PolicyReplayEnvelope; stateRoot: string }> + >; +} + +function sha256(value: string | Buffer): string { + return createHash("sha256").update(value).digest("hex"); +} + +function isContained(root: string, candidate: string): boolean { + const rel = relative(root, candidate); + return rel === "" || (!rel.startsWith(`..${sep}`) && rel !== ".." && !isAbsolute(rel)); +} + +function validateRelativeStatePath(value: string): void { + if ( + value.length === 0 || + value.includes("\\") || + value.includes("\0") || + isAbsolute(value) || + value.split("/").some((part) => part === "" || part === "." || part === "..") + ) { + throw new Error(`invalid policy state path: ${value}`); + } +} + +function readStableFile( + path: string, + maxBytes = STATE_MAX_FILE_BYTES, + requireMode0600 = false, +): Buffer { + const before = lstatSync(path); + if (before.isSymbolicLink()) throw new Error(`policy state contains symlink: ${path}`); + if (!before.isFile()) throw new Error(`policy state contains special file: ${path}`); + if (before.nlink !== 1) throw new Error(`policy state contains hardlink: ${path}`); + if (requireMode0600 && (before.mode & 0o7777) !== 0o600) { + throw new Error(`policy artifact mode is not 0600: ${path}`); + } + if (before.size > maxBytes) throw new Error(`policy artifact exceeds limit: ${path}`); + const realBefore = realpathSync(path); + const fd = openSync(path, constants.O_RDONLY | constants.O_NOFOLLOW); + try { + const opened = fstatSync(fd); + if ( + !opened.isFile() || + opened.nlink !== 1 || + (requireMode0600 && (opened.mode & 0o7777) !== 0o600) + ) { + throw new Error(`policy state contains non-regular file: ${path}`); + } + if (opened.dev !== before.dev || opened.ino !== before.ino || opened.size !== before.size) { + throw new Error(`policy state changed while opening: ${path}`); + } + const bytes = readFileSync(fd); + if (bytes.length !== opened.size || bytes.length > maxBytes) { + throw new Error(`policy state changed while reading: ${path}`); + } + const after = fstatSync(fd); + const pathAfter = lstatSync(path); + if ( + after.dev !== opened.dev || + after.ino !== opened.ino || + after.size !== opened.size || + after.mtimeMs !== opened.mtimeMs || + after.ctimeMs !== opened.ctimeMs || + pathAfter.isSymbolicLink() || + !pathAfter.isFile() || + pathAfter.nlink !== 1 || + (requireMode0600 && (pathAfter.mode & 0o7777) !== 0o600) || + pathAfter.dev !== after.dev || + pathAfter.ino !== after.ino || + realpathSync(path) !== realBefore + ) { + throw new Error(`policy state changed while reading: ${path}`); + } + return bytes; + } finally { + closeSync(fd); + } +} + +function collectStateEntries(stateRoot: string): StateEntry[] { + const rootStat = lstatSync(stateRoot); + if (rootStat.isSymbolicLink()) throw new Error(`policy state root is a symlink: ${stateRoot}`); + if (!rootStat.isDirectory()) + throw new Error(`policy state root is not a directory: ${stateRoot}`); + const rootReal = realpathSync(stateRoot); + const entries: StateEntry[] = []; + let totalBytes = 0; + + const visit = (directory: string): void => { + const directoryStat = lstatSync(directory); + if (directoryStat.isSymbolicLink()) + throw new Error(`policy state contains symlink: ${directory}`); + if (!directoryStat.isDirectory()) { + throw new Error(`policy state contains special directory entry: ${directory}`); + } + const directoryReal = realpathSync(directory); + if (!isContained(rootReal, directoryReal)) { + throw new Error(`policy state directory escapes root: ${directory}`); + } + const names = readdirSync(directory).sort((a, b) => a.localeCompare(b)); + for (const name of names) { + const path = join(directory, name); + const stat = lstatSync(path); + if (stat.isSymbolicLink()) throw new Error(`policy state contains symlink: ${path}`); + if (stat.isDirectory()) { + visit(path); + continue; + } + if (!stat.isFile()) throw new Error(`policy state contains special file: ${path}`); + const rel = relative(rootReal, realpathSync(path)).split(sep).join("/"); + validateRelativeStatePath(rel); + const bytes = readStableFile(path); + totalBytes += bytes.length; + if (entries.length + 1 > STATE_MAX_FILES) throw new Error("policy state exceeds file limit"); + if (totalBytes > STATE_MAX_TOTAL_BYTES) throw new Error("policy state exceeds byte limit"); + entries.push({ path: rel, size: bytes.length, sha256: sha256(bytes), bytes }); + } + }; + + visit(rootReal); + return entries.sort((a, b) => a.path.localeCompare(b.path)); +} + +function stateDigest(entries: StateEntry[]): string { + return sha256( + canonicalJson( + entries.map(({ path, size, sha256: contentSha256 }) => ({ + path, + size, + sha256: contentSha256, + })), + ), + ); +} + +export function digestPolicyState(stateRoot: string): string { + return stateDigest(collectStateEntries(stateRoot)); +} + +function exactDirectory(path: string): string { + const stat = lstatSync(path); + if (stat.isSymbolicLink() || !stat.isDirectory()) { + throw new Error(`expected an ordinary directory: ${path}`); + } + return realpathSync(path); +} + +function copyEntries(entries: StateEntry[], destinationRoot: string): void { + mkdirSync(destinationRoot, { recursive: true, mode: 0o700 }); + const destinationReal = realpathSync(destinationRoot); + for (const entry of entries) { + validateRelativeStatePath(entry.path); + const destination = resolve(destinationReal, entry.path); + if (!isContained(destinationReal, destination)) { + throw new Error(`policy state copy escapes destination: ${entry.path}`); + } + mkdirSync(dirname(destination), { recursive: true, mode: 0o700 }); + writeFileSync(destination, entry.bytes, { flag: "wx", mode: 0o600 }); + const copied = lstatSync(destination); + if (!copied.isFile() || copied.isSymbolicLink() || copied.nlink !== 1) { + throw new Error(`policy state copy did not produce a private file: ${entry.path}`); + } + } +} + +function sameStateEntry(left: StateEntry | undefined, right: StateEntry | undefined): boolean { + return left?.path === right?.path && left?.size === right?.size && left?.sha256 === right?.sha256; +} + +/** + * Apply the recorded production-state transition without erasing branch-local writes. + * + * This is a file-level three-way merge: previous capture is the base, next capture supplies + * exogenous production changes, and a branch-local change wins a same-file conflict. Store APIs + * still own the bytes and schema; replay does not model or reinterpret any learning format. + */ +function advanceBranchState(input: { + checkoutRoot: string; + previousStateSnapshotRoot: string; + nextStateSnapshotRoot: string; +}): void { + const previous = new Map( + collectStateEntries(input.previousStateSnapshotRoot).map((entry) => [entry.path, entry]), + ); + const next = new Map( + collectStateEntries(input.nextStateSnapshotRoot).map((entry) => [entry.path, entry]), + ); + const branchStateRoot = join(input.checkoutRoot, ".reviewgate"); + const branch = new Map(collectStateEntries(branchStateRoot).map((entry) => [entry.path, entry])); + const paths = [...new Set([...previous.keys(), ...next.keys(), ...branch.keys()])].sort((a, b) => + a.localeCompare(b), + ); + const merged: StateEntry[] = []; + for (const path of paths) { + const baseEntry = previous.get(path); + const nextEntry = next.get(path); + const branchEntry = branch.get(path); + const branchChanged = !sameStateEntry(branchEntry, baseEntry); + const productionChanged = !sameStateEntry(nextEntry, baseEntry); + const selected = + branchChanged && productionChanged && !sameStateEntry(branchEntry, nextEntry) + ? branchEntry + : branchChanged + ? branchEntry + : nextEntry; + if (selected !== undefined) merged.push(selected); + } + rmSync(branchStateRoot, { recursive: true, force: true }); + copyEntries(merged, branchStateRoot); +} + +function applyReplayDiff(input: { + checkoutRoot: string; + replayRoot: string; + exactDiff: string; + reverse: boolean; + label: string; +}): void { + if (input.exactDiff.length === 0) return; + const patchPath = join(input.replayRoot, `.${input.label}.patch`); + writeFileSync(patchPath, input.exactDiff, { flag: "wx", mode: 0o600 }); + try { + execFileSync( + "git", + ["apply", "--whitespace=nowarn", ...(input.reverse ? ["--reverse"] : []), patchPath], + { cwd: input.checkoutRoot, stdio: "pipe" }, + ); + } finally { + rmSync(patchPath, { force: true }); + } +} + +export function createPolicyStateSnapshot(input: { + sourceRepoRoot: string; + outputRoot: string; +}): PolicyStateSnapshot { + const repoReal = exactDirectory(input.sourceRepoRoot); + const outputReal = exactDirectory(input.outputRoot); + if (isContained(repoReal, outputReal) || isContained(outputReal, repoReal)) { + throw new Error("policy state output must be separate from the measured repository"); + } + const sourceState = join(repoReal, ".reviewgate"); + const entries = collectStateEntries(sourceState); + const stateSha256 = stateDigest(entries); + const stateRef = `policy-state/${stateSha256}/.reviewgate`; + if (!STATE_TREE_REF.test(stateRef)) throw new Error("invalid policy state reference"); + const stateDestination = resolve(outputReal, stateRef); + if (!isContained(outputReal, stateDestination)) + throw new Error("policy state output escapes root"); + if (!existsSync(stateDestination)) copyEntries(entries, stateDestination); + if (digestPolicyState(stateDestination) !== stateSha256) { + throw new Error("policy state snapshot digest mismatch"); + } + + const manifest: PolicyStateManifest = { + schema: "reviewgate.policy-state-snapshot.v1", + catalog_version: POLICY_CATALOG_VERSION, + state_sha256: stateSha256, + files: entries.map(({ path, size, sha256: contentSha256 }) => ({ + path, + size, + sha256: contentSha256, + })), + }; + const bytes = canonicalJson(manifest); + const manifestSha256 = sha256(bytes); + const ref = `policy-state/${manifestSha256}.json`; + if (!STATE_MANIFEST_REF.test(ref)) throw new Error("invalid policy state manifest reference"); + const destination = resolve(outputReal, ref); + if (!isContained(outputReal, destination)) throw new Error("policy state manifest escapes root"); + mkdirSync(dirname(destination), { recursive: true, mode: 0o700 }); + if (!writeFileIfAbsent(destination, bytes, { mode: 0o600 })) { + const existing = readStableFile(destination, STATE_MAX_FILE_BYTES, true); + if (!existing.equals(Buffer.from(bytes, "utf8"))) { + throw new Error("policy state manifest collision"); + } + } + return { ref, sha256: manifestSha256, stateRef, stateSha256 }; +} + +export function verifyPolicyStateSnapshot(input: { + outputRoot: string; + ref: string; + sha256: string; + expectedStateSha256: string; +}): { stateRoot: string; stateSha256: string } { + if ( + !STATE_MANIFEST_REF.test(input.ref) || + !/^[0-9a-f]{64}$/.test(input.sha256) || + !/^[0-9a-f]{64}$/.test(input.expectedStateSha256) + ) { + throw new Error("invalid policy state snapshot reference"); + } + const outputReal = exactDirectory(input.outputRoot); + const manifestPath = resolve(outputReal, input.ref); + if (!isContained(outputReal, manifestPath)) throw new Error("policy state manifest escapes root"); + const bytes = readStableFile(manifestPath, STATE_MAX_FILE_BYTES, true); + if (sha256(bytes) !== input.sha256 || input.ref !== `policy-state/${input.sha256}.json`) { + throw new Error("policy state manifest hash mismatch"); + } + let text: string; + try { + text = new TextDecoder("utf-8", { fatal: true }).decode(bytes); + } catch { + throw new Error("policy state manifest has invalid encoding"); + } + let decoded: unknown; + try { + decoded = JSON.parse(text); + } catch { + throw new Error("policy state manifest is corrupt"); + } + const manifest = PolicyStateManifestSchema.parse(decoded); + if (canonicalJson(manifest) !== text) throw new Error("policy state manifest is non-canonical"); + if (manifest.state_sha256 !== input.expectedStateSha256) { + throw new Error("policy state manifest digest mismatch"); + } + const stateRoot = resolve(outputReal, `policy-state/${manifest.state_sha256}/.reviewgate`); + if ( + !isContained(outputReal, stateRoot) || + !STATE_TREE_REF.test(relative(outputReal, stateRoot).split(sep).join("/")) + ) { + throw new Error("policy state tree escapes root"); + } + const entries = collectStateEntries(stateRoot); + if (stateDigest(entries) !== manifest.state_sha256) + throw new Error("policy state tree digest mismatch"); + const actualFiles = entries.map(({ path, size, sha256: contentSha256 }) => ({ + path, + size, + sha256: contentSha256, + })); + if (canonicalJson(actualFiles) !== canonicalJson(manifest.files)) { + throw new Error("policy state tree does not match manifest"); + } + return { stateRoot, stateSha256: manifest.state_sha256 }; +} + +function authority(code: RigAuthorityInvalidity, message: string): never { + throw new RigAuthorityError(code, message); +} + +function responseHashesFromCassette(bytes: Buffer): string[] { + let text: string; + try { + text = new TextDecoder("utf-8", { fatal: true }).decode(bytes); + } catch { + return authority("invalid-cassette", "cassette is not valid UTF-8"); + } + const hashes: string[] = []; + for (const [index, line] of text.split("\n").entries()) { + if (line.trim().length === 0) continue; + let decoded: unknown; + try { + decoded = JSON.parse(line); + } catch { + return authority("invalid-cassette", `cassette line ${index + 1} is not JSON`); + } + const parsed = CassetteEntrySchema.safeParse(decoded); + if (!parsed.success) { + return authority("invalid-cassette", `cassette line ${index + 1} is malformed`); + } + const entry = parsed.data; + let raw: string | undefined; + if (entry.method === "review" && "rawText" in entry.result) raw = entry.result.rawText; + if (entry.method === "complete" && "text" in entry.result) raw = entry.result.text; + if (raw !== undefined) hashes.push(sha256(raw)); + } + return hashes; +} + +/** Validate every artifact before authoritative harvest/replay is allowed to count anything. */ +export function validateRigPolicyReplayArtifacts(input: { + manifest: RigManifest; + manifestPath: string; +}): ValidatedRigPolicyReplay | null { + const metadata = input.manifest.policyReplay; + if (metadata === undefined) return null; + if (metadata.catalogVersion !== POLICY_CATALOG_VERSION) { + return authority( + "catalog-mismatch", + `recorded ${metadata.catalogVersion}, runtime ${POLICY_CATALOG_VERSION}`, + ); + } + + let outputRoot: string; + try { + outputRoot = exactDirectory(dirname(resolve(input.manifestPath))); + } catch (error) { + return authority("invalid-trace", `invalid Rig output root: ${String(error)}`); + } + let initial: { stateRoot: string; stateSha256: string }; + try { + initial = verifyPolicyStateSnapshot({ + outputRoot, + ref: metadata.initialStateRef, + sha256: metadata.initialStateSha256, + expectedStateSha256: metadata.initialStateDigest, + }); + } catch (error) { + return authority("state-digest-mismatch", String(error)); + } + + const cassettePath = resolve(outputRoot, metadata.cassetteRef); + if (!isContained(outputRoot, cassettePath) || !existsSync(cassettePath)) { + return authority("missing-cassette", `missing ${metadata.cassetteRef}`); + } + let cassetteBytes: Buffer; + try { + cassetteBytes = readStableFile(cassettePath, CASSETTE_MAX_BYTES, true); + } catch (error) { + return authority("invalid-cassette", String(error)); + } + if (sha256(cassetteBytes) !== metadata.cassetteSha256) { + return authority("cassette-hash-mismatch", "cassette bytes do not match the manifest"); + } + const cassetteResponseHashes = responseHashesFromCassette(cassetteBytes); + const requiredResponseHashes: string[] = []; + const turns = new Map< + number, + Array<{ ref: string; sha256: string; envelope: PolicyReplayEnvelope; stateRoot: string }> + >(); + const identities = new Set(); + const captureDir = resolve(outputRoot, metadata.captureDir); + if (!isContained(outputRoot, captureDir)) { + return authority("invalid-trace", "capture directory escapes the Rig output root"); + } + + for (const turn of input.manifest.turns) { + const replay = turn.policyReplay; + if (replay === undefined || replay.status === "missing") { + return authority("missing-trace", `turn ${turn.index} has no policy replay trace`); + } + if (replay.status === "overflow") { + return authority("trace-overflow", `turn ${turn.index} capture overflowed`); + } + if (replay.status !== "complete") { + return authority("trace-status", `turn ${turn.index} capture status is ${replay.status}`); + } + const validated: Array<{ + ref: string; + sha256: string; + envelope: PolicyReplayEnvelope; + stateRoot: string; + }> = []; + for (const trace of replay.traces) { + const verified = verifyPolicyReplayEnvelope({ + sinkDir: captureDir, + ref: trace.ref, + sha256: trace.sha256, + authoritative: true, + expectedCatalogVersion: POLICY_CATALOG_VERSION, + }); + if (!verified.ok) { + const code: RigAuthorityInvalidity = + verified.reason === "lossy" + ? "lossy-trace" + : verified.reason === "catalog-mismatch" + ? "catalog-mismatch" + : verified.reason === "non-canonical" + ? "non-canonical-trace" + : verified.reason === "too-large" + ? "trace-overflow" + : "invalid-trace"; + return authority(code, `turn ${turn.index} ${trace.ref}: ${verified.reason}`); + } + const envelope = verified.envelope; + if (envelope.source_commit !== metadata.sourceCommit) { + return authority( + "source-commit-mismatch", + `turn ${turn.index} trace ${trace.ref} names another source commit`, + ); + } + if (envelope.policy_trace.ablated.length > 0) { + return authority("invalid-trace", `turn ${turn.index} baseline trace is already ablated`); + } + const identity = `${envelope.run_id}:${envelope.iter}`; + if (identities.has(identity)) { + return authority("invalid-trace", `duplicate replay identity ${identity}`); + } + identities.add(identity); + const stateRoot = resolve(outputRoot, `policy-state/${envelope.state_sha256}/.reviewgate`); + try { + if ( + !isContained(outputRoot, stateRoot) || + digestPolicyState(stateRoot) !== envelope.state_sha256 + ) { + return authority( + "state-digest-mismatch", + `turn ${turn.index} trace ${trace.ref} state does not match`, + ); + } + } catch (error) { + return authority("state-digest-mismatch", String(error)); + } + requiredResponseHashes.push(...envelope.raw_response_sha256); + validated.push({ ...trace, envelope, stateRoot }); + } + const sequenceRunId = validated[0]?.envelope.run_id; + if ( + sequenceRunId === undefined || + validated.some( + (trace, index) => + trace.envelope.run_id !== sequenceRunId || trace.envelope.iter !== index + 1, + ) + ) { + return authority( + "invalid-trace", + `turn ${turn.index} replay inventory is not one complete ordered iteration sequence`, + ); + } + turns.set(turn.index, validated); + } + + if ( + cassetteResponseHashes.length !== requiredResponseHashes.length || + requiredResponseHashes.some((hash, index) => hash !== cassetteResponseHashes[index]) + ) { + return authority( + "response-hash-mismatch", + "cassette response hashes do not exactly match the captured order", + ); + } + + return { + sourceCommit: metadata.sourceCommit, + initialStateRoot: initial.stateRoot, + initialStateSha256: initial.stateSha256, + cassettePath, + turns, + }; +} + +function assertNoAliasedFiles( + leftRoot: string, + rightRoot: string, + requireEqualDigest = true, +): void { + const left = collectStateEntries(leftRoot); + const right = collectStateEntries(rightRoot); + if (requireEqualDigest && stateDigest(left) !== stateDigest(right)) { + throw new Error("policy state digest mismatch"); + } + const rightPaths = new Set(right.map((entry) => entry.path)); + for (const entry of left) { + if (!rightPaths.has(entry.path)) continue; + const leftStat = statSync(join(leftRoot, entry.path)); + const rightStat = statSync(join(rightRoot, entry.path)); + if (leftStat.dev === rightStat.dev && leftStat.ino === rightStat.ino) { + throw new Error(`policy state source alias detected: ${entry.path}`); + } + } +} + +function prepareBranch(input: { + destination: string; + sourceRepoRoot: string; + sourceCommit: string; + stateSnapshotRoot: string; + expectedStateSha256: string; + exactDiff: string; +}): ReplayBranch { + execFileSync("git", [ + "clone", + "--quiet", + "--no-hardlinks", + "--no-checkout", + input.sourceRepoRoot, + input.destination, + ]); + execFileSync("git", ["checkout", "--quiet", "--detach", input.sourceCommit], { + cwd: input.destination, + }); + const stateDestination = join(input.destination, ".reviewgate"); + if (existsSync(stateDestination)) rmSync(stateDestination, { recursive: true, force: true }); + const entries = collectStateEntries(input.stateSnapshotRoot); + if (stateDigest(entries) !== input.expectedStateSha256) { + throw new Error("policy state snapshot digest mismatch"); + } + copyEntries(entries, stateDestination); + if (digestPolicyState(stateDestination) !== input.expectedStateSha256) { + throw new Error("policy replay branch state digest mismatch"); + } + assertNoAliasedFiles(input.stateSnapshotRoot, stateDestination); + + applyReplayDiff({ + checkoutRoot: input.destination, + replayRoot: dirname(input.destination), + exactDiff: input.exactDiff, + reverse: false, + label: basename(input.destination), + }); + return { checkoutRoot: input.destination, startingStateSha256: input.expectedStateSha256 }; +} + +export function createReplayBranches(input: { + sourceRepoRoot: string; + sourceCommit: string; + stateSnapshotRoot: string; + expectedStateSha256: string; + exactDiff: string; +}): ReplayBranches { + if (!GIT_OBJECT_ID.test(input.sourceCommit)) throw new Error("invalid source commit"); + if (!/^[0-9a-f]{64}$/.test(input.expectedStateSha256)) { + throw new Error("invalid expected policy state digest"); + } + const sourceReal = exactDirectory(input.sourceRepoRoot); + const stateReal = exactDirectory(input.stateSnapshotRoot); + if (isContained(sourceReal, stateReal) || isContained(stateReal, sourceReal)) { + throw new Error("policy state snapshot must not alias the measured repository"); + } + const resolvedCommit = execFileSync( + "git", + ["rev-parse", "--verify", `${input.sourceCommit}^{commit}`], + { + cwd: sourceReal, + encoding: "utf8", + }, + ).trim(); + if (resolvedCommit !== input.sourceCommit) throw new Error("source commit identity mismatch"); + if (digestPolicyState(stateReal) !== input.expectedStateSha256) { + throw new Error("policy state snapshot digest mismatch"); + } + + const root = mkdtempSync(join(tmpdir(), "reviewgate-policy-replay-")); + try { + const baseline = prepareBranch({ + ...input, + sourceRepoRoot: sourceReal, + destination: join(root, "baseline"), + }); + const counterfactual = prepareBranch({ + ...input, + sourceRepoRoot: sourceReal, + destination: join(root, "counterfactual"), + }); + assertNoAliasedFiles( + join(baseline.checkoutRoot, ".reviewgate"), + join(counterfactual.checkoutRoot, ".reviewgate"), + ); + return { root, baseline, counterfactual }; + } catch (error) { + rmSync(root, { recursive: true, force: true }); + throw error; + } +} + +/** Move one persistent replay pair to the next captured iteration. */ +export function advanceReplayBranches(input: { + branches: ReplayBranches; + sourceRepoRoot: string; + sourceCommit: string; + previousExactDiff: string; + nextExactDiff: string; + previousStateSnapshotRoot: string; + previousStateSha256: string; + nextStateSnapshotRoot: string; + nextStateSha256: string; +}): void { + if (!GIT_OBJECT_ID.test(input.sourceCommit)) throw new Error("invalid source commit"); + if ( + !/^[0-9a-f]{64}$/.test(input.previousStateSha256) || + !/^[0-9a-f]{64}$/.test(input.nextStateSha256) + ) { + throw new Error("invalid policy state transition digest"); + } + const sourceReal = exactDirectory(input.sourceRepoRoot); + const replayReal = exactDirectory(input.branches.root); + for (const snapshotRoot of [input.previousStateSnapshotRoot, input.nextStateSnapshotRoot]) { + const snapshotReal = exactDirectory(snapshotRoot); + if ( + isContained(sourceReal, snapshotReal) || + isContained(snapshotReal, sourceReal) || + isContained(replayReal, snapshotReal) || + isContained(snapshotReal, replayReal) + ) { + throw new Error("policy state transition snapshot aliases source or replay state"); + } + } + if (digestPolicyState(input.previousStateSnapshotRoot) !== input.previousStateSha256) { + throw new Error("previous policy state snapshot digest mismatch"); + } + if (digestPolicyState(input.nextStateSnapshotRoot) !== input.nextStateSha256) { + throw new Error("next policy state snapshot digest mismatch"); + } + for (const [label, branch] of [ + ["baseline", input.branches.baseline], + ["counterfactual", input.branches.counterfactual], + ] as const) { + const head = execFileSync("git", ["rev-parse", "HEAD"], { + cwd: branch.checkoutRoot, + encoding: "utf8", + }).trim(); + if (head !== input.sourceCommit) throw new Error(`${label} source commit drifted`); + applyReplayDiff({ + checkoutRoot: branch.checkoutRoot, + replayRoot: input.branches.root, + exactDiff: input.previousExactDiff, + reverse: true, + label: `${label}-previous`, + }); + applyReplayDiff({ + checkoutRoot: branch.checkoutRoot, + replayRoot: input.branches.root, + exactDiff: input.nextExactDiff, + reverse: false, + label: `${label}-next`, + }); + advanceBranchState({ + checkoutRoot: branch.checkoutRoot, + previousStateSnapshotRoot: input.previousStateSnapshotRoot, + nextStateSnapshotRoot: input.nextStateSnapshotRoot, + }); + } + if ( + digestPolicyState(join(input.branches.baseline.checkoutRoot, ".reviewgate")) !== + input.nextStateSha256 + ) { + throw new Error("baseline replay state does not reproduce the next captured digest"); + } + assertNoAliasedFiles( + join(input.branches.baseline.checkoutRoot, ".reviewgate"), + join(input.branches.counterfactual.checkoutRoot, ".reviewgate"), + false, + ); +} + +export function cleanupReplayBranches(branches: ReplayBranches): void { + const temporaryRoot = realpathSync(tmpdir()); + const unresolved = resolve(branches.root); + const stat = lstatSync(unresolved); + if (stat.isSymbolicLink() || !stat.isDirectory()) { + throw new Error("refusing to remove non-directory replay branch root"); + } + const candidate = realpathSync(unresolved); + if ( + !isContained(temporaryRoot, candidate) || + dirname(candidate) !== temporaryRoot || + !basename(candidate).startsWith("reviewgate-policy-replay-") + ) { + throw new Error("refusing to remove untrusted replay branch root"); + } + rmSync(candidate, { recursive: true, force: true }); +} diff --git a/src/rig/replay.ts b/src/rig/replay.ts index 1da1ba6..a7f5482 100644 --- a/src/rig/replay.ts +++ b/src/rig/replay.ts @@ -1,35 +1,33 @@ // src/rig/replay.ts -// `reviewgate rig replay` — a self-check of the HARNESS, never a counterfactual. -// -// WHAT THIS IS NOT, and why the plan's original shape could not be built. -// -// Task 5 Step 4 specified re-running the gate pipeline under `REVIEWGATE_CASSETTE=replay:…` -// with `ReplayAdapter` in strict mode, then asserting two RigResults match. That is not -// implementable against any run recorded so far, and the reason is worth writing down so -// nobody re-attempts it and concludes the cassette is broken: -// -// * a pipeline re-run has to BUILD each reviewer prompt, which needs the diff the gate -// reviewed at that iteration; -// * the run recorded `.reviewgate/` snapshots and the cassette, and the cassette stores -// only `promptSha256`, never the prompt text (deliberately — it is already a -// secret-leak-at-rest surface); -// * so the prompts strict mode compares against cannot be reconstructed, and strict mode -// would report drift on every entry: a loud failure that means nothing. -// -// Per-turn `diff.patch` recording (driver, 2026-08-05) is the first half of closing that gap; -// a future run that also records per-ITERATION prompts could implement the literal spec. -// -// WHAT THIS IS: the acceptance test the aggregator refactor actually needs. That refactor -// must be behaviour-neutral, and "behaviour" here means the numbers a recorded run yields. -// Both paths this checks are PURE functions of on-disk artifacts — `harvest()` over the -// snapshots and `ablate()` over the harvested findings — so any difference across two runs -// is nondeterminism in our own code (Map/readdir ordering, a stray Date, a mutated input), -// which is exactly what would make a refactor's "no change" claim unfalsifiable. +// Exact new runs replay captured policy inputs through production pass functions in isolated +// checkouts. Legacy runs retain the older deterministic harvest/heuristic self-check, explicitly +// non-authoritative for policy ablation rather than pretending missing opportunities were zero. import { existsSync, readFileSync } from "node:fs"; +import { canonicalJson } from "../audit/canonical.ts"; +import { type AggregateInput, aggregate } from "../core/aggregator.ts"; +import { validateFindingFacts } from "../core/fact-check.ts"; +import { applyGroundingJudgeVerdicts, groundFindings } from "../core/grounding.ts"; +import { demoteHypotheticalCriticals } from "../core/hypothetical-demote.ts"; +import type { PolicyPassId } from "../core/policy/catalog.ts"; +import { POLICY_PASS_IDS } from "../core/policy/catalog.ts"; +import { PolicyTraceRecorder } from "../core/policy/trace.ts"; +import { demoteSelfRefuting } from "../core/self-refutation.ts"; +import { parseDeletedPaths } from "../diff/hunks.ts"; import { CassetteEntrySchema } from "../schemas/cassette.ts"; +import type { PolicyReplayEnvelope } from "../schemas/policy-replay.ts"; +import type { PolicyTrace } from "../schemas/policy-trace.ts"; +import { RigManifestSchema } from "../schemas/rig-manifest.ts"; import type { RigResult } from "../schemas/rig-result.ts"; import { type RigAblation, SUPPRESSION_LAYERS, ablate, seededTagsFromScript } from "./ablate.ts"; import { harvest } from "./harvest.ts"; +import { + type ReplayBranches, + RigAuthorityError, + advanceReplayBranches, + cleanupReplayBranches, + createReplayBranches, + validateRigPolicyReplayArtifacts, +} from "./policy-replay-state.ts"; export interface CassetteIntegrity { entries: number; @@ -48,6 +46,426 @@ export interface ReplayReport { differences: string[]; cassette: CassetteIntegrity | null; turns: number; + policy?: { + authoritative: boolean; + envelopes: number; + passIds: PolicyPassId[]; + }; +} + +export interface PolicyReplayPair { + baseline: PolicyTrace; + counterfactual: PolicyTrace; +} + +export interface PolicyReplaySequenceItem { + envelope: PolicyReplayEnvelope; + stateSnapshotRoot: string; +} + +export interface PolicyReplaySequenceStep { + index: number; + item: PolicyReplaySequenceItem; + branches: ReplayBranches; + pair: PolicyReplayPair; +} + +export interface RigPolicyAblationRow { + passId: PolicyPassId; + authoritative: boolean; + reason: string | null; + envelopes: number; + opportunities: number; + applied: number; + wouldApplyWithoutMutation: number; + baselineBlocking: number; + counterfactualBlocking: number; +} + +/** + * Hard offline ceiling around the synchronous production-policy replay. + * + * Replay has no adapter parameter, but this second boundary converts an accidental network or + * Bun subprocess call added inside a policy pass into the typed authority failure promised by the + * CLI. The originals are restored even when the attempted call throws. + */ +export function runWithReplayProviderCeiling(operation: () => T): T { + const bunRuntime = Bun as unknown as { spawn: (...args: unknown[]) => unknown }; + const originalFetch = globalThis.fetch; + const originalSpawn = bunRuntime.spawn; + const reject = (capability: string): never => { + throw new RigAuthorityError( + "live-provider-call", + `authoritative policy replay attempted ${capability}`, + ); + }; + globalThis.fetch = (() => reject("a network call")) as unknown as typeof fetch; + bunRuntime.spawn = () => reject("a provider subprocess call"); + try { + return operation(); + } finally { + globalThis.fetch = originalFetch; + bunRuntime.spawn = originalSpawn; + } +} + +function aggregateInputFromEnvelope( + envelope: PolicyReplayEnvelope, + findings: PolicyReplayEnvelope["aggregate"]["findings"], + runtime: PolicyTraceRecorder, +): AggregateInput { + const aggregate = envelope.aggregate; + const policyInactive = Object.fromEntries( + aggregate.policy_inactive.map((entry) => [entry.pass_id, entry.reason_code]), + ) as NonNullable; + return { + findings, + reviewersTotal: aggregate.reviewers_total, + changedRanges: new Map( + aggregate.changed_ranges.map((entry) => [ + entry.file, + entry.ranges.map((range) => [range.start, range.end] as [number, number]), + ]), + ), + scopeToDiff: aggregate.scope_to_diff, + outOfDiffBlocking: [...aggregate.out_of_diff_blocking], + confidenceFloor: aggregate.confidence_floor, + demoteCorrectness: aggregate.demote_correctness, + corroborateCritical: aggregate.corroborate_critical, + demoteTestSecurity: aggregate.demote_test_security, + capDocsSeverity: aggregate.cap_docs_severity, + critic: new Map( + aggregate.critic.map(({ signature, verdict, reason }) => [ + signature, + { verdict, ...(reason === undefined ? {} : { reason }) }, + ]), + ), + fpActive: new Map(aggregate.fp_active.map(({ signature, id }) => [signature, { id }])), + fpActiveClusters: new Map( + aggregate.fp_active_clusters.map(({ key, member_ids }) => [ + key, + { key, member_ids: [...member_ids] }, + ]), + ), + repUnreliable: new Set(aggregate.rep_unreliable), + protectedReviewers: new Set(aggregate.protected_reviewers), + foreignFiles: new Set(aggregate.foreign_files), + cycleRejected: new Set(aggregate.cycle_rejected), + claimedFixed: new Map(aggregate.claimed_fixed.map(({ signature, iter }) => [signature, iter])), + deltaScope: new Set(aggregate.delta_scope), + rejectedRegions: structuredClone(aggregate.rejected_regions), + policyRuntime: runtime, + policyInactive, + }; +} + +function replayEnvelopeProductionPath(input: { + envelope: PolicyReplayEnvelope; + checkoutRoot: string; + ablated: ReadonlySet; + verifyOriginal: boolean; +}): PolicyTrace { + const { envelope } = input; + const runtime = PolicyTraceRecorder.start({ + runId: envelope.run_id, + iter: envelope.iter, + ablated: input.ablated, + }); + const factChecked = validateFindingFacts( + structuredClone(envelope.pre_policy_findings), + input.checkoutRoot, + parseDeletedPaths(envelope.exact_diff), + runtime, + ); + if (!envelope.pre_policy.self_refutation_enabled) { + runtime.markInactive("evidence.self-refutation", "configured-off"); + } + const selfScreened = demoteSelfRefuting( + factChecked, + envelope.pre_policy.self_refutation_enabled, + runtime, + ); + if (!envelope.pre_policy.hypothetical_enabled) { + runtime.markInactive("judgment.hypothetical", "configured-off"); + } + const hypothetical = demoteHypotheticalCriticals( + selfScreened, + envelope.pre_policy.hypothetical_enabled, + runtime, + ); + let grounded = groundFindings(hypothetical, envelope.grounding.corpus, runtime); + const groundingSummary = envelope.policy_trace.passes.find( + (pass) => pass.pass_id === "judgment.grounding-llm", + ); + if (groundingSummary?.status === "not-run") { + const reason = groundingSummary.reason_code; + if (reason !== "configured-off" && reason !== "stage-precondition-miss") { + throw new Error("captured grounding inactivity reason is invalid"); + } + runtime.markInactive("judgment.grounding-llm", reason); + } else { + grounded = applyGroundingJudgeVerdicts( + grounded, + new Map( + envelope.grounding.verdicts.map(({ signature, grounded: isGrounded, reason }) => [ + signature, + { grounded: isGrounded, ...(reason === undefined ? {} : { reason }) }, + ]), + ), + runtime, + ); + } + if (canonicalJson(grounded) !== canonicalJson(envelope.aggregate.findings)) { + throw new Error("captured aggregate findings do not match production pre-policy replay"); + } + const result = aggregate(aggregateInputFromEnvelope(envelope, grounded, runtime)); + if ( + input.verifyOriginal && + canonicalJson(result.dedupedFindings) !== canonicalJson(envelope.policy_final_findings) + ) { + throw new Error("captured policy output does not match production aggregate replay"); + } + const trace = runtime.finalize({ + rawResponseSha256: [...envelope.raw_response_sha256], + verdict: result.verdict, + finalFindings: result.dedupedFindings, + }); + if (trace === null) throw new Error("production replay trace instrumentation failed"); + if (input.verifyOriginal) { + const original = envelope.policy_trace; + const expectedPolicySignatures = original.final.finding_signatures.slice( + 0, + trace.final.finding_signatures.length, + ); + const expectedPolicySeverities = original.final.finding_severities.slice( + 0, + trace.final.finding_severities.length, + ); + const additiveSeverities = original.final.finding_severities.slice( + trace.final.finding_severities.length, + ); + const expectedPolicyCounts = { + critical: original.final.counts.critical, + warn: original.final.counts.warn, + info: original.final.counts.info - additiveSeverities.length, + }; + const equal = + canonicalJson(trace.raw_response_sha256) === canonicalJson(original.raw_response_sha256) && + canonicalJson(trace.ablated) === canonicalJson(original.ablated) && + canonicalJson(trace.passes) === canonicalJson(original.passes) && + canonicalJson(trace.evaluations) === canonicalJson(original.evaluations) && + canonicalJson(trace.stages) === canonicalJson(original.stages) && + trace.final.verdict === original.final.verdict && + additiveSeverities.every((finding) => finding.severity === "INFO") && + canonicalJson(trace.final.counts) === canonicalJson(expectedPolicyCounts) && + canonicalJson(trace.final.finding_signatures) === canonicalJson(expectedPolicySignatures) && + canonicalJson(trace.final.finding_severities) === canonicalJson(expectedPolicySeverities); + if (!equal) throw new Error("production baseline replay does not reproduce its policy trace"); + } + return trace; +} + +function replayPolicyEnvelopeInBranches(input: { + envelope: PolicyReplayEnvelope; + passId: PolicyPassId; + branches: ReplayBranches; +}): PolicyReplayPair { + return runWithReplayProviderCeiling(() => { + const baseline = replayEnvelopeProductionPath({ + envelope: input.envelope, + checkoutRoot: input.branches.baseline.checkoutRoot, + ablated: new Set(), + verifyOriginal: true, + }); + const counterfactual = replayEnvelopeProductionPath({ + envelope: input.envelope, + checkoutRoot: input.branches.counterfactual.checkoutRoot, + ablated: new Set([input.passId]), + verifyOriginal: false, + }); + if ( + canonicalJson(baseline.raw_response_sha256) !== + canonicalJson(counterfactual.raw_response_sha256) + ) { + throw new Error("baseline and counterfactual ordered response hashes differ"); + } + return { baseline, counterfactual }; + }); +} + +/** One exact baseline/counterfactual pair. No adapter/provider capability enters this API. */ +export function replayPolicyEnvelopePair(input: { + sourceRepoRoot: string; + envelope: PolicyReplayEnvelope; + stateSnapshotRoot: string; + passId: PolicyPassId; +}): PolicyReplayPair { + const branches = createReplayBranches({ + sourceRepoRoot: input.sourceRepoRoot, + sourceCommit: input.envelope.source_commit, + stateSnapshotRoot: input.stateSnapshotRoot, + expectedStateSha256: input.envelope.state_sha256, + exactDiff: input.envelope.exact_diff, + }); + try { + return replayPolicyEnvelopeInBranches({ + envelope: input.envelope, + passId: input.passId, + branches, + }); + } finally { + cleanupReplayBranches(branches); + } +} + +/** + * Replay an ordered multi-turn sequence in one persistent branch pair. The optional callback is + * the branch-local boundary used by existing production Store APIs; replay itself never invents a + * store transition or interprets a learning schema. + */ +export async function replayPolicyEnvelopeSequence(input: { + sourceRepoRoot: string; + items: PolicyReplaySequenceItem[]; + passId: PolicyPassId; + afterEnvelope?: (step: PolicyReplaySequenceStep) => void | Promise; +}): Promise { + const first = input.items[0]; + if (first === undefined) { + throw new RigAuthorityError("missing-trace", "policy replay sequence is empty"); + } + if (input.items.some((item) => item.envelope.source_commit !== first.envelope.source_commit)) { + throw new RigAuthorityError( + "source-commit-mismatch", + "policy replay sequence crosses source commits", + ); + } + const branches = createReplayBranches({ + sourceRepoRoot: input.sourceRepoRoot, + sourceCommit: first.envelope.source_commit, + stateSnapshotRoot: first.stateSnapshotRoot, + expectedStateSha256: first.envelope.state_sha256, + exactDiff: first.envelope.exact_diff, + }); + const pairs: PolicyReplayPair[] = []; + try { + for (const [index, item] of input.items.entries()) { + const previous = input.items[index - 1]; + if (previous !== undefined) { + advanceReplayBranches({ + branches, + sourceRepoRoot: input.sourceRepoRoot, + sourceCommit: item.envelope.source_commit, + previousExactDiff: previous.envelope.exact_diff, + nextExactDiff: item.envelope.exact_diff, + previousStateSnapshotRoot: previous.stateSnapshotRoot, + previousStateSha256: previous.envelope.state_sha256, + nextStateSnapshotRoot: item.stateSnapshotRoot, + nextStateSha256: item.envelope.state_sha256, + }); + } + const pair = replayPolicyEnvelopeInBranches({ + envelope: item.envelope, + passId: input.passId, + branches, + }); + pairs.push(pair); + await input.afterEnvelope?.({ index, item, branches, pair }); + } + return pairs; + } finally { + cleanupReplayBranches(branches); + } +} + +export async function replayPolicyAblations(input: { + manifestPath: string; + sourceRepoRoot: string; +}): Promise { + const manifest = RigManifestSchema.parse( + JSON.parse(readFileSync(input.manifestPath, "utf8")) as unknown, + ); + const validated = validateRigPolicyReplayArtifacts({ + manifest, + manifestPath: input.manifestPath, + }); + if (validated === null) { + throw new RigAuthorityError( + "missing-trace", + "legacy four-layer analysis has no exact policy replay envelopes", + ); + } + const envelopes = [...validated.turns.values()].flat(); + const items = envelopes.map(({ envelope, stateRoot }) => ({ + envelope, + stateSnapshotRoot: stateRoot, + })); + const rows: RigPolicyAblationRow[] = []; + for (const passId of POLICY_PASS_IDS) { + let opportunities = 0; + let applied = 0; + let wouldApplyWithoutMutation = 0; + let baselineBlocking = 0; + let counterfactualBlocking = 0; + let ran = 0; + let pairs: PolicyReplayPair[]; + try { + pairs = await replayPolicyEnvelopeSequence({ + sourceRepoRoot: input.sourceRepoRoot, + items, + passId, + }); + } catch (error) { + if (error instanceof RigAuthorityError) throw error; + throw new RigAuthorityError( + /alias/i.test(String(error)) ? "source-state-alias" : "invalid-trace", + error instanceof Error ? error.message : String(error), + ); + } + for (const pair of pairs) { + const baselineRow = pair.baseline.passes.find((row) => row.pass_id === passId); + const counterfactualRow = pair.counterfactual.passes.find((row) => row.pass_id === passId); + if (baselineRow === undefined || counterfactualRow === undefined) { + throw new RigAuthorityError("invalid-trace", `missing catalog row ${passId}`); + } + if (baselineRow.status === "ran" && counterfactualRow.status === "ran") { + ran += 1; + opportunities += baselineRow.opportunities; + applied += baselineRow.applied; + wouldApplyWithoutMutation += counterfactualRow.would_apply; + } + baselineBlocking += pair.baseline.final.counts.critical + pair.baseline.final.counts.warn; + counterfactualBlocking += + pair.counterfactual.final.counts.critical + pair.counterfactual.final.counts.warn; + } + rows.push({ + passId, + authoritative: ran > 0, + reason: ran > 0 ? null : "pass was inactive for every captured envelope", + envelopes: envelopes.length, + opportunities, + applied, + wouldApplyWithoutMutation, + baselineBlocking, + counterfactualBlocking, + }); + } + return rows; +} + +export function renderPolicyAblationRows(rows: RigPolicyAblationRow[]): string { + const lines = ["Reviewgate rig — exact policy ablation (closed catalog)", ""]; + for (const row of rows) { + const delta = row.counterfactualBlocking - row.baselineBlocking; + lines.push( + ` ${row.passId.padEnd(30)} opportunities ${String(row.opportunities).padStart(4)} applied ${String(row.applied).padStart(4)} blocking Δ ${delta >= 0 ? "+" : ""}${delta} ${row.authoritative ? "exact" : "inactive/non-authoritative"}`, + ); + } + lines.push( + "", + "Lore is excluded: it is additive and verdict-neutral. Legacy critic/reputation/fp-ledger/lore", + "rows remain diagnostic only and are never interpreted as zero policy opportunities.", + ); + return `${lines.join("\n")}\n`; } /** @@ -166,25 +584,29 @@ export function renderReplayReport(r: ReplayReport): string { lines.push(` ⚠ malformed lines : ${c.malformedLines} — the recording is incomplete`); } } + if (r.policy) { + lines.push( + ` exact policy replay : ${r.policy.authoritative ? `AUTHORITATIVE (${r.policy.envelopes} envelope(s), ${r.policy.passIds.length} catalog passes)` : "LEGACY / NON-AUTHORITATIVE"}`, + ); + } if (!r.deterministic) { lines.push("", "Differences:"); for (const d of r.differences) lines.push(` · ${d}`); } lines.push( "", - "This checks the HARNESS, not the gate: it re-derives the metrics from recorded artifacts", - "twice and asserts they match. It is NOT a counterfactual and never re-drives the agent.", - "A true pipeline replay additionally needs per-iteration reviewer prompts, which no run has", - "recorded yet (the cassette stores only their SHA-256).", + "New exact runs replay policy locally from validated envelopes and never invoke a provider.", + "Legacy runs only re-derive the old harvest/heuristic analysis and are non-authoritative.", ); return lines.join("\n"); } -export function replay(input: { +export async function replay(input: { manifestPath: string; scriptPath: string; cassettePath?: string | undefined; -}): ReplayReport { + sourceRepoRoot?: string | undefined; +}): Promise { const report = checkDeterminism(input.manifestPath, input.scriptPath); if (input.cassettePath !== undefined) { if (!existsSync(input.cassettePath)) { @@ -194,5 +616,44 @@ export function replay(input: { } report.cassette = checkCassette(input.cassettePath); } + const manifest = RigManifestSchema.parse( + JSON.parse(readFileSync(input.manifestPath, "utf8")) as unknown, + ); + const policy = validateRigPolicyReplayArtifacts({ manifest, manifestPath: input.manifestPath }); + if (policy !== null) { + if (input.sourceRepoRoot === undefined) { + throw new RigAuthorityError( + "source-state-alias", + "exact policy replay requires the measured source repository root", + ); + } + let envelopeCount = 0; + try { + const items = [...policy.turns.values()].flat().map(({ envelope, stateRoot }) => ({ + envelope, + stateSnapshotRoot: stateRoot, + })); + await replayPolicyEnvelopeSequence({ + sourceRepoRoot: input.sourceRepoRoot, + items, + passId: POLICY_PASS_IDS[0], + }); + envelopeCount = items.length; + } catch (error) { + if (error instanceof RigAuthorityError) throw error; + const message = error instanceof Error ? error.message : String(error); + throw new RigAuthorityError( + /alias/i.test(message) ? "source-state-alias" : "invalid-trace", + message, + ); + } + report.policy = { + authoritative: true, + envelopes: envelopeCount, + passIds: [...POLICY_PASS_IDS], + }; + } else { + report.policy = { authoritative: false, envelopes: 0, passIds: [] }; + } return report; } diff --git a/src/schemas/policy-replay.ts b/src/schemas/policy-replay.ts new file mode 100644 index 0000000..00667d0 --- /dev/null +++ b/src/schemas/policy-replay.ts @@ -0,0 +1,271 @@ +import { z } from "zod"; +import { + POLICY_CATALOG_VERSION, + POLICY_PASS_IDS, + POLICY_REASON_CODES, +} from "../core/policy/catalog.ts"; +import { compareCodeUnits } from "../utils/compare.ts"; +import { isAuthoritativeThrowableString } from "./bench-result.ts"; +import { FindingCategory, FindingSchema } from "./finding.ts"; +import { PolicyTraceSchema } from "./policy-trace.ts"; + +const Sha256Schema = z.string().regex(/^[0-9a-f]{64}$/); +const GitObjectIdSchema = z.string().regex(/^(?:[0-9a-f]{40}|[0-9a-f]{64})$/); +const PolicyPassIdSchema = z.enum(POLICY_PASS_IDS); +const PolicyInactiveReasonSchema = z.enum(["configured-off", "stage-precondition-miss"]); + +const ChangedRangeSchema = z + .object({ start: z.number().int().nonnegative(), end: z.number().int().positive() }) + .strict() + .refine((value) => value.end > value.start, "range end must be greater than start"); + +const ChangedRangesSchema = z + .array( + z + .object({ + file: z.string().min(1), + ranges: z.array(ChangedRangeSchema), + }) + .strict() + .superRefine((value, ctx) => { + for (let index = 1; index < value.ranges.length; index += 1) { + const previous = value.ranges[index - 1]; + const current = value.ranges[index]; + if ( + previous !== undefined && + current !== undefined && + (previous.start > current.start || + (previous.start === current.start && previous.end >= current.end)) + ) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + path: ["ranges", index], + message: "changed ranges must be uniquely sorted", + }); + } + } + }), + ) + .superRefine((value, ctx) => { + for (let index = 1; index < value.length; index += 1) { + if (compareCodeUnits(value[index - 1]?.file ?? "", value[index]?.file ?? "") >= 0) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + path: [index, "file"], + message: "changed-range files must be uniquely sorted", + }); + } + } + }); + +function uniquelySortedBy( + values: T[], + key: (value: T) => string, + ctx: z.RefinementCtx, + path: Array, +): void { + for (let index = 1; index < values.length; index += 1) { + if (compareCodeUnits(key(values[index - 1] as T), key(values[index] as T)) >= 0) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + path: [...path, index], + message: "entries must be uniquely sorted", + }); + } + } +} + +const CriticEntrySchema = z + .object({ + signature: z.string().min(1), + verdict: z.enum(["keep", "likely_fp"]), + reason: z.string().optional(), + }) + .strict(); + +const AggregateInputsSchema = z + .object({ + findings: z.array(FindingSchema), + reviewers_total: z.number().int().nonnegative(), + changed_ranges: ChangedRangesSchema, + scope_to_diff: z.boolean(), + out_of_diff_blocking: z.array(FindingCategory), + confidence_floor: z.number().min(0).max(1), + demote_correctness: z.boolean(), + corroborate_critical: z.boolean(), + demote_test_security: z.boolean(), + cap_docs_severity: z.boolean(), + critic: z.array(CriticEntrySchema), + fp_active: z.array(z.object({ signature: z.string(), id: z.string() }).strict()), + fp_active_clusters: z.array( + z + .object({ key: z.string(), member_ids: z.array(z.string()) }) + .strict() + .superRefine((value, ctx) => uniquelySortedBy(value.member_ids, (entry) => entry, ctx, [])), + ), + rep_unreliable: z.array(z.string()), + protected_reviewers: z.array(z.string()), + foreign_files: z.array(z.string()), + cycle_rejected: z.array(z.string()), + claimed_fixed: z.array( + z.object({ signature: z.string(), iter: z.number().int().positive() }).strict(), + ), + delta_scope: z.array(z.string()), + rejected_regions: z.array( + z + .object({ + file: z.string(), + start_line: z.number().int().positive(), + end_line: z.number().int().positive(), + severity: z.enum(["CRITICAL", "WARN", "INFO"]), + categories: z.array(FindingCategory), + reason: z.string(), + distinct_count: z.number().int().positive(), + }) + .strict(), + ), + policy_inactive: z.array( + z.object({ pass_id: PolicyPassIdSchema, reason_code: PolicyInactiveReasonSchema }).strict(), + ), + }) + .strict() + .superRefine((value, ctx) => { + uniquelySortedBy(value.critic, (entry) => entry.signature, ctx, ["critic"]); + uniquelySortedBy(value.fp_active, (entry) => entry.signature, ctx, ["fp_active"]); + uniquelySortedBy(value.fp_active_clusters, (entry) => entry.key, ctx, ["fp_active_clusters"]); + uniquelySortedBy(value.rep_unreliable, (entry) => entry, ctx, ["rep_unreliable"]); + uniquelySortedBy(value.protected_reviewers, (entry) => entry, ctx, ["protected_reviewers"]); + uniquelySortedBy(value.foreign_files, (entry) => entry, ctx, ["foreign_files"]); + uniquelySortedBy(value.cycle_rejected, (entry) => entry, ctx, ["cycle_rejected"]); + uniquelySortedBy(value.claimed_fixed, (entry) => entry.signature, ctx, ["claimed_fixed"]); + uniquelySortedBy(value.delta_scope, (entry) => entry, ctx, ["delta_scope"]); + uniquelySortedBy( + value.rejected_regions, + (entry) => + `${entry.file}\u0000${String(entry.start_line).padStart(12, "0")}\u0000${String(entry.end_line).padStart(12, "0")}`, + ctx, + ["rejected_regions"], + ); + uniquelySortedBy(value.policy_inactive, (entry) => entry.pass_id, ctx, ["policy_inactive"]); + }); + +const GroundingVerdictSchema = z + .object({ signature: z.string(), grounded: z.boolean(), reason: z.string().optional() }) + .strict(); + +const PolicyReplayEnvelopeBaseSchema = z + .object({ + schema: z.literal("reviewgate.policy-replay-envelope.v1"), + catalog_version: z.literal(POLICY_CATALOG_VERSION), + run_id: z.string().min(1), + iter: z.number().int().positive(), + source_commit: GitObjectIdSchema, + exact_diff: z.string(), + pre_policy_findings: z.array(FindingSchema), + grounding: z + .object({ + corpus: z.string(), + verdicts: z.array(GroundingVerdictSchema), + llm_status: z.enum(["ran", "not-run", "error"]), + }) + .strict() + .superRefine((value, ctx) => + uniquelySortedBy(value.verdicts, (entry) => entry.signature, ctx, ["verdicts"]), + ), + aggregate: AggregateInputsSchema, + /** Aggregate output before additive Lore findings; Lore remains non-ablatable. */ + policy_final_findings: z.array(FindingSchema), + pre_policy: z + .object({ self_refutation_enabled: z.boolean(), hypothetical_enabled: z.boolean() }) + .strict(), + state_sha256: Sha256Schema, + raw_response_sha256: z.array(Sha256Schema), + /** Original production trace; replay must reproduce it byte-for-byte before ablation. */ + policy_trace: PolicyTraceSchema, + lossless: z.boolean(), + }) + .strict(); + +function visitStrings( + value: unknown, + visit: (value: string, path: Array) => void, +): void { + const walk = (candidate: unknown, path: Array) => { + if (typeof candidate === "string") { + visit(candidate, path); + return; + } + if (Array.isArray(candidate)) { + candidate.forEach((entry, index) => walk(entry, [...path, index])); + return; + } + if (candidate !== null && typeof candidate === "object") { + for (const [key, entry] of Object.entries(candidate as Record)) { + walk(entry, [...path, key]); + } + } + }; + walk(value, []); +} + +export const PolicyReplayEnvelopeSchema = PolicyReplayEnvelopeBaseSchema.superRefine( + (value, ctx) => { + if (value.lossless) { + visitStrings(value, (stringValue, path) => { + if (!isAuthoritativeThrowableString(stringValue)) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + path, + message: "lossless replay envelope contains unsafe string data", + }); + } + }); + } + if (value.grounding.llm_status !== "ran" && value.grounding.verdicts.length > 0) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + path: ["grounding", "verdicts"], + message: "grounding verdicts require a ran LLM stage", + }); + } + if ( + value.policy_trace.run_id !== value.run_id || + value.policy_trace.iter !== value.iter || + value.policy_trace.catalog_version !== value.catalog_version + ) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + path: ["policy_trace"], + message: "policy trace identity must match its replay envelope", + }); + } + if ( + value.raw_response_sha256.length !== value.policy_trace.raw_response_sha256.length || + value.raw_response_sha256.some( + (hash, index) => hash !== value.policy_trace.raw_response_sha256[index], + ) + ) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + path: ["raw_response_sha256"], + message: "ordered response hashes must match the production policy trace", + }); + } + for (const row of value.aggregate.policy_inactive) { + if (!POLICY_REASON_CODES.includes(row.reason_code)) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + path: ["aggregate", "policy_inactive"], + message: "policy inactive reason must be catalogued", + }); + } + } + }, +); + +/** Structural input accepted only at the in-memory capture boundary, before redaction. */ +export const PolicyReplayEnvelopeInputSchema = PolicyReplayEnvelopeBaseSchema; + +export type PolicyReplayEnvelope = z.infer; +export type PolicyReplayEnvelopeInput = z.input; +export type PolicyReplayAggregateInputs = z.infer; diff --git a/src/schemas/rig-manifest.ts b/src/schemas/rig-manifest.ts index 14848ea..5ab4d78 100644 --- a/src/schemas/rig-manifest.ts +++ b/src/schemas/rig-manifest.ts @@ -9,6 +9,29 @@ // the writer and the reader cannot drift apart. import { z } from "zod"; +const Sha256Schema = z.string().regex(/^[0-9a-f]{64}$/); +const GitObjectIdSchema = z.string().regex(/^(?:[0-9a-f]{40}|[0-9a-f]{64})$/); + +export const RigPolicyReplayTraceSchema = z + .object({ + ref: z.string().regex(/^[0-9a-f]{12}-i(?:0|[1-9]\d*)-[0-9a-f]{12}\.json$/), + sha256: Sha256Schema, + }) + .strict(); + +export const RigPolicyReplayMetadataSchema = z + .object({ + catalogVersion: z.string().min(1), + sourceCommit: GitObjectIdSchema, + initialStateRef: z.string().regex(/^policy-state\/[0-9a-f]{64}\.json$/), + initialStateSha256: Sha256Schema, + initialStateDigest: Sha256Schema, + cassetteSha256: Sha256Schema, + cassetteRef: z.literal("cassette.jsonl"), + captureDir: z.literal("policy-replay"), + }) + .strict(); + export const RigManifestTurnSchema = z .object({ index: z.number().int().positive(), @@ -52,6 +75,23 @@ export const RigManifestTurnSchema = z * catch, and the run scored it as a reviewer miss. Optional so older manifests parse. */ diffBytes: z.number().int().nonnegative().nullable().optional(), + /** Exact per-iteration capture artifacts produced while this agent turn ran. */ + policyReplay: z + .object({ + status: z.enum(["complete", "missing", "error", "overflow"]), + traces: z.array(RigPolicyReplayTraceSchema), + }) + .strict() + .superRefine((value, ctx) => { + if ((value.status === "complete") !== value.traces.length > 0) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + path: ["traces"], + message: "complete requires traces and non-complete status forbids them", + }); + } + }) + .optional(), }) .strict(); @@ -63,6 +103,8 @@ export const RigManifestSchema = z outDir: z.string().min(1), /** Cassette the run recorded into, or null when it was not recording. */ cassettePath: z.string().nullable().optional(), + /** Present only for new exact policy-capture runs; absence is legacy/non-authoritative. */ + policyReplay: RigPolicyReplayMetadataSchema.optional(), turns: z.array(RigManifestTurnSchema), }) .strict(); diff --git a/src/schemas/rig-result.ts b/src/schemas/rig-result.ts index ae4ffa2..46a272d 100644 --- a/src/schemas/rig-result.ts +++ b/src/schemas/rig-result.ts @@ -11,6 +11,7 @@ // read as "no false positives on a turn that had findings", a different and flattering // claim. 7 of the 12 pilot turns are clean, so this is the common case. import { z } from "zod"; +import { POLICY_CATALOG_VERSION, POLICY_PASS_IDS } from "../core/policy/catalog.ts"; import { MetricSchema, SpreadStatSchema } from "./bench-result.ts"; import { FindingSchema } from "./finding.ts"; import { CriticInfoSchema } from "./pending-report.ts"; @@ -112,12 +113,49 @@ export const RigTurnRecordSchema = z * `src/rig/ablate.ts` — it is the reason two of the four layers can only be bounded. */ findings: z.array(FindingSchema), + /** Optional so results harvested before exact replay remain parseable. */ + policyReplay: z + .object({ + status: z.enum(["complete", "missing", "error", "overflow", "invalid"]), + traces: z.array( + z + .object({ + ref: z.string().min(1), + sha256: z.string().regex(/^[0-9a-f]{64}$/), + runId: z.string().min(1), + iter: z.number().int().positive(), + stateSha256: z.string().regex(/^[0-9a-f]{64}$/), + lossless: z.boolean(), + }) + .strict(), + ), + reason: z.string().nullable(), + }) + .strict() + .optional(), }) .strict() // The two null contracts, enforced rather than documented: a future edit that computes // `caught` for a clean turn or `0` for a zero-finding turn fails validation here instead of // publishing a number whose meaning silently changed. .superRefine((t, ctx) => { + if (t.policyReplay !== undefined) { + const complete = t.policyReplay.status === "complete"; + if (complete !== (t.policyReplay.traces.length > 0 && t.policyReplay.reason === null)) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + message: "complete replay requires traces and no reason; other statuses require a reason", + path: ["policyReplay"], + }); + } + if (!complete && t.policyReplay.traces.length > 0) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + message: "non-complete replay status cannot carry authoritative traces", + path: ["policyReplay", "traces"], + }); + } + } if ((t.seededId === null) !== (t.caught === null)) { ctx.addIssue({ code: z.ZodIssueCode.custom, @@ -232,6 +270,40 @@ export const RigResultSchema = z provenance: RigProvenanceSchema, turns: z.array(RigTurnRecordSchema), metrics: RigMetricsSchema, + /** Closed policy authority statement. Legacy artifacts omit it and remain parseable. */ + policyReplay: z + .object({ + authoritative: z.boolean(), + catalogVersion: z.literal(POLICY_CATALOG_VERSION).nullable(), + sourceCommit: z + .string() + .regex(/^(?:[0-9a-f]{40}|[0-9a-f]{64})$/) + .nullable(), + passIds: z.array(z.enum(POLICY_PASS_IDS)), + reason: z.string().nullable(), + }) + .strict() + .superRefine((value, ctx) => { + if (value.authoritative !== (value.reason === null)) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + path: ["reason"], + message: "authoritative requires no reason; non-authoritative requires one", + }); + } + if ( + value.authoritative && + (value.passIds.length !== POLICY_PASS_IDS.length || + POLICY_PASS_IDS.some((passId, index) => value.passIds[index] !== passId)) + ) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + path: ["passIds"], + message: "authoritative replay requires the complete policy catalog", + }); + } + }) + .optional(), /** * Everything that failed, was skipped, or could not be read — one line each. * Task 6's honesty rule ("a run with three timed-out turns that reports only the nine diff --git a/tests/unit/cli-required-args.test.ts b/tests/unit/cli-required-args.test.ts index fcf831c..25ced03 100644 --- a/tests/unit/cli-required-args.test.ts +++ b/tests/unit/cli-required-args.test.ts @@ -8,6 +8,8 @@ // marker). The pre-fix manual checks instead printed "... is required" and // exited 2, so the citty message + exit-1 distinguishes fixed from unfixed. import { describe, expect, it } from "bun:test"; +import { mkdtempSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; import { join } from "node:path"; const CLI = join(import.meta.dir, "..", "..", "src", "cli", "index.ts"); @@ -45,3 +47,50 @@ describe("CLI required-arg declarations (F-079)", () => { }); } }); + +describe("Rig authority exit code", () => { + it("maps a typed cross-catalog harvest invalidity to exact exit 4", async () => { + const root = mkdtempSync(join(tmpdir(), "rg-rig-authority-cli-")); + const scriptPath = join(root, "script.json"); + writeFileSync( + scriptPath, + JSON.stringify({ + schema: "reviewgate.rig.turn-script.v1", + id: "authority", + turns: [{ index: 1, prompt: "turn", seeded: null }], + }), + ); + const manifestPath = join(root, "manifest.json"); + writeFileSync( + manifestPath, + JSON.stringify({ + schema: "reviewgate.rig.manifest.v1", + runId: "authority-run", + scriptId: "authority", + outDir: root, + turns: [], + policyReplay: { + catalogVersion: "reviewgate.policy-catalog.future", + sourceCommit: "a".repeat(40), + initialStateRef: `policy-state/${"b".repeat(64)}.json`, + initialStateSha256: "b".repeat(64), + initialStateDigest: "c".repeat(64), + cassetteSha256: "d".repeat(64), + cassetteRef: "cassette.jsonl", + captureDir: "policy-replay", + }, + }), + ); + + const { code, stderr } = await run([ + "rig", + "harvest", + "--manifest", + manifestPath, + "--script", + scriptPath, + ]); + expect(code).toBe(4); + expect(stderr).toContain("catalog-mismatch"); + }); +}); diff --git a/tests/unit/orchestrator-policy-trace.test.ts b/tests/unit/orchestrator-policy-trace.test.ts index e59943d..a61b5d9 100644 --- a/tests/unit/orchestrator-policy-trace.test.ts +++ b/tests/unit/orchestrator-policy-trace.test.ts @@ -56,4 +56,14 @@ describe("policy ablations stay internal", () => { expect(source, path).not.toContain("policyAblations"); } }); + + it("lets Gate read only the Rig capture sink, never a pass or ablation control", () => { + const gate = readFileSync(join(REPO_ROOT, "src/cli/commands/gate.ts"), "utf8"); + const replayEnvReads = [...gate.matchAll(/process\.env\.([A-Z0-9_]*RIG[A-Z0-9_]*)/g)].map( + (match) => match[1], + ); + expect(replayEnvReads).toEqual(["REVIEWGATE_RIG_REPLAY_DIR"]); + expect(gate).not.toMatch(/REVIEWGATE_(?:POLICY_)?ABLATION/); + expect(gate).not.toMatch(/REVIEWGATE_POLICY_PASS/); + }); }); diff --git a/tests/unit/policy-replay-capture.test.ts b/tests/unit/policy-replay-capture.test.ts new file mode 100644 index 0000000..f03ab5e --- /dev/null +++ b/tests/unit/policy-replay-capture.test.ts @@ -0,0 +1,419 @@ +import { describe, expect, test } from "bun:test"; +import { execFileSync } from "node:child_process"; +import { + chmodSync, + existsSync, + linkSync, + lstatSync, + mkdirSync, + mkdtempSync, + readFileSync, + readdirSync, + realpathSync, + symlinkSync, + writeFileSync, +} from "node:fs"; +import { tmpdir } from "node:os"; +import { join, relative } from "node:path"; +import { canonicalJson } from "../../src/audit/canonical.ts"; +import { aggregate } from "../../src/core/aggregator.ts"; +import { POLICY_CATALOG_VERSION } from "../../src/core/policy/catalog.ts"; +import { + capturePolicyReplayEnvelope, + verifyPolicyReplayEnvelope, +} from "../../src/core/policy/replay-capture.ts"; +import { PolicyTraceRecorder } from "../../src/core/policy/trace.ts"; +import { + cleanupReplayBranches, + createPolicyStateSnapshot, + createReplayBranches, + digestPolicyState, +} from "../../src/rig/policy-replay-state.ts"; +import { + type PolicyReplayEnvelope, + PolicyReplayEnvelopeSchema, +} from "../../src/schemas/policy-replay.ts"; + +const H = "a".repeat(64); + +function finding(signature = "sig-a") { + return { + id: "raw-1", + signature, + severity: "WARN" as const, + category: "quality" as const, + rule_id: "safe-rule", + file: "src/example.ts", + line_start: 1, + line_end: 1, + message: "A real issue", + details: "The value is not checked.", + reviewer: { provider: "codex", model: "gpt-5", persona: "correctness" }, + confidence: 0.9, + consensus: "singleton" as const, + }; +} + +function envelope(overrides: Partial = {}): PolicyReplayEnvelope { + const runtime = PolicyTraceRecorder.start({ + runId: "rig-run-1", + iter: 1, + ablated: new Set(), + }); + const aggregateResult = aggregate({ + findings: [], + reviewersTotal: 1, + changedRanges: new Map(), + scopeToDiff: true, + outOfDiffBlocking: [], + confidenceFloor: 0, + demoteCorrectness: true, + corroborateCritical: true, + demoteTestSecurity: true, + capDocsSeverity: true, + policyRuntime: runtime, + }); + const policyTrace = runtime.finalize({ + rawResponseSha256: ["b".repeat(64), "c".repeat(64)], + verdict: aggregateResult.verdict, + finalFindings: aggregateResult.dedupedFindings, + }); + if (policyTrace === null) throw new Error("policy trace fixture failed"); + return PolicyReplayEnvelopeSchema.parse({ + schema: "reviewgate.policy-replay-envelope.v1", + catalog_version: POLICY_CATALOG_VERSION, + run_id: "rig-run-1", + iter: 1, + source_commit: H, + exact_diff: [ + "diff --git a/src/example.ts b/src/example.ts", + "--- a/src/example.ts", + "+++ b/src/example.ts", + "@@ -0,0 +1 @@", + "+export const value = 1;", + "", + ].join("\n"), + pre_policy_findings: [finding()], + grounding: { + corpus: "export const value = 1;", + verdicts: [], + llm_status: "not-run", + }, + aggregate: { + findings: [finding()], + reviewers_total: 1, + changed_ranges: [{ file: "src/example.ts", ranges: [{ start: 1, end: 2 }] }], + scope_to_diff: true, + out_of_diff_blocking: [], + confidence_floor: 0, + demote_correctness: true, + corroborate_critical: true, + demote_test_security: true, + cap_docs_severity: true, + critic: [], + fp_active: [], + fp_active_clusters: [], + rep_unreliable: [], + protected_reviewers: [], + foreign_files: [], + cycle_rejected: [], + claimed_fixed: [], + delta_scope: [], + rejected_regions: [], + policy_inactive: [], + }, + policy_final_findings: [finding()], + pre_policy: { self_refutation_enabled: true, hypothetical_enabled: true }, + state_sha256: H, + raw_response_sha256: ["b".repeat(64), "c".repeat(64)], + policy_trace: policyTrace, + lossless: true, + ...overrides, + }); +} + +function gitRepo(): string { + const root = mkdtempSync(join(tmpdir(), "rg-policy-replay-source-")); + execFileSync("git", ["init", "-q", "."], { cwd: root }); + execFileSync("git", ["config", "user.email", "rig@example.invalid"], { cwd: root }); + execFileSync("git", ["config", "user.name", "rig"], { cwd: root }); + mkdirSync(join(root, "src")); + writeFileSync(join(root, "src", "example.ts"), "export const value = 0;\n"); + mkdirSync(join(root, ".reviewgate", "reputation"), { recursive: true }); + writeFileSync(join(root, ".reviewgate", "fp-ledger.jsonl"), '{"id":"fp-1"}\n'); + writeFileSync(join(root, ".reviewgate", "reputation", "events.jsonl"), '{"tp":1}\n'); + execFileSync("git", ["add", "src/example.ts"], { cwd: root }); + execFileSync("git", ["commit", "-qm", "initial"], { cwd: root }); + return root; +} + +describe("policy replay envelope schema", () => { + test("is strict, closed-catalog, deterministically ordered, and preserves response order", () => { + expect(PolicyReplayEnvelopeSchema.parse(envelope()).raw_response_sha256).toEqual([ + "b".repeat(64), + "c".repeat(64), + ]); + expect(() => + PolicyReplayEnvelopeSchema.parse({ ...envelope(), catalog_version: "future" }), + ).toThrow(/catalog/i); + expect(() => + PolicyReplayEnvelopeSchema.parse({ + ...envelope(), + aggregate: { + ...envelope().aggregate, + changed_ranges: [ + { file: "z.ts", ranges: [{ start: 1, end: 2 }] }, + { file: "a.ts", ranges: [{ start: 1, end: 2 }] }, + ], + }, + }), + ).toThrow(/sort/i); + expect(() => PolicyReplayEnvelopeSchema.parse({ ...envelope(), extra: true })).toThrow(); + expect(() => + PolicyReplayEnvelopeSchema.parse({ + ...envelope(), + raw_response_sha256: ["c".repeat(64), "b".repeat(64)], + }), + ).toThrow(/ordered response hashes/i); + }); +}); + +describe("policy replay capture", () => { + test("writes canonical mode-0600 data outside the measured repo and verifies its identity", () => { + const measuredRepoRoot = gitRepo(); + const outputRoot = mkdtempSync(join(tmpdir(), "rg-policy-replay-output-")); + const sinkDir = join(outputRoot, "policy-replay"); + mkdirSync(sinkDir, { mode: 0o700 }); + + const stored = capturePolicyReplayEnvelope({ sinkDir, measuredRepoRoot, envelope: envelope() }); + expect(stored.status).toBe("complete"); + if (stored.status !== "complete") throw new Error("capture failed"); + const path = join(sinkDir, stored.ref); + expect(realpathSync(path).startsWith(realpathSync(outputRoot))).toBe(true); + expect(realpathSync(path).startsWith(realpathSync(measuredRepoRoot))).toBe(false); + expect(lstatSync(path).mode & 0o7777).toBe(0o600); + expect(readFileSync(path, "utf8")).toBe(canonicalJson(stored.envelope)); + expect(verifyPolicyReplayEnvelope({ sinkDir, ref: stored.ref, sha256: stored.sha256 })).toEqual( + { + ok: true, + envelope: stored.envelope, + }, + ); + }); + + test("redaction makes the artifact diagnostic-only instead of claiming exact authority", () => { + const measuredRepoRoot = gitRepo(); + const sinkDir = mkdtempSync(join(tmpdir(), "rg-policy-replay-sink-")); + const leaked = "ghp_abcdefghijklmnopqrstuvwxyz123456"; + const candidate = { + ...envelope(), + pre_policy_findings: [finding("sig-secret")], + grounding: { + corpus: `Authorization: Bearer ${leaked}`, + verdicts: [], + llm_status: "not-run" as const, + }, + }; + const stored = capturePolicyReplayEnvelope({ sinkDir, measuredRepoRoot, envelope: candidate }); + expect(stored.status).toBe("complete"); + if (stored.status !== "complete") throw new Error("capture failed"); + expect(stored.envelope.lossless).toBe(false); + expect(readFileSync(join(sinkDir, stored.ref), "utf8")).not.toContain(leaked); + expect( + verifyPolicyReplayEnvelope({ + sinkDir, + ref: stored.ref, + sha256: stored.sha256, + authoritative: true, + }), + ).toEqual({ ok: false, reason: "lossy" }); + }); + + test("persists a bounded mode-0600 overflow status for the driver", () => { + const measuredRepoRoot = gitRepo(); + const sinkDir = mkdtempSync(join(tmpdir(), "rg-policy-replay-sink-")); + expect( + capturePolicyReplayEnvelope({ + sinkDir, + measuredRepoRoot, + envelope: envelope(), + maxBytes: 1, + }), + ).toEqual({ status: "overflow", reason: "too-large" }); + const statusFiles = readdirSync(sinkDir).filter((name) => name.endsWith(".overflow")); + expect(statusFiles).toHaveLength(1); + expect(lstatSync(join(sinkDir, statusFiles[0] as string)).mode & 0o7777).toBe(0o600); + }); + + test("fails closed on measured-repo sinks, mode drift, symlink escape, and tamper", () => { + const measuredRepoRoot = gitRepo(); + const inside = join(measuredRepoRoot, "capture"); + mkdirSync(inside); + expect( + capturePolicyReplayEnvelope({ sinkDir: inside, measuredRepoRoot, envelope: envelope() }), + ).toEqual({ status: "error", reason: "sink-inside-measured-repo" }); + + const sinkDir = mkdtempSync(join(tmpdir(), "rg-policy-replay-sink-")); + const stored = capturePolicyReplayEnvelope({ sinkDir, measuredRepoRoot, envelope: envelope() }); + if (stored.status !== "complete") throw new Error("capture failed"); + const path = join(sinkDir, stored.ref); + chmodSync(path, 0o644); + expect(verifyPolicyReplayEnvelope({ sinkDir, ref: stored.ref, sha256: stored.sha256 })).toEqual( + { + ok: false, + reason: "not-a-file", + }, + ); + + chmodSync(path, 0o600); + writeFileSync(path, `${readFileSync(path, "utf8")} `, { mode: 0o600 }); + expect(verifyPolicyReplayEnvelope({ sinkDir, ref: stored.ref, sha256: stored.sha256 })).toEqual( + { + ok: false, + reason: "hash-mismatch", + }, + ); + + const outside = mkdtempSync(join(tmpdir(), "rg-policy-replay-outside-")); + const escapedSink = join(mkdtempSync(join(tmpdir(), "rg-policy-replay-parent-")), "sink"); + symlinkSync(outside, escapedSink); + const traversing = envelope(); + traversing.run_id = "../../escape"; + traversing.policy_trace.run_id = "../../escape"; + expect( + capturePolicyReplayEnvelope({ + sinkDir: escapedSink, + measuredRepoRoot, + envelope: traversing, + }), + ).toEqual({ status: "error", reason: "invalid-sink" }); + }); +}); + +describe("policy replay state isolation", () => { + test("snapshots production-like state and creates independent same-commit branches", () => { + const sourceRepoRoot = gitRepo(); + const outputRoot = mkdtempSync(join(tmpdir(), "rg-policy-state-output-")); + const sourceBytes = readFileSync(join(sourceRepoRoot, ".reviewgate", "fp-ledger.jsonl")); + const sourceDigest = digestPolicyState(join(sourceRepoRoot, ".reviewgate")); + const snapshot = createPolicyStateSnapshot({ sourceRepoRoot, outputRoot }); + expect(snapshot.stateSha256).toBe(sourceDigest); + expect(snapshot.sha256).toMatch(/^[0-9a-f]{64}$/); + expect(lstatSync(join(outputRoot, snapshot.ref)).mode & 0o7777).toBe(0o600); + + const sourceCommit = execFileSync("git", ["rev-parse", "HEAD"], { + cwd: sourceRepoRoot, + encoding: "utf8", + }).trim(); + const branches = createReplayBranches({ + sourceRepoRoot, + sourceCommit, + stateSnapshotRoot: join(outputRoot, snapshot.stateRef), + expectedStateSha256: sourceDigest, + exactDiff: [ + "diff --git a/src/example.ts b/src/example.ts", + "--- a/src/example.ts", + "+++ b/src/example.ts", + "@@ -1 +1 @@", + "-export const value = 0;", + "+export const value = 1;", + "", + ].join("\n"), + }); + try { + expect(branches.baseline.startingStateSha256).toBe(sourceDigest); + expect(branches.counterfactual.startingStateSha256).toBe(sourceDigest); + expect(readFileSync(join(branches.baseline.checkoutRoot, "src", "example.ts"), "utf8")).toBe( + "export const value = 1;\n", + ); + expect( + readFileSync(join(branches.counterfactual.checkoutRoot, "src", "example.ts"), "utf8"), + ).toBe("export const value = 1;\n"); + const sourceStateReal = realpathSync(join(sourceRepoRoot, ".reviewgate")); + for (const branch of [branches.baseline, branches.counterfactual]) { + const branchStateReal = realpathSync(join(branch.checkoutRoot, ".reviewgate")); + expect(relative(sourceStateReal, branchStateReal).startsWith("..")).toBe(true); + expect(lstatSync(join(branch.checkoutRoot, ".reviewgate", "fp-ledger.jsonl")).ino).not.toBe( + lstatSync(join(sourceRepoRoot, ".reviewgate", "fp-ledger.jsonl")).ino, + ); + } + writeFileSync( + join(branches.baseline.checkoutRoot, ".reviewgate", "fp-ledger.jsonl"), + '{"id":"baseline-only"}\n', + ); + writeFileSync( + join(branches.counterfactual.checkoutRoot, ".reviewgate", "fp-ledger.jsonl"), + '{"id":"counterfactual-only"}\n', + ); + expect(digestPolicyState(join(branches.baseline.checkoutRoot, ".reviewgate"))).not.toBe( + digestPolicyState(join(branches.counterfactual.checkoutRoot, ".reviewgate")), + ); + expect(readFileSync(join(sourceRepoRoot, ".reviewgate", "fp-ledger.jsonl"))).toEqual( + sourceBytes, + ); + } finally { + cleanupReplayBranches(branches); + } + expect(existsSync(branches.root)).toBe(false); + }); + + test("rejects symlinked state before hashing or copying it", () => { + const sourceRepoRoot = gitRepo(); + const outside = join(mkdtempSync(join(tmpdir(), "rg-policy-state-outside-")), "secret.json"); + writeFileSync(outside, "secret\n"); + symlinkSync(outside, join(sourceRepoRoot, ".reviewgate", "linked.json")); + expect(() => digestPolicyState(join(sourceRepoRoot, ".reviewgate"))).toThrow(/symlink/i); + expect(() => + createPolicyStateSnapshot({ + sourceRepoRoot, + outputRoot: mkdtempSync(join(tmpdir(), "rg-policy-state-output-")), + }), + ).toThrow(/symlink/i); + }); + + test("rejects hardlinked and special state entries", () => { + const hardlinkedRepo = gitRepo(); + linkSync( + join(hardlinkedRepo, ".reviewgate", "fp-ledger.jsonl"), + join(hardlinkedRepo, ".reviewgate", "hardlink.jsonl"), + ); + expect(() => digestPolicyState(join(hardlinkedRepo, ".reviewgate"))).toThrow(/hardlink/i); + + const specialRepo = gitRepo(); + const fifo = join(specialRepo, ".reviewgate", "state.fifo"); + execFileSync("mkfifo", [fifo]); + expect(() => digestPolicyState(join(specialRepo, ".reviewgate"))).toThrow(/special/i); + }); + + test("rejects source-state aliasing and unequal requested starting digests", () => { + const sourceRepoRoot = gitRepo(); + const sourceCommit = execFileSync("git", ["rev-parse", "HEAD"], { + cwd: sourceRepoRoot, + encoding: "utf8", + }).trim(); + const stateRoot = join(sourceRepoRoot, ".reviewgate"); + const actualDigest = digestPolicyState(stateRoot); + + expect(() => + createReplayBranches({ + sourceRepoRoot, + sourceCommit, + stateSnapshotRoot: stateRoot, + expectedStateSha256: actualDigest, + exactDiff: "", + }), + ).toThrow(/must not alias/i); + + const outputRoot = mkdtempSync(join(tmpdir(), "rg-policy-state-output-")); + const snapshot = createPolicyStateSnapshot({ sourceRepoRoot, outputRoot }); + expect(() => + createReplayBranches({ + sourceRepoRoot, + sourceCommit, + stateSnapshotRoot: join(outputRoot, snapshot.stateRef), + expectedStateSha256: "f".repeat(64), + exactDiff: "", + }), + ).toThrow(/digest mismatch/i); + }); +}); diff --git a/tests/unit/rig-ablate.test.ts b/tests/unit/rig-ablate.test.ts index ecc8f52..7397039 100644 --- a/tests/unit/rig-ablate.test.ts +++ b/tests/unit/rig-ablate.test.ts @@ -1,6 +1,8 @@ import { describe, expect, test } from "bun:test"; import { makeMetric, summarizeSpread } from "../../src/bench/metrics.ts"; +import { POLICY_PASS_IDS } from "../../src/core/policy/catalog.ts"; import { ablate, renderAblationMatrix } from "../../src/rig/ablate.ts"; +import { renderPolicyAblationRows } from "../../src/rig/replay.ts"; import type { Finding } from "../../src/schemas/finding.ts"; import type { RigResult, RigTurnRecord } from "../../src/schemas/rig-result.ts"; @@ -86,6 +88,32 @@ function result(turns: RigTurnRecord[], over: Partial = {}): RigResul const NO_TAGS = new Map(); describe("rig ablate", () => { + test("exact rows use every closed catalog ID and keep Lore separate", () => { + const rendered = renderPolicyAblationRows( + POLICY_PASS_IDS.map((passId) => ({ + passId, + authoritative: true, + reason: null, + envelopes: 1, + opportunities: 1, + applied: 1, + wouldApplyWithoutMutation: 1, + baselineBlocking: 0, + counterfactualBlocking: 1, + })), + ); + for (const passId of POLICY_PASS_IDS) expect(rendered).toContain(passId); + expect(rendered).toContain("Lore is excluded"); + expect(rendered).not.toMatch(/^\s+lore\s/m); + }); + + test("the four historical layers identify themselves as legacy and non-authoritative", () => { + const base = result([turn({ index: 1, findings: [] })]); + const legacy = ablate(base, "critic", NO_TAGS); + expect(legacy.authoritative).toBe(false); + expect(renderAblationMatrix(base, [legacy])).toContain("NON-AUTHORITATIVE LEGACY"); + }); + // The smallest assertion that proves the toggle is wired to something real rather than to // nothing: one critic-demoted finding, one more blocking finding when the critic is off. test("ablating the critic raises the blocking count by exactly one", () => { diff --git a/tests/unit/rig-driver.test.ts b/tests/unit/rig-driver.test.ts index 1f20976..e34206a 100644 --- a/tests/unit/rig-driver.test.ts +++ b/tests/unit/rig-driver.test.ts @@ -11,7 +11,9 @@ import { import { tmpdir } from "node:os"; import { join } from "node:path"; import { runRigRun } from "../../src/cli/commands/rig.ts"; +import { POLICY_CATALOG_VERSION } from "../../src/core/policy/catalog.ts"; import { runDriver } from "../../src/rig/driver.ts"; +import { createPolicyStateSnapshot } from "../../src/rig/policy-replay-state.ts"; // The driver spawns a real agent process per turn. Every test here injects a FAKE agent // instead — a shell one-liner that writes a marker. A test that spent `claude -p` quota @@ -55,6 +57,115 @@ const appendingAgent = (root: string) => (prompt: string) => [ ]; describe("rig driver", () => { + test("exports only the replay sink and records immutable trace/state identity outside the repo", async () => { + const { root, scriptPath } = sandbox(1); + execFileSync("git", ["init", "-q", "."], { cwd: root }); + execFileSync("git", ["config", "user.email", "rig@example.invalid"], { cwd: root }); + execFileSync("git", ["config", "user.name", "rig"], { cwd: root }); + writeFileSync(join(root, "tracked.ts"), "export const tracked = true;\n"); + execFileSync("git", ["add", "tracked.ts"], { cwd: root }); + execFileSync("git", ["commit", "-qm", "source"], { cwd: root }); + const sourceCommit = execFileSync("git", ["rev-parse", "HEAD"], { + cwd: root, + encoding: "utf8", + }).trim(); + const outDir = mkdtempSync(join(tmpdir(), "rg-rig-policy-out-")); + const state = createPolicyStateSnapshot({ sourceRepoRoot: root, outputRoot: outDir }); + const sinkDir = join(outDir, "policy-replay"); + mkdirSync(sinkDir, { mode: 0o700 }); + const cassettePath = join(root, "cassette.jsonl"); + writeFileSync(cassettePath, "", { mode: 0o600 }); + const emptyHash = new Bun.CryptoHasher("sha256").update("").digest("hex"); + const refs = [ + `${"a".repeat(12)}-i10-${"b".repeat(12)}.json`, + `${"a".repeat(12)}-i2-${"c".repeat(12)}.json`, + `${"a".repeat(12)}-i1-${"d".repeat(12)}.json`, + ] as const; + + const manifest = await runDriver({ + scriptPath, + outDir, + repoRoot: root, + agentCmd: () => [ + "bash", + "-c", + 'test "$REVIEWGATE_RIG_REPLAY_DIR" = "$1"; sink="$1"; shift; for ref in "$@"; do printf "{}" > "$sink/$ref"; done', + "fake-agent", + sinkDir, + ...refs, + ], + maxTurns: 1, + policyReplay: { + sinkDir, + cassettePath, + metadata: { + catalogVersion: POLICY_CATALOG_VERSION, + sourceCommit, + initialStateRef: state.ref, + initialStateSha256: state.sha256, + initialStateDigest: state.stateSha256, + cassetteSha256: emptyHash, + cassetteRef: "cassette.jsonl", + captureDir: "policy-replay", + }, + }, + }); + + expect(manifest.policyReplay?.initialStateDigest).toBe(state.stateSha256); + expect(manifest.turns[0]?.policyReplay?.status).toBe("complete"); + expect(manifest.turns[0]?.policyReplay?.traces.map((trace) => trace.ref)).toEqual([ + refs[2], + refs[1], + refs[0], + ]); + expect(existsSync(join(outDir, "cassette.jsonl"))).toBe(true); + expect(existsSync(join(root, ".reviewgate", "policy-replay"))).toBe(false); + }); + + test("carries a capture overflow marker into the authoritative turn status", async () => { + const { root, scriptPath } = sandbox(1); + execFileSync("git", ["init", "-q", "."], { cwd: root }); + execFileSync("git", ["config", "user.email", "rig@example.invalid"], { cwd: root }); + execFileSync("git", ["config", "user.name", "rig"], { cwd: root }); + writeFileSync(join(root, "tracked.ts"), "export const tracked = true;\n"); + execFileSync("git", ["add", "tracked.ts"], { cwd: root }); + execFileSync("git", ["commit", "-qm", "source"], { cwd: root }); + const sourceCommit = execFileSync("git", ["rev-parse", "HEAD"], { + cwd: root, + encoding: "utf8", + }).trim(); + const outDir = mkdtempSync(join(tmpdir(), "rg-rig-policy-out-")); + const state = createPolicyStateSnapshot({ sourceRepoRoot: root, outputRoot: outDir }); + const sinkDir = join(outDir, "policy-replay"); + mkdirSync(sinkDir, { mode: 0o700 }); + const cassettePath = join(root, "cassette.jsonl"); + writeFileSync(cassettePath, "", { mode: 0o600 }); + const emptyHash = new Bun.CryptoHasher("sha256").update("").digest("hex"); + const marker = `${"a".repeat(12)}-i1.overflow`; + const manifest = await runDriver({ + scriptPath, + outDir, + repoRoot: root, + agentCmd: () => ["bash", "-c", 'printf "{}" > "$1/$2"', "agent", sinkDir, marker], + maxTurns: 1, + policyReplay: { + sinkDir, + cassettePath, + metadata: { + catalogVersion: POLICY_CATALOG_VERSION, + sourceCommit, + initialStateRef: state.ref, + initialStateSha256: state.sha256, + initialStateDigest: state.stateSha256, + cassetteSha256: emptyHash, + cassetteRef: "cassette.jsonl", + captureDir: "policy-replay", + }, + }, + }); + expect(manifest.turns[0]?.policyReplay).toEqual({ status: "overflow", traces: [] }); + }); + test("runs one snapshot per turn and honours maxTurns", async () => { const { root, scriptPath } = sandbox(3); const manifest = await runDriver({ diff --git a/tests/unit/rig-harvest.test.ts b/tests/unit/rig-harvest.test.ts index e8b8755..f487b33 100644 --- a/tests/unit/rig-harvest.test.ts +++ b/tests/unit/rig-harvest.test.ts @@ -734,6 +734,27 @@ describe("rig harvest", () => { const fx = canonicalFixture(); const result = harvest(fx.manifestPath, fx.scriptPath); expect(() => RigResultSchema.parse(result)).not.toThrow(); + expect(result.policyReplay).toEqual({ + authoritative: false, + catalogVersion: null, + sourceCommit: null, + passIds: [], + reason: + "legacy run: no exact policy replay metadata; four-layer counts are non-authoritative", + }); + expect(() => + RigResultSchema.parse({ + ...result, + turns: result.turns.map((turn, index) => + index === 0 + ? { + ...turn, + policyReplay: { status: "complete", traces: [], reason: null }, + } + : turn, + ), + }), + ).toThrow(/complete/i); }); test("provenance names the panel it was measured on", () => { diff --git a/tests/unit/rig-replay.test.ts b/tests/unit/rig-replay.test.ts index e67fab9..db01bc1 100644 --- a/tests/unit/rig-replay.test.ts +++ b/tests/unit/rig-replay.test.ts @@ -1,8 +1,37 @@ import { describe, expect, test } from "bun:test"; -import { mkdirSync, mkdtempSync, writeFileSync } from "node:fs"; +import { execFileSync } from "node:child_process"; +import { createHash } from "node:crypto"; +import { cpSync, mkdirSync, mkdtempSync, readFileSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; -import { checkCassette, checkDeterminism, replay } from "../../src/rig/replay.ts"; +import { aggregate } from "../../src/core/aggregator.ts"; +import { validateFindingFacts } from "../../src/core/fact-check.ts"; +import { groundFindings } from "../../src/core/grounding.ts"; +import { demoteHypotheticalCriticals } from "../../src/core/hypothetical-demote.ts"; +import { POLICY_CATALOG_VERSION } from "../../src/core/policy/catalog.ts"; +import { capturePolicyReplayEnvelope } from "../../src/core/policy/replay-capture.ts"; +import { PolicyTraceRecorder } from "../../src/core/policy/trace.ts"; +import { ReputationStore } from "../../src/core/reputation/store.ts"; +import { demoteSelfRefuting } from "../../src/core/self-refutation.ts"; +import { + RigAuthorityError, + createPolicyStateSnapshot, + digestPolicyState, + validateRigPolicyReplayArtifacts, +} from "../../src/rig/policy-replay-state.ts"; +import { + checkCassette, + checkDeterminism, + replay, + replayPolicyEnvelopePair, + replayPolicyEnvelopeSequence, + runWithReplayProviderCeiling, +} from "../../src/rig/replay.ts"; +import { + type PolicyReplayEnvelopeInput, + PolicyReplayEnvelopeSchema, +} from "../../src/schemas/policy-replay.ts"; +import type { RigManifest } from "../../src/schemas/rig-manifest.ts"; /** Smallest run that harvests: one turn, one snapshot laid out as a repo root. */ function miniRun(): { manifestPath: string; scriptPath: string; root: string } { @@ -85,6 +114,284 @@ const entry = (key: string, result: Record | "empty") => }, }); +function sha256(value: string | Buffer): string { + return createHash("sha256").update(value).digest("hex"); +} + +function emptyPolicyTrace( + rawResponseSha256: string[], + identity: { runId: string; iter: number } = { runId: "exact-run", iter: 1 }, +) { + const runtime = PolicyTraceRecorder.start({ + runId: identity.runId, + iter: identity.iter, + ablated: new Set(), + }); + const result = aggregate({ + findings: [], + reviewersTotal: 1, + changedRanges: new Map(), + scopeToDiff: true, + outOfDiffBlocking: [], + confidenceFloor: 0, + demoteCorrectness: true, + corroborateCritical: true, + demoteTestSecurity: true, + capDocsSeverity: true, + critic: new Map(), + fpActive: new Map(), + fpActiveClusters: new Map(), + repUnreliable: new Set(), + protectedReviewers: new Set(), + foreignFiles: new Set(), + cycleRejected: new Set(), + claimedFixed: new Map(), + deltaScope: new Set(), + rejectedRegions: [], + policyRuntime: runtime, + policyInactive: {}, + }); + const trace = runtime.finalize({ + rawResponseSha256, + verdict: result.verdict, + finalFindings: result.dedupedFindings, + }); + if (trace === null) throw new Error("trace fixture failed"); + return trace; +} + +function exactRun(): { + sourceRepoRoot: string; + root: string; + manifestPath: string; + manifest: RigManifest; + envelope: PolicyReplayEnvelopeInput; + stateRoot: string; + sinkDir: string; +} { + const sourceRepoRoot = mkdtempSync(join(tmpdir(), "rg-policy-replay-source-")); + execFileSync("git", ["init", "-q", "."], { cwd: sourceRepoRoot }); + execFileSync("git", ["config", "user.email", "rig@example.invalid"], { + cwd: sourceRepoRoot, + }); + execFileSync("git", ["config", "user.name", "rig"], { cwd: sourceRepoRoot }); + mkdirSync(join(sourceRepoRoot, "src")); + writeFileSync(join(sourceRepoRoot, "src", "x.ts"), "export const x = 1;\n"); + mkdirSync(join(sourceRepoRoot, ".reviewgate")); + writeFileSync(join(sourceRepoRoot, ".reviewgate", "fp-ledger.jsonl"), ""); + execFileSync("git", ["add", "src/x.ts"], { cwd: sourceRepoRoot }); + execFileSync("git", ["commit", "-qm", "source"], { cwd: sourceRepoRoot }); + const sourceCommit = execFileSync("git", ["rev-parse", "HEAD"], { + cwd: sourceRepoRoot, + encoding: "utf8", + }).trim(); + + const root = mkdtempSync(join(tmpdir(), "rg-policy-replay-run-")); + const state = createPolicyStateSnapshot({ sourceRepoRoot, outputRoot: root }); + const rawText = "safe recorded review response"; + const rawHash = sha256(rawText); + const policyTrace = emptyPolicyTrace([rawHash]); + const envelope: PolicyReplayEnvelopeInput = { + schema: "reviewgate.policy-replay-envelope.v1", + catalog_version: POLICY_CATALOG_VERSION, + run_id: "exact-run", + iter: 1, + source_commit: sourceCommit, + exact_diff: "", + pre_policy_findings: [], + grounding: { corpus: "", verdicts: [], llm_status: "not-run" }, + aggregate: { + findings: [], + reviewers_total: 1, + changed_ranges: [], + scope_to_diff: true, + out_of_diff_blocking: [], + confidence_floor: 0, + demote_correctness: true, + corroborate_critical: true, + demote_test_security: true, + cap_docs_severity: true, + critic: [], + fp_active: [], + fp_active_clusters: [], + rep_unreliable: [], + protected_reviewers: [], + foreign_files: [], + cycle_rejected: [], + claimed_fixed: [], + delta_scope: [], + rejected_regions: [], + policy_inactive: [], + }, + policy_final_findings: [], + pre_policy: { self_refutation_enabled: true, hypothetical_enabled: true }, + state_sha256: state.stateSha256, + raw_response_sha256: [rawHash], + policy_trace: policyTrace, + lossless: true, + }; + const sinkDir = join(root, "policy-replay"); + mkdirSync(sinkDir, { mode: 0o700 }); + const stored = capturePolicyReplayEnvelope({ + sinkDir, + measuredRepoRoot: sourceRepoRoot, + envelope, + }); + if (stored.status !== "complete") throw new Error("capture fixture failed"); + const cassettePath = join(root, "cassette.jsonl"); + writeFileSync(cassettePath, `${entry("openrouter-security", { rawText })}\n`, { mode: 0o600 }); + const manifestPath = join(root, "manifest.json"); + const manifest: RigManifest = { + schema: "reviewgate.rig.manifest.v1", + runId: "exact-rig", + scriptId: "exact-script", + outDir: root, + turns: [ + { + index: 1, + snapshotDir: join(root, "turns", "1"), + agentExitCode: 0, + wallMs: 1, + policyReplay: { status: "complete", traces: [{ ref: stored.ref, sha256: stored.sha256 }] }, + }, + ], + policyReplay: { + catalogVersion: POLICY_CATALOG_VERSION, + sourceCommit, + initialStateRef: state.ref, + initialStateSha256: state.sha256, + initialStateDigest: state.stateSha256, + cassetteSha256: sha256(readFileSync(cassettePath)), + cassetteRef: "cassette.jsonl", + captureDir: "policy-replay", + }, + }; + writeFileSync(manifestPath, JSON.stringify(manifest)); + return { + sourceRepoRoot, + root, + manifestPath, + manifest, + envelope, + stateRoot: join(root, state.stateRef), + sinkDir, + }; +} + +function replaceTrace( + fixture: ReturnType, + envelope: PolicyReplayEnvelopeInput, +): void { + const stored = capturePolicyReplayEnvelope({ + sinkDir: fixture.sinkDir, + measuredRepoRoot: fixture.sourceRepoRoot, + envelope, + }); + if (stored.status !== "complete") throw new Error("replacement capture failed"); + const turn = fixture.manifest.turns[0]; + if (turn === undefined) throw new Error("turn fixture missing"); + turn.policyReplay = { status: "complete", traces: [{ ref: stored.ref, sha256: stored.sha256 }] }; +} + +function confidenceEnvelope(fixture: ReturnType): PolicyReplayEnvelopeInput { + const finding = { + id: "confidence-1", + signature: "confidence-sig", + severity: "WARN" as const, + category: "quality" as const, + rule_id: "confidence-rule", + file: "src/x.ts", + line_start: 1, + line_end: 1, + message: "Low-confidence warning", + details: "The reviewer explicitly reports uncertainty.", + reviewer: { provider: "codex", model: "gpt-5", persona: "quality" }, + confidence: 0.2, + consensus: "singleton" as const, + }; + const runtime = PolicyTraceRecorder.start({ + runId: "exact-run", + iter: 1, + ablated: new Set(), + }); + const factChecked = validateFindingFacts([finding], fixture.sourceRepoRoot, new Set(), runtime); + const selfScreened = demoteSelfRefuting(factChecked, true, runtime); + const hypothetical = demoteHypotheticalCriticals(selfScreened, true, runtime); + const grounded = groundFindings(hypothetical, "", runtime); + runtime.markInactive("judgment.grounding-llm", "configured-off"); + const policyInactive = { + "judgment.critic": "configured-off" as const, + "scope.diff": "configured-off" as const, + "scope.delta": "stage-precondition-miss" as const, + "scope.session": "stage-precondition-miss" as const, + }; + const result = aggregate({ + findings: grounded, + reviewersTotal: 1, + changedRanges: new Map(), + scopeToDiff: false, + outOfDiffBlocking: [], + confidenceFloor: 0.8, + demoteCorrectness: true, + corroborateCritical: true, + demoteTestSecurity: true, + capDocsSeverity: true, + critic: new Map(), + fpActive: new Map(), + fpActiveClusters: new Map(), + repUnreliable: new Set(), + protectedReviewers: new Set(), + foreignFiles: new Set(), + cycleRejected: new Set(), + claimedFixed: new Map(), + deltaScope: new Set(), + rejectedRegions: [], + policyRuntime: runtime, + policyInactive, + }); + const policyTrace = runtime.finalize({ + rawResponseSha256: [...fixture.envelope.raw_response_sha256], + verdict: result.verdict, + finalFindings: result.dedupedFindings, + }); + if (policyTrace === null) throw new Error("confidence trace fixture failed"); + return { + ...fixture.envelope, + pre_policy_findings: [finding], + grounding: { corpus: "", verdicts: [], llm_status: "not-run" }, + aggregate: { + findings: grounded, + reviewers_total: 1, + changed_ranges: [], + scope_to_diff: false, + out_of_diff_blocking: [], + confidence_floor: 0.8, + demote_correctness: true, + corroborate_critical: true, + demote_test_security: true, + cap_docs_severity: true, + critic: [], + fp_active: [], + fp_active_clusters: [], + rep_unreliable: [], + protected_reviewers: [], + foreign_files: [], + cycle_rejected: [], + claimed_fixed: [], + delta_scope: [], + rejected_regions: [], + policy_inactive: Object.entries(policyInactive) + .map(([pass_id, reason_code]) => ({ + pass_id: pass_id as keyof typeof policyInactive, + reason_code, + })) + .sort((left, right) => left.pass_id.localeCompare(right.pass_id)), + }, + policy_final_findings: result.dedupedFindings, + policy_trace: policyTrace, + }; +} + describe("rig replay — determinism self-check", () => { test("a run whose metrics re-derive identically is DETERMINISTIC", () => { const { manifestPath, scriptPath } = miniRun(); @@ -147,10 +454,280 @@ describe("rig replay — cassette integrity", () => { expect(c.malformedLines).toBe(1); }); - test("a --cassette path that does not exist fails loudly", () => { + test("a --cassette path that does not exist fails loudly", async () => { const { manifestPath, scriptPath, root } = miniRun(); - expect(() => + await expect( replay({ manifestPath, scriptPath, cassettePath: join(root, "nope.jsonl") }), - ).toThrow(/no cassette at/); + ).rejects.toThrow(/no cassette at/); + }); +}); + +describe("rig replay — exact policy authority", () => { + test("turns any attempted network or provider subprocess call into authority exit 4", () => { + const attempts: Array<() => unknown> = [ + () => fetch("https://example.invalid"), + () => Bun.spawn(["true"]), + ]; + for (const attempt of attempts) { + try { + runWithReplayProviderCeiling(attempt); + throw new Error("live provider attempt unexpectedly escaped the ceiling"); + } catch (error) { + expect(error).toBeInstanceOf(RigAuthorityError); + expect((error as RigAuthorityError).code).toBe("live-provider-call"); + expect((error as RigAuthorityError).exitCode).toBe(4); + } + } + }); + + test("replays production policy in isolated branches without a live provider capability", () => { + const fixture = exactRun(); + const validated = validateRigPolicyReplayArtifacts({ + manifest: fixture.manifest, + manifestPath: fixture.manifestPath, + }); + expect(validated).not.toBeNull(); + const item = validated?.turns.get(1)?.[0]; + if (item === undefined) throw new Error("validated trace missing"); + const pair = replayPolicyEnvelopePair({ + sourceRepoRoot: fixture.sourceRepoRoot, + envelope: item.envelope, + stateSnapshotRoot: item.stateRoot, + passId: "judgment.confidence", + }); + expect(pair.baseline.raw_response_sha256).toEqual(pair.counterfactual.raw_response_sha256); + expect(pair.counterfactual.ablated).toEqual(["judgment.confidence"]); + }); + + test("allows the ablated production pass to change output after reproducing the baseline", () => { + const fixture = exactRun(); + const candidate = PolicyReplayEnvelopeSchema.parse(confidenceEnvelope(fixture)); + const pair = replayPolicyEnvelopePair({ + sourceRepoRoot: fixture.sourceRepoRoot, + envelope: candidate, + stateSnapshotRoot: fixture.stateRoot, + passId: "judgment.confidence", + }); + expect(pair.baseline.final.counts.info).toBe(1); + expect(pair.counterfactual.final.counts.warn).toBe(1); + }); + + test("keeps one branch pair and branch-local Store writes across a multi-envelope sequence", async () => { + const fixture = exactRun(); + const sourceStateBefore = digestPolicyState(join(fixture.sourceRepoRoot, ".reviewgate")); + const sourceFileBefore = readFileSync(join(fixture.sourceRepoRoot, "src", "x.ts"), "utf8"); + const expectedRepo = mkdtempSync(join(tmpdir(), "rg-policy-sequence-expected-")); + cpSync(fixture.stateRoot, join(expectedRepo, ".reviewgate"), { recursive: true }); + const event = { + reviewerKey: "codex:quality", + eid: "exact-run:1:confidence-sig:codex:quality", + ts: "2026-08-11T12:00:00.000Z", + }; + await new ReputationStore(expectedRepo).record([{ ...event, outcome: "correct" }], { + now: new Date(event.ts), + }); + const expectedOutput = mkdtempSync(join(tmpdir(), "rg-policy-sequence-state-")); + const secondState = createPolicyStateSnapshot({ + sourceRepoRoot: expectedRepo, + outputRoot: expectedOutput, + }); + const secondTrace = emptyPolicyTrace([...fixture.envelope.raw_response_sha256], { + runId: "exact-run", + iter: 2, + }); + const secondEnvelope = PolicyReplayEnvelopeSchema.parse({ + ...fixture.envelope, + iter: 2, + exact_diff: [ + "diff --git a/src/x.ts b/src/x.ts", + "--- a/src/x.ts", + "+++ b/src/x.ts", + "@@ -1 +1 @@", + "-export const x = 1;", + "+export const x = 2;", + "", + ].join("\n"), + state_sha256: secondState.stateSha256, + policy_trace: secondTrace, + }); + const branchRoots: Array<{ baseline: string; counterfactual: string }> = []; + + await replayPolicyEnvelopeSequence({ + sourceRepoRoot: fixture.sourceRepoRoot, + passId: "judgment.confidence", + items: [ + { + envelope: PolicyReplayEnvelopeSchema.parse(fixture.envelope), + stateSnapshotRoot: fixture.stateRoot, + }, + { + envelope: secondEnvelope, + stateSnapshotRoot: join(expectedOutput, secondState.stateRef), + }, + ], + afterEnvelope: async ({ index, branches }) => { + branchRoots.push({ + baseline: branches.baseline.checkoutRoot, + counterfactual: branches.counterfactual.checkoutRoot, + }); + if (index === 0) { + await new ReputationStore(branches.baseline.checkoutRoot).record( + [{ ...event, outcome: "correct" }], + { now: new Date(event.ts) }, + ); + await new ReputationStore(branches.counterfactual.checkoutRoot).record( + [{ ...event, outcome: "wrong" }], + { now: new Date(event.ts) }, + ); + return; + } + expect(readFileSync(join(branches.baseline.checkoutRoot, "src", "x.ts"), "utf8")).toBe( + "export const x = 2;\n", + ); + const baselineRep = await new ReputationStore(branches.baseline.checkoutRoot).snapshot(); + const counterfactualRep = await new ReputationStore( + branches.counterfactual.checkoutRoot, + ).snapshot(); + expect(baselineRep.reviewers["codex:quality"]?.correct).toHaveLength(1); + expect(counterfactualRep.reviewers["codex:quality"]?.wrong).toHaveLength(1); + expect(digestPolicyState(join(branches.baseline.checkoutRoot, ".reviewgate"))).not.toBe( + digestPolicyState(join(branches.counterfactual.checkoutRoot, ".reviewgate")), + ); + }, + }); + + expect(branchRoots).toHaveLength(2); + expect(branchRoots[1]).toEqual(branchRoots[0]); + expect(digestPolicyState(join(fixture.sourceRepoRoot, ".reviewgate"))).toBe(sourceStateBefore); + expect(readFileSync(join(fixture.sourceRepoRoot, "src", "x.ts"), "utf8")).toBe( + sourceFileBefore, + ); + }); + + test("rejects the authoritative invalidity matrix before metrics", () => { + const cases: Array<{ + name: string; + code: RigAuthorityError["code"]; + mutate: (fixture: ReturnType) => void; + }> = [ + { + name: "missing", + code: "missing-trace", + mutate: (fixture) => { + const turn = fixture.manifest.turns[0]; + if (turn) turn.policyReplay = { status: "missing", traces: [] }; + }, + }, + { + name: "cross catalog", + code: "catalog-mismatch", + mutate: (fixture) => { + if (fixture.manifest.policyReplay) + fixture.manifest.policyReplay.catalogVersion = "future"; + }, + }, + { + name: "tampered", + code: "invalid-trace", + mutate: (fixture) => { + const ref = fixture.manifest.turns[0]?.policyReplay?.traces[0]?.ref; + if (ref) writeFileSync(join(fixture.sinkDir, ref), "{}", { mode: 0o600 }); + }, + }, + { + name: "lossy", + code: "lossy-trace", + mutate: (fixture) => replaceTrace(fixture, { ...fixture.envelope, lossless: false }), + }, + { + name: "state digest", + code: "state-digest-mismatch", + mutate: (fixture) => + replaceTrace(fixture, { ...fixture.envelope, state_sha256: "f".repeat(64) }), + }, + { + name: "source commit", + code: "source-commit-mismatch", + mutate: (fixture) => { + if (fixture.manifest.policyReplay) { + fixture.manifest.policyReplay.sourceCommit = "e".repeat(40); + } + }, + }, + { + name: "response hash", + code: "response-hash-mismatch", + mutate: (fixture) => { + const unknown = "e".repeat(64); + const policyTrace = emptyPolicyTrace([unknown]); + replaceTrace(fixture, { + ...fixture.envelope, + raw_response_sha256: [unknown], + policy_trace: policyTrace, + }); + }, + }, + { + name: "response order", + code: "response-hash-mismatch", + mutate: (fixture) => { + const first = "safe first recorded response"; + const second = "safe second recorded response"; + const hashes = [sha256(first), sha256(second)]; + replaceTrace(fixture, { + ...fixture.envelope, + raw_response_sha256: hashes, + policy_trace: emptyPolicyTrace(hashes), + }); + const reversed = `${entry("openrouter-second", { rawText: second })}\n${entry("openrouter-first", { rawText: first })}\n`; + const cassettePath = join(fixture.root, "cassette.jsonl"); + writeFileSync(cassettePath, reversed, { mode: 0o600 }); + if (fixture.manifest.policyReplay) { + fixture.manifest.policyReplay.cassetteSha256 = sha256(reversed); + } + }, + }, + { + name: "overflow", + code: "trace-overflow", + mutate: (fixture) => { + const bytes = "x".repeat(1_048_577); + const hash = sha256(bytes); + const run = sha256("exact-run").slice(0, 12); + const ref = `${run}-i1-${hash.slice(0, 12)}.json`; + writeFileSync(join(fixture.sinkDir, ref), bytes, { mode: 0o600 }); + const turn = fixture.manifest.turns[0]; + if (turn) turn.policyReplay = { status: "complete", traces: [{ ref, sha256: hash }] }; + }, + }, + { + name: "non-canonical", + code: "non-canonical-trace", + mutate: (fixture) => { + const bytes = JSON.stringify(fixture.envelope, null, 2); + const hash = sha256(bytes); + const run = sha256("exact-run").slice(0, 12); + const ref = `${run}-i1-${hash.slice(0, 12)}.json`; + writeFileSync(join(fixture.sinkDir, ref), bytes, { mode: 0o600 }); + const turn = fixture.manifest.turns[0]; + if (turn) turn.policyReplay = { status: "complete", traces: [{ ref, sha256: hash }] }; + }, + }, + ]; + + for (const row of cases) { + const fixture = exactRun(); + row.mutate(fixture); + try { + validateRigPolicyReplayArtifacts({ + manifest: fixture.manifest, + manifestPath: fixture.manifestPath, + }); + throw new Error(`${row.name} unexpectedly passed`); + } catch (error) { + expect(error, row.name).toBeInstanceOf(RigAuthorityError); + expect((error as RigAuthorityError).code, row.name).toBe(row.code); + } + } }); }); From f1b52431e8e4c9093519263e3443e932dd97fd61 Mon Sep 17 00:00:00 2001 From: Codevena Date: Tue, 11 Aug 2026 17:37:35 +0200 Subject: [PATCH 46/55] fix(rig): enforce authoritative replay state --- src/cassette/recording-adapter.ts | 55 +- src/cli/commands/rig.ts | 33 +- src/cli/index.ts | 18 +- src/core/orchestrator.ts | 271 +++++- src/providers/adapter-base.ts | 19 + src/rig/policy-replay-state.ts | 195 ++++- src/rig/replay.ts | 307 ++++++- src/schemas/cassette.ts | 15 + src/schemas/policy-replay.ts | 143 ++++ tests/unit/cassette-recording-adapter.test.ts | 106 +++ tests/unit/cassette-schema.test.ts | 32 + tests/unit/orchestrator-policy-trace.test.ts | 177 +++- tests/unit/policy-replay-capture.test.ts | 178 ++++ tests/unit/rig-ablate.test.ts | 81 +- tests/unit/rig-replay.test.ts | 794 ++++++++++++++++-- 15 files changed, 2241 insertions(+), 183 deletions(-) diff --git a/src/cassette/recording-adapter.ts b/src/cassette/recording-adapter.ts index d55251f..4e463e6 100644 --- a/src/cassette/recording-adapter.ts +++ b/src/cassette/recording-adapter.ts @@ -6,6 +6,7 @@ import type { EmbedOptions } from "../core/brain/embeddings.ts"; import { redactHighEntropy } from "../diff/sanitizer.ts"; import type { CompleteOptions, + PolicyReplayCallContext, Preflight, ProviderAdapter, ProviderConfig, @@ -14,6 +15,7 @@ import type { } from "../providers/adapter-base.ts"; import type { ProviderId } from "../providers/registry.ts"; import type { CassetteEntry } from "../schemas/cassette.ts"; +import { policyReplayCallId } from "../schemas/policy-replay.ts"; import { completeKey, embedKey, reviewKey, sha256 } from "./matching.ts"; import { appendEntry } from "./store.ts"; @@ -123,16 +125,19 @@ export class RecordingAdapter implements ProviderAdapter { this.complete = async (prompt, opts) => { const text = await realComplete(prompt, opts); const promptSha256 = sha256(prompt); - await this.append({ - method: "complete", - // Key by the prompt hash so each judge phase replays the response - // recorded for ITS exact prompt — a shared per-provider FIFO returned - // a sibling phase's response when pop-order skewed across phases. - // (Replay must match: replay-adapter pops `completeKey(id, sha256(prompt))`.) - key: completeKey(this.id, promptSha256), - promptSha256, - result: { text }, - }); + await this.append( + { + method: "complete", + // Key by the prompt hash so each judge phase replays the response + // recorded for ITS exact prompt — a shared per-provider FIFO returned + // a sibling phase's response when pop-order skewed across phases. + // (Replay must match: replay-adapter pops `completeKey(id, sha256(prompt))`.) + key: completeKey(this.id, promptSha256), + promptSha256, + result: { text }, + }, + opts.policyReplayCall, + ); return text; }; } @@ -146,12 +151,15 @@ export class RecordingAdapter implements ProviderAdapter { input: ReviewInput & { cfg: ProviderConfig; reviewerId: string }, ): Promise { const result = await this.real.review(input); - await this.append({ - method: "review", - key: reviewKey(input.reviewerId), - promptSha256: this.hashFile(input.promptFile), - result, - }); + await this.append( + { + method: "review", + key: reviewKey(input.reviewerId), + promptSha256: this.hashFile(input.promptFile), + result, + }, + input.policyReplayCall, + ); return result; } @@ -165,12 +173,27 @@ export class RecordingAdapter implements ProviderAdapter { private async append( partial: Pick, + policyReplayCall?: PolicyReplayCallContext, ): Promise { try { await appendEntry(this.path, { schema: "reviewgate.cassette.entry.v1", provider: this.id, ...partial, + ...(policyReplayCall === undefined || partial.method === "embed" + ? {} + : { + policyReplayCall: { + ...policyReplayCall, + callId: policyReplayCallId({ + ...policyReplayCall, + provider: this.id, + method: partial.method, + key: partial.key, + promptSha256: partial.promptSha256, + }), + }, + }), // Redact secrets from the stored body (leak-at-rest defense). Cast: the // shape is preserved (only string leaves change) so it still satisfies // CassetteEntry["result"]; loadCassette re-validates against the schema. diff --git a/src/cli/commands/rig.ts b/src/cli/commands/rig.ts index 385d634..4192075 100644 --- a/src/cli/commands/rig.ts +++ b/src/cli/commands/rig.ts @@ -14,12 +14,17 @@ import { realpathSync, } from "node:fs"; import { dirname, isAbsolute, relative, resolve, sep } from "node:path"; -import { POLICY_CATALOG_VERSION } from "../../core/policy/catalog.ts"; +import { + POLICY_CATALOG_VERSION, + POLICY_PASS_IDS, + type PolicyPassId, +} from "../../core/policy/catalog.ts"; import { resolvePolicyReplayCaptureSink } from "../../core/policy/replay-capture.ts"; import { SUPPRESSION_LAYERS, type SuppressionLayer, ablate, + isSuppressionLayer, renderAblationMatrix, seededTagsFromScript, } from "../../rig/ablate.ts"; @@ -206,10 +211,19 @@ export interface RigAblateInput { resultPath: string; scriptPath: string; /** Omitted → every layer, as a matrix. */ - layer?: SuppressionLayer | undefined; + layer?: string | undefined; sourceRepoRoot?: string | undefined; } +export class RigLayerSelectorError extends Error { + readonly exitCode = 2; + + constructor(message: string) { + super(message); + this.name = "RigLayerSelectorError"; + } +} + /** * Exact traced results replay every closed-catalog pass in isolated branches. Legacy results * retain the old four-layer heuristic with a mandatory non-authoritative label. @@ -217,6 +231,12 @@ export interface RigAblateInput { export async function runRigAblate(input: RigAblateInput): Promise { const base = loadResult(input.resultPath); if (base.policyReplay?.authoritative === true) { + const passId = input.layer; + if (passId !== undefined && !(POLICY_PASS_IDS as readonly string[]).includes(passId)) { + throw new RigLayerSelectorError( + `rig ablate: exact --layer must be one closed-catalog id: ${POLICY_PASS_IDS.join(", ")}`, + ); + } const siblingManifest = resolve(dirname(input.resultPath), "manifest.json"); const manifestPath = existsSync(siblingManifest) ? siblingManifest @@ -225,11 +245,18 @@ export async function runRigAblate(input: RigAblateInput): Promise { await replayPolicyAblations({ manifestPath, sourceRepoRoot: input.sourceRepoRoot ?? process.cwd(), + ...(passId === undefined ? {} : { passId: passId as PolicyPassId }), }), ); } + if (input.layer !== undefined && !isSuppressionLayer(input.layer)) { + throw new RigLayerSelectorError( + `rig ablate: legacy --layer must be one of ${SUPPRESSION_LAYERS.join(", ")}`, + ); + } const tags = seededTagsFromScript(input.scriptPath); - const layers = input.layer === undefined ? [...SUPPRESSION_LAYERS] : [input.layer]; + const layers: SuppressionLayer[] = + input.layer === undefined ? [...SUPPRESSION_LAYERS] : [input.layer as SuppressionLayer]; return `NON-AUTHORITATIVE LEGACY ANALYSIS — exact policy opportunities were not captured.\n${renderAblationMatrix( base, layers.map((l) => ablate(base, l, tags)), diff --git a/src/cli/index.ts b/src/cli/index.ts index 42c18ae..4ea1394 100644 --- a/src/cli/index.ts +++ b/src/cli/index.ts @@ -3,10 +3,11 @@ import { homedir } from "node:os"; import { defineCommand, runMain } from "citty"; import { fmtMetric } from "../bench/report.ts"; import { controlPlaneStatus } from "../config/control-plane.ts"; +import { POLICY_PASS_IDS } from "../core/policy/catalog.ts"; import type { AgentHostSelection } from "../hosts/hooks.ts"; import { repoClaudeHookActive } from "../hosts/user-hooks.ts"; import type { ProviderId } from "../providers/registry.ts"; -import { SUPPRESSION_LAYERS, type SuppressionLayer, isSuppressionLayer } from "../rig/ablate.ts"; +import { SUPPRESSION_LAYERS } from "../rig/ablate.ts"; import { RigAuthorityError } from "../rig/policy-replay-state.ts"; import { RG_VERSION } from "../version.ts"; import { runAuditVerify } from "./commands/audit.ts"; @@ -42,6 +43,7 @@ import { runReport } from "./commands/report.ts"; import { runReset } from "./commands/reset.ts"; import { runReviewPlan } from "./commands/review-plan.ts"; import { + RigLayerSelectorError, runRigAblate, runRigHarvest, runRigReplay, @@ -64,6 +66,10 @@ async function runRigAuthorityCommand(run: () => T | Promise): Promise try { return await run(); } catch (error) { + if (error instanceof RigLayerSelectorError) { + process.stderr.write(`${error.message}\n`); + process.exit(error.exitCode); + } if (!(error instanceof RigAuthorityError)) throw error; process.stderr.write(`${error.message}\n`); process.exit(error.exitCode); @@ -1141,24 +1147,18 @@ const rig = defineCommand({ }, layer: { type: "string", - description: `One of ${SUPPRESSION_LAYERS.join(" | ")} (default: all)`, + description: `Exact: one closed-catalog id (${POLICY_PASS_IDS.join(" | ")}); legacy: ${SUPPRESSION_LAYERS.join(" | ")} (default: all)`, }, }, async run({ args }) { const layer = args.layer as string | undefined; - if (layer !== undefined && !isSuppressionLayer(layer)) { - console.error( - `reviewgate rig ablate: --layer must be one of ${SUPPRESSION_LAYERS.join(", ")}`, - ); - process.exit(2); - } process.stdout.write( await runRigAuthorityCommand(() => runRigAblate({ resultPath: args.result as string, scriptPath: args.script as string, sourceRepoRoot: process.cwd(), - ...(layer === undefined ? {} : { layer: layer as SuppressionLayer }), + ...(layer === undefined ? {} : { layer }), }), ), ); diff --git a/src/core/orchestrator.ts b/src/core/orchestrator.ts index 8a17848..0f6fbb1 100644 --- a/src/core/orchestrator.ts +++ b/src/core/orchestrator.ts @@ -14,6 +14,7 @@ import { dirname, isAbsolute, join, relative } from "node:path"; import type { AuditLogger } from "../audit/logger.ts"; import { computeBehaviorHash } from "../cache/behavior-hash.ts"; import { computeCacheKey, getCachedReview, putCachedReview } from "../cache/cache.ts"; +import { completeKey, reviewKey } from "../cassette/matching.ts"; import { cassetteFromEnv } from "../cassette/store.ts"; import { BUDGET_ATTRIBUTION_SLACK_MS, @@ -26,7 +27,12 @@ import type { ReviewgateConfig } from "../config/define-config.ts"; import { parseChangedRanges, parseDeletedPaths } from "../diff/hunks.ts"; import { sanitizeDiff } from "../diff/sanitizer.ts"; import { computeSignature } from "../diff/signature.ts"; -import type { ProviderAdapter, ProviderConfig, ReviewResult } from "../providers/adapter-base.ts"; +import type { + PolicyReplayCallContext, + ProviderAdapter, + ProviderConfig, + ReviewResult, +} from "../providers/adapter-base.ts"; import { isProviderAvailable } from "../providers/availability.ts"; import { type ProviderId, SUBPROCESSLESS_PROVIDERS } from "../providers/registry.ts"; import { parseReviewOutput } from "../providers/review-output.ts"; @@ -53,6 +59,7 @@ import type { RunSummary } from "../schemas/audit-event.ts"; import { type MemoryProposal, VALID_EVIDENCE_KINDS } from "../schemas/brain.ts"; import type { Finding, FindingCategory } from "../schemas/finding.ts"; import { NO_PANEL_REVIEWER_ID } from "../schemas/pending-report.ts"; +import { type PolicyReplayEnvelopeInput, policyReplayCallId } from "../schemas/policy-replay.ts"; import { type PolicySummary, PolicySummarySchema, @@ -79,7 +86,7 @@ import { decayPass } from "./brain/lifecycle.ts"; import { ProposalStore } from "./brain/proposal-store.ts"; import { BrainStore } from "./brain/store.ts"; import { runChecks } from "./checks/runner.ts"; -import { type CriticVerdict, runCritic } from "./critic.ts"; +import { type CriticVerdict, buildCriticPrompt, runCritic } from "./critic.ts"; import { attestEvidence, validateFindingFacts } from "./fact-check.ts"; import { computeFpClusters } from "./fp-ledger/clusters.ts"; import { buildFpFewShot } from "./fp-ledger/few-shot.ts"; @@ -92,7 +99,12 @@ import { fragmentingFpClasses, } from "./fp-ledger/fragmentation.ts"; import { FpLedgerStore } from "./fp-ledger/store.ts"; -import { applyGroundingJudgeVerdicts, groundFindings, judgeGrounding } from "./grounding.ts"; +import { + applyGroundingJudgeVerdicts, + buildGroundingJudgePrompt, + groundFindings, + judgeGrounding, +} from "./grounding.ts"; import { renderHouseRules } from "./house-rules.ts"; import { demoteHypotheticalCriticals } from "./hypothetical-demote.ts"; import { ImplicitOutcomeStore, deriveImplicitOutcomes } from "./learnings/implicit-outcomes.ts"; @@ -701,6 +713,12 @@ interface ReviewerRun { provider: ProviderId; persona: string; model: string; + /** Actual adapter-call attempt within this logical reviewer slot (skips do not count). */ + attempt?: number; + /** Exact prompt digest used by RecordingAdapter for this call. */ + promptSha256?: string; + /** Exact caller identity sent to RecordingAdapter for authoritative Rig capture. */ + policyReplayCall?: PolicyReplayCallContext; // Deadline-aware budgets: true when this run's granted window was MATERIALLY // shortened by the remaining-budget clamp (more than BUDGET_ATTRIBUTION_SLACK_MS // below the provider's configured timeoutMs). A timeout under such a window is @@ -951,9 +969,8 @@ export class Orchestrator { const fpFullSnapshot = fpStore ? await fpStore.snapshot() : undefined; // Pass the run timestamp so a sticky/active whose window has expired is // re-evaluated at read time and never served as suppressing (F-017). - const fpActiveSnapshot = fpStore - ? await fpStore.activeSnapshot(this.input.now?.() ?? new Date()) - : undefined; + const fpObservedAt = this.input.now?.() ?? new Date(); + const fpActiveSnapshot = fpStore ? await fpStore.activeSnapshot(fpObservedAt) : undefined; // M6: Context7 library docs. Fetched PRE-CACHE — before the behavior-hash — // so the docs-corpus identity feeds the cache key (a docs change must @@ -1647,9 +1664,30 @@ export class Orchestrator { findingsPath: string, diffPath: string, tmpDir: string, + attempt: number, + logicalSlot: number, ): Promise => { const adapter = this.input.adapters[provider]; const reviewStart = Date.now(); + let promptSha256: string | undefined; + try { + promptSha256 = createHash("sha256").update(readFileSync(promptFile)).digest("hex"); + } catch { + // Capture-only identity. The provider call retains its production behavior; an + // unavailable prompt digest simply prevents an authoritative replay envelope. + } + const policyReplayCall: PolicyReplayCallContext | undefined = + this.input.policyReplayCapture === undefined || promptSha256 === undefined + ? undefined + : { + runId: opts.runId, + iter: opts.iter, + kind: "reviewer", + ordinal: logicalSlot, + slot: logicalSlot, + attempt, + occurrence: 0, + }; // #7: clamp this reviewer's per-run timeout to the triage cap for a small diff (never // ABOVE the provider's own timeout). The full panel still runs — only the wall-clock // ceiling drops, so a tiny change can't stall behind one slow slot for the full default. @@ -1684,6 +1722,9 @@ export class Orchestrator { provider, persona, model, + attempt, + ...(promptSha256 === undefined ? {} : { promptSha256 }), + ...(policyReplayCall === undefined ? {} : { policyReplayCall }), }; } // Build a per-reviewer sandbox profile and forward { profile, mode } so the @@ -1728,8 +1769,18 @@ export class Orchestrator { diffPath, ...(opts.signal ? { signal: opts.signal } : {}), ...(sandbox ? { sandbox } : {}), + ...(policyReplayCall === undefined ? {} : { policyReplayCall }), }); - return { res, provider, persona, model, budgetCapped }; + return { + res, + provider, + persona, + model, + budgetCapped, + attempt, + ...(promptSha256 === undefined ? {} : { promptSha256 }), + ...(policyReplayCall === undefined ? {} : { policyReplayCall }), + }; } catch (err) { // strict + isolation-unavailable: fail closed for this reviewer with a // legible statusDetail, so the "0 ok reviewers → ERROR/block" gate handles @@ -1754,6 +1805,9 @@ export class Orchestrator { persona, model, budgetCapped, + attempt, + ...(promptSha256 === undefined ? {} : { promptSha256 }), + ...(policyReplayCall === undefined ? {} : { policyReplayCall }), }; } }; @@ -1794,7 +1848,7 @@ export class Orchestrator { } const tasks = panelReviewers.map( - async (r): Promise<{ run: ReviewerRun; effects: CooldownEffect[] } | null> => { + async (r, logicalSlot): Promise<{ run: ReviewerRun; effects: CooldownEffect[] } | null> => { const adapter = this.input.adapters[r.provider]; const providerCfg = this.input.config.providers[r.provider] as ProviderConfig | undefined; if (!adapter || !providerCfg || !providerCfg.enabled) return null; @@ -1813,6 +1867,7 @@ export class Orchestrator { // default perms — it MUST be removed even on a thrown adapter, else every // review leaks a /tmp dir with the (untrusted) diff in it. try { + let actualAttempt = 0; const promptFile = join(runDir, "prompt.txt"); const findingsPath = join(runDir, "findings.md"); const diffPath = join(runDir, "diff.patch"); @@ -1997,6 +2052,8 @@ export class Orchestrator { findingsPath, diffPath, runDir, + ++actualAttempt, + logicalSlot, ); const eff = effectFor(r.provider, run.res, run.budgetCapped); if (eff) effects.push(eff); @@ -2037,6 +2094,8 @@ export class Orchestrator { findingsPath, diffPath, runDir, + ++actualAttempt, + logicalSlot, ); run.res.statusDetail = `[fallback from ${fromProvider}: ${fromStatus}] ${run.res.statusDetail ?? ""}` @@ -2086,6 +2145,8 @@ export class Orchestrator { findingsPath, diffPath, runDir, + ++actualAttempt, + logicalSlot, ); run.res.statusDetail = `[last-resort from ${fromProvider}: ${fromStatus}] ${run.res.statusDetail ?? ""}` @@ -2264,9 +2325,68 @@ export class Orchestrator { .filter((f) => !isExcludedFromReview(f.file)); const symbolFindings = await this.applySymbolSignatures(rawFindings); const reviewerResponseHashes = new OrderedResponseHashes(); + const responseCalls: PolicyReplayEnvelopeInput["response_calls"] = []; + const recordResponseCall = (input: { + kind: "reviewer" | "grounding" | "critic"; + provider: ProviderId; + method: "review" | "complete"; + key: string; + promptSha256: string; + ordinal: number; + slot: number; + attempt: number; + occurrence: number; + responseSha256: string; + }): void => { + responseCalls.push({ + call_id: policyReplayCallId({ + runId: opts.runId, + iter: opts.iter, + kind: input.kind, + provider: input.provider, + method: input.method, + key: input.key, + promptSha256: input.promptSha256, + ordinal: input.ordinal, + slot: input.slot, + attempt: input.attempt, + occurrence: input.occurrence, + }), + kind: input.kind, + provider: input.provider, + method: input.method, + key: input.key, + prompt_sha256: input.promptSha256, + ordinal: input.ordinal, + slot: input.slot, + attempt: input.attempt, + occurrence: input.occurrence, + response_sha256: input.responseSha256, + }); + }; if (policyExecution.trace !== "off") { for (const [ordinal, run] of settled.entries()) { reviewerResponseHashes.record(`reviewer:${run.provider}`, ordinal, run.res.rawText); + if ( + run.res.rawText !== undefined && + run.promptSha256 !== undefined && + run.policyReplayCall !== undefined + ) { + recordResponseCall({ + kind: "reviewer", + provider: run.provider, + method: "review", + key: reviewKey(run.res.reviewerId), + promptSha256: run.promptSha256, + ordinal: run.policyReplayCall.ordinal, + slot: run.policyReplayCall.slot, + attempt: run.policyReplayCall.attempt, + occurrence: run.policyReplayCall.occurrence, + responseSha256: createHash("sha256") + .update(Buffer.from(run.res.rawText, "utf8")) + .digest("hex"), + }); + } } } const rawResponseSha256 = reviewerResponseHashes.values(); @@ -2340,6 +2460,14 @@ export class Orchestrator { | ProviderConfig | undefined; if (gAdapter && gProviderCfg) { + const groundingPromptSha256 = createHash("sha256") + .update( + buildGroundingJudgePrompt( + groundedFindings.filter((finding) => finding.severity === "CRITICAL"), + groundingCorpus, + ), + ) + .digest("hex"); const { map, status: groundingStatus, @@ -2352,6 +2480,19 @@ export class Orchestrator { ...(gProviderCfg.auth ? { auth: gProviderCfg.auth } : {}), timeoutMs: gProviderCfg.timeoutMs, ...(opts.signal ? { signal: opts.signal } : {}), + ...(this.input.policyReplayCapture === undefined + ? {} + : { + policyReplayCall: { + runId: opts.runId, + iter: opts.iter, + kind: "grounding" as const, + ordinal: panelReviewers.length, + slot: 0, + attempt: 1, + occurrence: 0, + }, + }), ...(gProviderCfg.openrouterProvider ? { openrouterProvider: gProviderCfg.openrouterProvider } : {}), @@ -2364,6 +2505,18 @@ export class Orchestrator { groundingStatus === "ran" ? "ran" : groundingStatus === "error" ? "error" : "not-run"; if (groundingResponseSha256 !== undefined) { rawResponseSha256.push(groundingResponseSha256); + recordResponseCall({ + kind: "grounding", + provider: groundingCfg.provider, + method: "complete", + key: completeKey(groundingCfg.provider, groundingPromptSha256), + promptSha256: groundingPromptSha256, + ordinal: panelReviewers.length, + slot: 0, + attempt: 1, + occurrence: 0, + responseSha256: groundingResponseSha256, + }); groundedFindings = applyGroundingJudgeVerdicts(groundedFindings, map, policyRuntime); } else { policyRuntime?.markInactive("judgment.grounding-llm", "stage-precondition-miss"); @@ -2417,8 +2570,38 @@ export class Orchestrator { // and makes the critic a silent no-op. No cost is attributed: complete() // returns only text (no usage envelope), so the critic phase is $0 here. criticAttempted = true; + const criticPromptSha256 = createHash("sha256") + .update(buildCriticPrompt(groundedFindings)) + .digest("hex"); + let criticPhysicalAttempt = 0; + const successfulCriticCalls: PolicyReplayCallContext[] = []; + const replayCriticAdapter: Pick = + this.input.policyReplayCapture === undefined || + typeof criticAdapter.complete !== "function" + ? criticAdapter + : { + complete: async (prompt, completeOptions) => { + const attempt = ++criticPhysicalAttempt; + const policyReplayCall: PolicyReplayCallContext = { + runId: opts.runId, + iter: opts.iter, + kind: "critic", + ordinal: panelReviewers.length + attempt, + slot: 0, + attempt, + occurrence: attempt - 1, + }; + const text = await criticAdapter.complete?.call(criticAdapter, prompt, { + ...completeOptions, + policyReplayCall, + }); + if (text === undefined) throw new Error("critic completion unavailable"); + successfulCriticCalls.push(policyReplayCall); + return text; + }, + }; const r = await runCritic( - criticAdapter, + replayCriticAdapter, criticCfg.provider, { model: criticCfg.model ?? cProviderCfg.model, @@ -2437,9 +2620,40 @@ export class Orchestrator { criticMap = r.map; criticInfo = r.info; if (r.rawResponseSha256s !== undefined) { - rawResponseSha256.push(...r.rawResponseSha256s); + for (const [index, responseSha256] of r.rawResponseSha256s.entries()) { + rawResponseSha256.push(responseSha256); + const policyReplayCall = successfulCriticCalls[index]; + if (policyReplayCall === undefined) continue; + recordResponseCall({ + kind: "critic", + provider: criticCfg.provider, + method: "complete", + key: completeKey(criticCfg.provider, criticPromptSha256), + promptSha256: criticPromptSha256, + ordinal: policyReplayCall.ordinal, + slot: policyReplayCall.slot, + attempt: policyReplayCall.attempt, + occurrence: policyReplayCall.occurrence, + responseSha256, + }); + } } else if (r.rawResponseSha256 !== undefined) { rawResponseSha256.push(r.rawResponseSha256); + const policyReplayCall = successfulCriticCalls[0]; + if (policyReplayCall !== undefined) { + recordResponseCall({ + kind: "critic", + provider: criticCfg.provider, + method: "complete", + key: completeKey(criticCfg.provider, criticPromptSha256), + promptSha256: criticPromptSha256, + ordinal: policyReplayCall.ordinal, + slot: policyReplayCall.slot, + attempt: policyReplayCall.attempt, + occurrence: policyReplayCall.occurrence, + responseSha256: r.rawResponseSha256, + }); + } } } else { criticInfo = { provider: criticCfg.provider, status: "misconfigured", verdicts: 0 }; @@ -2619,12 +2833,14 @@ export class Orchestrator { // outcomes so downstream learners have signal. NEVER changes the verdict or // report — a failure here is swallowed. const ioCfg = this.input.config.phases.implicitOutcomes; + let implicitOutcomeCreatedAt: string | null = null; if (ioCfg?.enabled) { try { + implicitOutcomeCreatedAt = new Date().toISOString(); const outcomes = deriveImplicitOutcomes(agg.dedupedFindings, agg.criticDropped, { runId: opts.runId, iter: opts.iter, - nowIso: new Date().toISOString(), + nowIso: implicitOutcomeCreatedAt, }); await new ImplicitOutcomeStore(repo).append(outcomes, ioCfg.cap); } catch (err) { @@ -2847,6 +3063,39 @@ export class Orchestrator { }, state_sha256: policyReplayStateSha256, raw_response_sha256: [...rawResponseSha256], + response_calls: responseCalls, + history: { + fp_ledger: + fpStore === null + ? { enabled: false } + : { + enabled: true, + active_at: fpObservedAt.toISOString(), + clusters_at: now.toISOString(), + }, + reputation: + repCfg?.enabled !== true + ? { enabled: false } + : { + enabled: true, + observed_at: now.toISOString(), + min_samples: repCfg.minSamples, + trust_floor: repCfg.trustFloor, + half_life_days: repCfg.halfLifeDays, + }, + cycle_state: { + source: "state.json", + region_rejected_enabled: activeRegions !== null, + }, + implicit_outcomes: + ioCfg?.enabled !== true || implicitOutcomeCreatedAt === null + ? { enabled: false } + : { + enabled: true, + cap: ioCfg.cap, + created_at: implicitOutcomeCreatedAt, + }, + }, policy_trace: policyTrace, lossless: true, }, diff --git a/src/providers/adapter-base.ts b/src/providers/adapter-base.ts index 3c8db27..d94bb47 100644 --- a/src/providers/adapter-base.ts +++ b/src/providers/adapter-base.ts @@ -49,6 +49,21 @@ export interface Preflight { error: string | null; } +/** Internal recording-only identity. It carries no prompt, path, config, environment, or secret. */ +export interface PolicyReplayCallContext { + runId: string; + iter: number; + kind: "reviewer" | "grounding" | "critic"; + /** Sparse logical response order across the policy iteration. */ + ordinal: number; + /** Kind-local logical call slot (for example the configured reviewer slot). */ + slot: number; + /** Physical provider attempt within the logical slot. */ + attempt: number; + /** Repeated invocation of the same logical identity. */ + occurrence: number; +} + export interface ReviewInput { promptFile: string; /** Optional in-memory prompt used by capability probes that must not create a @@ -71,6 +86,8 @@ export interface ReviewInput { /** Execute at most one physical provider invocation. Benchmark call ceilings * set this so an adapter-local retry cannot escape the recorded budget. */ disableRetries?: boolean | undefined; + /** Internal Rig recording metadata; adapters other than RecordingAdapter ignore it. */ + policyReplayCall?: PolicyReplayCallContext | undefined; } export type ReviewStatus = "ok" | "error" | "abstain" | "timeout" | "quota-exhausted"; @@ -139,6 +156,8 @@ export interface CompleteOptions { // grounding lookup) reasoning adds no value, so disabling it makes the output // deterministic and cheap. OpenRouter honors this via `reasoning:{enabled:false}`. disableReasoning?: boolean; + /** Internal Rig recording metadata; adapters other than RecordingAdapter ignore it. */ + policyReplayCall?: PolicyReplayCallContext | undefined; } export interface ProviderAdapter { diff --git a/src/rig/policy-replay-state.ts b/src/rig/policy-replay-state.ts index 18a78d0..2198eb2 100644 --- a/src/rig/policy-replay-state.ts +++ b/src/rig/policy-replay-state.ts @@ -26,6 +26,7 @@ import { CassetteEntrySchema } from "../schemas/cassette.ts"; import type { PolicyReplayEnvelope } from "../schemas/policy-replay.ts"; import type { RigManifest } from "../schemas/rig-manifest.ts"; import { writeFileIfAbsent } from "../utils/atomic-write.ts"; +import { compareCodeUnits } from "../utils/compare.ts"; const STATE_MAX_FILE_BYTES = 8 * 1024 * 1024; const STATE_MAX_TOTAL_BYTES = 64 * 1024 * 1024; @@ -70,7 +71,10 @@ const PolicyStateManifestSchema = z message: "invalid state path", }); } - if (index > 0 && (value.files[index - 1]?.path ?? "") >= (entry?.path ?? "")) { + if ( + index > 0 && + compareCodeUnits(value.files[index - 1]?.path ?? "", entry?.path ?? "") >= 0 + ) { ctx.addIssue({ code: z.ZodIssueCode.custom, path: ["files", index, "path"], @@ -216,7 +220,7 @@ function readStableFile( } } -function collectStateEntries(stateRoot: string): StateEntry[] { +function collectStateEntries(stateRoot: string, requireMode0600 = false): StateEntry[] { const rootStat = lstatSync(stateRoot); if (rootStat.isSymbolicLink()) throw new Error(`policy state root is a symlink: ${stateRoot}`); if (!rootStat.isDirectory()) @@ -236,7 +240,7 @@ function collectStateEntries(stateRoot: string): StateEntry[] { if (!isContained(rootReal, directoryReal)) { throw new Error(`policy state directory escapes root: ${directory}`); } - const names = readdirSync(directory).sort((a, b) => a.localeCompare(b)); + const names = readdirSync(directory).sort(compareCodeUnits); for (const name of names) { const path = join(directory, name); const stat = lstatSync(path); @@ -248,7 +252,7 @@ function collectStateEntries(stateRoot: string): StateEntry[] { if (!stat.isFile()) throw new Error(`policy state contains special file: ${path}`); const rel = relative(rootReal, realpathSync(path)).split(sep).join("/"); validateRelativeStatePath(rel); - const bytes = readStableFile(path); + const bytes = readStableFile(path, STATE_MAX_FILE_BYTES, requireMode0600); totalBytes += bytes.length; if (entries.length + 1 > STATE_MAX_FILES) throw new Error("policy state exceeds file limit"); if (totalBytes > STATE_MAX_TOTAL_BYTES) throw new Error("policy state exceeds byte limit"); @@ -257,7 +261,7 @@ function collectStateEntries(stateRoot: string): StateEntry[] { }; visit(rootReal); - return entries.sort((a, b) => a.path.localeCompare(b.path)); + return entries.sort((a, b) => compareCodeUnits(a.path, b.path)); } function stateDigest(entries: StateEntry[]): string { @@ -284,21 +288,70 @@ function exactDirectory(path: string): string { return realpathSync(path); } +function mkdirIfMissing(path: string): void { + if (existsSync(path)) return; + try { + mkdirSync(path, { mode: 0o700 }); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "EEXIST") throw error; + } +} + +function ensureContainedDirectory( + rootPath: string, + rootReal: string, + parent: string, + name: string, +): string { + if (name.length === 0 || name === "." || name === ".." || name.includes(sep)) { + throw new Error(`invalid policy state directory component: ${name}`); + } + const parentReal = exactDirectory(parent); + if (!isContained(rootReal, parentReal)) { + throw new Error(`policy state directory parent escapes root: ${parent}`); + } + const path = join(parent, name); + if (!isContained(rootPath, path)) throw new Error(`policy state directory escapes root: ${path}`); + mkdirIfMissing(path); + const pathReal = exactDirectory(path); + if (!isContained(rootReal, pathReal)) { + throw new Error(`policy state directory escapes real root: ${path}`); + } + return path; +} + +function ensureDirectoryChain(rootPath: string, components: string[]): string { + const rootReal = exactDirectory(rootPath); + let current = rootReal; + for (const component of components) { + current = ensureContainedDirectory(rootReal, rootReal, current, component); + } + return current; +} + +function ensureRelativeParent(destinationRoot: string, relativePath: string): string { + const components = relativePath.split("/").slice(0, -1); + return ensureDirectoryChain(destinationRoot, components); +} + function copyEntries(entries: StateEntry[], destinationRoot: string): void { - mkdirSync(destinationRoot, { recursive: true, mode: 0o700 }); - const destinationReal = realpathSync(destinationRoot); + const destinationReal = exactDirectory(destinationRoot); for (const entry of entries) { validateRelativeStatePath(entry.path); const destination = resolve(destinationReal, entry.path); if (!isContained(destinationReal, destination)) { throw new Error(`policy state copy escapes destination: ${entry.path}`); } - mkdirSync(dirname(destination), { recursive: true, mode: 0o700 }); + ensureRelativeParent(destinationReal, entry.path); writeFileSync(destination, entry.bytes, { flag: "wx", mode: 0o600 }); const copied = lstatSync(destination); if (!copied.isFile() || copied.isSymbolicLink() || copied.nlink !== 1) { throw new Error(`policy state copy did not produce a private file: ${entry.path}`); } + const copiedBytes = readStableFile(destination, STATE_MAX_FILE_BYTES, true); + if (!copiedBytes.equals(entry.bytes)) { + throw new Error(`policy state copy changed bytes: ${entry.path}`); + } } } @@ -326,8 +379,8 @@ function advanceBranchState(input: { ); const branchStateRoot = join(input.checkoutRoot, ".reviewgate"); const branch = new Map(collectStateEntries(branchStateRoot).map((entry) => [entry.path, entry])); - const paths = [...new Set([...previous.keys(), ...next.keys(), ...branch.keys()])].sort((a, b) => - a.localeCompare(b), + const paths = [...new Set([...previous.keys(), ...next.keys(), ...branch.keys()])].sort( + compareCodeUnits, ); const merged: StateEntry[] = []; for (const path of paths) { @@ -345,6 +398,7 @@ function advanceBranchState(input: { if (selected !== undefined) merged.push(selected); } rmSync(branchStateRoot, { recursive: true, force: true }); + ensureDirectoryChain(input.checkoutRoot, [".reviewgate"]); copyEntries(merged, branchStateRoot); } @@ -386,8 +440,13 @@ export function createPolicyStateSnapshot(input: { const stateDestination = resolve(outputReal, stateRef); if (!isContained(outputReal, stateDestination)) throw new Error("policy state output escapes root"); - if (!existsSync(stateDestination)) copyEntries(entries, stateDestination); - if (digestPolicyState(stateDestination) !== stateSha256) { + if (!existsSync(stateDestination)) { + ensureDirectoryChain(outputReal, ["policy-state", stateSha256, ".reviewgate"]); + copyEntries(entries, stateDestination); + } else { + exactDirectory(stateDestination); + } + if (stateDigest(collectStateEntries(stateDestination, true)) !== stateSha256) { throw new Error("policy state snapshot digest mismatch"); } @@ -407,7 +466,7 @@ export function createPolicyStateSnapshot(input: { if (!STATE_MANIFEST_REF.test(ref)) throw new Error("invalid policy state manifest reference"); const destination = resolve(outputReal, ref); if (!isContained(outputReal, destination)) throw new Error("policy state manifest escapes root"); - mkdirSync(dirname(destination), { recursive: true, mode: 0o700 }); + ensureDirectoryChain(outputReal, ["policy-state"]); if (!writeFileIfAbsent(destination, bytes, { mode: 0o600 })) { const existing = readStableFile(destination, STATE_MAX_FILE_BYTES, true); if (!existing.equals(Buffer.from(bytes, "utf8"))) { @@ -461,7 +520,7 @@ export function verifyPolicyStateSnapshot(input: { ) { throw new Error("policy state tree escapes root"); } - const entries = collectStateEntries(stateRoot); + const entries = collectStateEntries(stateRoot, true); if (stateDigest(entries) !== manifest.state_sha256) throw new Error("policy state tree digest mismatch"); const actualFiles = entries.map(({ path, size, sha256: contentSha256 }) => ({ @@ -479,14 +538,32 @@ function authority(code: RigAuthorityInvalidity, message: string): never { throw new RigAuthorityError(code, message); } -function responseHashesFromCassette(bytes: Buffer): string[] { +interface CassetteResponseBinding { + provider: string; + method: "review" | "complete"; + key: string; + promptSha256: string; + responseSha256: string | null; + policyReplayCall?: { + callId: string; + runId: string; + iter: number; + kind: "reviewer" | "grounding" | "critic"; + ordinal: number; + slot: number; + attempt: number; + occurrence: number; + }; +} + +function responseBindingsFromCassette(bytes: Buffer): CassetteResponseBinding[] { let text: string; try { text = new TextDecoder("utf-8", { fatal: true }).decode(bytes); } catch { return authority("invalid-cassette", "cassette is not valid UTF-8"); } - const hashes: string[] = []; + const bindings: CassetteResponseBinding[] = []; for (const [index, line] of text.split("\n").entries()) { if (line.trim().length === 0) continue; let decoded: unknown; @@ -500,12 +577,23 @@ function responseHashesFromCassette(bytes: Buffer): string[] { return authority("invalid-cassette", `cassette line ${index + 1} is malformed`); } const entry = parsed.data; + if (entry.method === "embed") continue; + if (!/^[0-9a-f]{64}$/.test(entry.promptSha256)) { + return authority("invalid-cassette", `cassette line ${index + 1} has an invalid prompt hash`); + } let raw: string | undefined; if (entry.method === "review" && "rawText" in entry.result) raw = entry.result.rawText; if (entry.method === "complete" && "text" in entry.result) raw = entry.result.text; - if (raw !== undefined) hashes.push(sha256(raw)); + bindings.push({ + provider: entry.provider, + method: entry.method, + key: entry.key, + promptSha256: entry.promptSha256, + responseSha256: raw === undefined ? null : sha256(raw), + ...(entry.policyReplayCall === undefined ? {} : { policyReplayCall: entry.policyReplayCall }), + }); } - return hashes; + return bindings; } /** Validate every artifact before authoritative harvest/replay is allowed to count anything. */ @@ -553,8 +641,16 @@ export function validateRigPolicyReplayArtifacts(input: { if (sha256(cassetteBytes) !== metadata.cassetteSha256) { return authority("cassette-hash-mismatch", "cassette bytes do not match the manifest"); } - const cassetteResponseHashes = responseHashesFromCassette(cassetteBytes); - const requiredResponseHashes: string[] = []; + const cassetteResponses = responseBindingsFromCassette(cassetteBytes); + const cassetteByCallId = new Map(); + for (const response of cassetteResponses) { + const callId = response.policyReplayCall?.callId; + if (callId === undefined) continue; + if (cassetteByCallId.has(callId)) { + return authority("invalid-cassette", `duplicate policy replay call id ${callId}`); + } + cassetteByCallId.set(callId, response); + } const turns = new Map< number, Array<{ ref: string; sha256: string; envelope: PolicyReplayEnvelope; stateRoot: string }> @@ -622,7 +718,7 @@ export function validateRigPolicyReplayArtifacts(input: { try { if ( !isContained(outputRoot, stateRoot) || - digestPolicyState(stateRoot) !== envelope.state_sha256 + stateDigest(collectStateEntries(stateRoot, true)) !== envelope.state_sha256 ) { return authority( "state-digest-mismatch", @@ -632,7 +728,32 @@ export function validateRigPolicyReplayArtifacts(input: { } catch (error) { return authority("state-digest-mismatch", String(error)); } - requiredResponseHashes.push(...envelope.raw_response_sha256); + for (const call of envelope.response_calls) { + const recorded = cassetteByCallId.get(call.call_id); + const metadata = recorded?.policyReplayCall; + if ( + recorded === undefined || + metadata === undefined || + recorded.provider !== call.provider || + recorded.method !== call.method || + recorded.key !== call.key || + recorded.promptSha256 !== call.prompt_sha256 || + metadata.runId !== envelope.run_id || + metadata.iter !== envelope.iter || + metadata.kind !== call.kind || + metadata.ordinal !== call.ordinal || + metadata.slot !== call.slot || + metadata.attempt !== call.attempt || + metadata.occurrence !== call.occurrence || + recorded.responseSha256 === null || + recorded.responseSha256 !== call.response_sha256 + ) { + return authority( + "response-hash-mismatch", + `logical response call ${call.call_id} does not exactly match its cassette recording`, + ); + } + } validated.push({ ...trace, envelope, stateRoot }); } const sequenceRunId = validated[0]?.envelope.run_id; @@ -651,16 +772,6 @@ export function validateRigPolicyReplayArtifacts(input: { turns.set(turn.index, validated); } - if ( - cassetteResponseHashes.length !== requiredResponseHashes.length || - requiredResponseHashes.some((hash, index) => hash !== cassetteResponseHashes[index]) - ) { - return authority( - "response-hash-mismatch", - "cassette response hashes do not exactly match the captured order", - ); - } - return { sourceCommit: metadata.sourceCommit, initialStateRoot: initial.stateRoot, @@ -712,7 +823,8 @@ function prepareBranch(input: { }); const stateDestination = join(input.destination, ".reviewgate"); if (existsSync(stateDestination)) rmSync(stateDestination, { recursive: true, force: true }); - const entries = collectStateEntries(input.stateSnapshotRoot); + ensureDirectoryChain(exactDirectory(input.destination), [".reviewgate"]); + const entries = collectStateEntries(input.stateSnapshotRoot, true); if (stateDigest(entries) !== input.expectedStateSha256) { throw new Error("policy state snapshot digest mismatch"); } @@ -757,7 +869,7 @@ export function createReplayBranches(input: { }, ).trim(); if (resolvedCommit !== input.sourceCommit) throw new Error("source commit identity mismatch"); - if (digestPolicyState(stateReal) !== input.expectedStateSha256) { + if (stateDigest(collectStateEntries(stateReal, true)) !== input.expectedStateSha256) { throw new Error("policy state snapshot digest mismatch"); } @@ -816,10 +928,15 @@ export function advanceReplayBranches(input: { throw new Error("policy state transition snapshot aliases source or replay state"); } } - if (digestPolicyState(input.previousStateSnapshotRoot) !== input.previousStateSha256) { + if ( + stateDigest(collectStateEntries(input.previousStateSnapshotRoot, true)) !== + input.previousStateSha256 + ) { throw new Error("previous policy state snapshot digest mismatch"); } - if (digestPolicyState(input.nextStateSnapshotRoot) !== input.nextStateSha256) { + if ( + stateDigest(collectStateEntries(input.nextStateSnapshotRoot, true)) !== input.nextStateSha256 + ) { throw new Error("next policy state snapshot digest mismatch"); } for (const [label, branch] of [ @@ -851,12 +968,6 @@ export function advanceReplayBranches(input: { nextStateSnapshotRoot: input.nextStateSnapshotRoot, }); } - if ( - digestPolicyState(join(input.branches.baseline.checkoutRoot, ".reviewgate")) !== - input.nextStateSha256 - ) { - throw new Error("baseline replay state does not reproduce the next captured digest"); - } assertNoAliasedFiles( join(input.branches.baseline.checkoutRoot, ".reviewgate"), join(input.branches.counterfactual.checkoutRoot, ".reviewgate"), diff --git a/src/rig/replay.ts b/src/rig/replay.ts index a7f5482..a29d1e1 100644 --- a/src/rig/replay.ts +++ b/src/rig/replay.ts @@ -3,21 +3,34 @@ // checkouts. Legacy runs retain the older deterministic harvest/heuristic self-check, explicitly // non-authoritative for policy ablation rather than pretending missing opportunities were zero. import { existsSync, readFileSync } from "node:fs"; +import { join } from "node:path"; import { canonicalJson } from "../audit/canonical.ts"; import { type AggregateInput, aggregate } from "../core/aggregator.ts"; import { validateFindingFacts } from "../core/fact-check.ts"; +import { computeFpClusters } from "../core/fp-ledger/clusters.ts"; +import { learnFromDecisions } from "../core/fp-ledger/learn.ts"; +import { FpLedgerStore } from "../core/fp-ledger/store.ts"; import { applyGroundingJudgeVerdicts, groundFindings } from "../core/grounding.ts"; import { demoteHypotheticalCriticals } from "../core/hypothetical-demote.ts"; +import { + ImplicitOutcomeStore, + deriveImplicitOutcomes, +} from "../core/learnings/implicit-outcomes.ts"; import type { PolicyPassId } from "../core/policy/catalog.ts"; import { POLICY_PASS_IDS } from "../core/policy/catalog.ts"; import { PolicyTraceRecorder } from "../core/policy/trace.ts"; +import { mergeRegions } from "../core/region-memory.ts"; +import { learnReputationFromDecisions } from "../core/reputation/learn.ts"; +import { ReputationStore } from "../core/reputation/store.ts"; import { demoteSelfRefuting } from "../core/self-refutation.ts"; +import { StateStore } from "../core/state-store.ts"; import { parseDeletedPaths } from "../diff/hunks.ts"; import { CassetteEntrySchema } from "../schemas/cassette.ts"; import type { PolicyReplayEnvelope } from "../schemas/policy-replay.ts"; import type { PolicyTrace } from "../schemas/policy-trace.ts"; import { RigManifestSchema } from "../schemas/rig-manifest.ts"; import type { RigResult } from "../schemas/rig-result.ts"; +import { compareCodeUnits } from "../utils/compare.ts"; import { type RigAblation, SUPPRESSION_LAYERS, ablate, seededTagsFromScript } from "./ablate.ts"; import { harvest } from "./harvest.ts"; import { @@ -26,6 +39,7 @@ import { advanceReplayBranches, cleanupReplayBranches, createReplayBranches, + digestPolicyState, validateRigPolicyReplayArtifacts, } from "./policy-replay-state.ts"; @@ -56,6 +70,17 @@ export interface ReplayReport { export interface PolicyReplayPair { baseline: PolicyTrace; counterfactual: PolicyTrace; + state: { + baseline: PolicyReplayBranchStateEvidence; + counterfactual: PolicyReplayBranchStateEvidence; + }; +} + +export interface PolicyReplayBranchStateEvidence { + digest: string; + implicit_outcomes: number; + history_reads: number; + history_writes: number; } export interface PolicyReplaySequenceItem { @@ -63,13 +88,6 @@ export interface PolicyReplaySequenceItem { stateSnapshotRoot: string; } -export interface PolicyReplaySequenceStep { - index: number; - item: PolicyReplaySequenceItem; - branches: ReplayBranches; - pair: PolicyReplayPair; -} - export interface RigPolicyAblationRow { passId: PolicyPassId; authoritative: boolean; @@ -101,14 +119,150 @@ export function runWithReplayProviderCeiling(operation: () => T): T { }; globalThis.fetch = (() => reject("a network call")) as unknown as typeof fetch; bunRuntime.spawn = () => reject("a provider subprocess call"); - try { - return operation(); - } finally { + const restore = (): void => { globalThis.fetch = originalFetch; bunRuntime.spawn = originalSpawn; + }; + try { + const result = operation(); + if (result instanceof Promise) { + return result.finally(restore) as T; + } + restore(); + return result; + } catch (error) { + restore(); + throw error; } } +function historyMismatch(label: string): never { + throw new RigAuthorityError( + "state-digest-mismatch", + `branch-local production history does not match captured ${label}`, + ); +} + +async function assertBranchHistoryInputs( + envelope: PolicyReplayEnvelope, + checkoutRoot: string, +): Promise { + let reads = 0; + if (envelope.history.fp_ledger.enabled) { + const store = new FpLedgerStore(checkoutRoot); + const full = await store.snapshot(); + reads += 1; + const active = await store.activeSnapshot(new Date(envelope.history.fp_ledger.active_at)); + reads += 1; + const actualActive = [...active] + .map(([signature, value]) => ({ signature, id: value.id })) + .sort((left, right) => compareCodeUnits(left.signature, right.signature)); + const actualClusters = computeFpClusters(full.entries, envelope.history.fp_ledger.clusters_at) + .filter((cluster) => cluster.stage === "active" || cluster.stage === "sticky") + .map((cluster) => ({ + key: cluster.key, + member_ids: [...cluster.member_ids].sort(compareCodeUnits), + })) + .sort((left, right) => compareCodeUnits(left.key, right.key)); + if (canonicalJson(actualActive) !== canonicalJson(envelope.aggregate.fp_active)) { + historyMismatch("fp-ledger active signatures"); + } + if (canonicalJson(actualClusters) !== canonicalJson(envelope.aggregate.fp_active_clusters)) { + historyMismatch("fp-ledger clusters"); + } + } else if ( + envelope.aggregate.fp_active.length > 0 || + envelope.aggregate.fp_active_clusters.length > 0 + ) { + historyMismatch("disabled fp-ledger inputs"); + } + + if (envelope.history.reputation.enabled) { + const config = envelope.history.reputation; + const unreliable = await new ReputationStore(checkoutRoot).unreliableReviewers( + { + enabled: true, + minSamples: config.min_samples, + trustFloor: config.trust_floor, + halfLifeDays: config.half_life_days, + }, + new Date(config.observed_at), + ); + reads += 1; + const actual = [...unreliable].sort(compareCodeUnits); + if (canonicalJson(actual) !== canonicalJson(envelope.aggregate.rep_unreliable)) { + historyMismatch("reviewer reputation"); + } + } else if (envelope.aggregate.rep_unreliable.length > 0) { + historyMismatch("disabled reputation inputs"); + } + + const state = await new StateStore(checkoutRoot).load(); + reads += 1; + const cycleRejected = [...state.cycle_rejected_signatures].sort(compareCodeUnits); + const claimedFixed = Object.entries(state.claimed_fixed_signatures) + .map(([signature, iter]) => ({ signature, iter })) + .sort((left, right) => compareCodeUnits(left.signature, right.signature)); + const rejectedRegions = envelope.history.cycle_state.region_rejected_enabled + ? mergeRegions(state.cycle_rejected_dispositions) + .map((region) => ({ + ...region, + categories: [...region.categories].sort(compareCodeUnits), + })) + .sort( + (left, right) => + compareCodeUnits(left.file, right.file) || + left.start_line - right.start_line || + left.end_line - right.end_line, + ) + : []; + if (canonicalJson(cycleRejected) !== canonicalJson(envelope.aggregate.cycle_rejected)) { + historyMismatch("cycle-rejected signatures"); + } + if (canonicalJson(claimedFixed) !== canonicalJson(envelope.aggregate.claimed_fixed)) { + historyMismatch("claimed-fixed signatures"); + } + if (canonicalJson(rejectedRegions) !== canonicalJson(envelope.aggregate.rejected_regions)) { + historyMismatch("cycle region memory"); + } + return reads; +} + +async function applyCapturedHumanLearning( + envelope: PolicyReplayEnvelope, + checkoutRoot: string, +): Promise { + const state = await new StateStore(checkoutRoot).load(); + if (state.iteration < 1) return 0; + let writes = 0; + if (envelope.history.fp_ledger.enabled) { + const store = new FpLedgerStore(checkoutRoot); + await learnFromDecisions({ + repoRoot: checkoutRoot, + prevIter: state.iteration, + sessionId: state.session_id, + cycleSeq: state.reputation_cycle_seq, + store, + nowIso: envelope.history.fp_ledger.active_at, + }); + await store.decayPass(envelope.history.fp_ledger.active_at); + writes += 1; + } + if (envelope.history.reputation.enabled) { + await learnReputationFromDecisions({ + repoRoot: checkoutRoot, + iter: state.iteration, + sessionId: state.session_id, + cycleSeq: state.reputation_cycle_seq, + store: new ReputationStore(checkoutRoot), + nowIso: envelope.history.reputation.observed_at, + halfLifeDays: envelope.history.reputation.half_life_days, + }); + writes += 1; + } + return writes; +} + function aggregateInputFromEnvelope( envelope: PolicyReplayEnvelope, findings: PolicyReplayEnvelope["aggregate"]["findings"], @@ -159,12 +313,18 @@ function aggregateInputFromEnvelope( }; } +interface ReplayPolicyExecution { + trace: PolicyTrace; + dedupedFindings: PolicyReplayEnvelope["policy_final_findings"]; + criticDropped: PolicyReplayEnvelope["policy_final_findings"]; +} + function replayEnvelopeProductionPath(input: { envelope: PolicyReplayEnvelope; checkoutRoot: string; ablated: ReadonlySet; verifyOriginal: boolean; -}): PolicyTrace { +}): ReplayPolicyExecution { const { envelope } = input; const runtime = PolicyTraceRecorder.start({ runId: envelope.run_id, @@ -262,44 +422,118 @@ function replayEnvelopeProductionPath(input: { canonicalJson(trace.final.finding_severities) === canonicalJson(expectedPolicySeverities); if (!equal) throw new Error("production baseline replay does not reproduce its policy trace"); } - return trace; + return { + trace, + dedupedFindings: result.dedupedFindings, + criticDropped: result.criticDropped, + }; +} + +async function persistBranchPolicyOutcomes(input: { + envelope: PolicyReplayEnvelope; + checkoutRoot: string; + execution: ReplayPolicyExecution; +}): Promise { + if (!input.envelope.history.implicit_outcomes.enabled) return 0; + const outcomes = deriveImplicitOutcomes( + input.execution.dedupedFindings, + input.execution.criticDropped, + { + runId: input.envelope.run_id, + iter: input.envelope.iter, + nowIso: input.envelope.history.implicit_outcomes.created_at, + }, + ); + await new ImplicitOutcomeStore(input.checkoutRoot).append( + outcomes, + input.envelope.history.implicit_outcomes.cap, + ); + return outcomes.length; +} + +async function branchStateEvidence(input: { + checkoutRoot: string; + historyReads: number; + historyWrites: number; +}): Promise { + return { + digest: digestPolicyState(join(input.checkoutRoot, ".reviewgate")), + implicit_outcomes: (await new ImplicitOutcomeStore(input.checkoutRoot).load()).length, + history_reads: input.historyReads, + history_writes: input.historyWrites, + }; } -function replayPolicyEnvelopeInBranches(input: { +async function replayPolicyEnvelopeInBranches(input: { envelope: PolicyReplayEnvelope; passId: PolicyPassId; branches: ReplayBranches; -}): PolicyReplayPair { - return runWithReplayProviderCeiling(() => { - const baseline = replayEnvelopeProductionPath({ + humanLearningWrites?: { baseline: number; counterfactual: number }; +}): Promise { + return runWithReplayProviderCeiling(async () => { + const humanLearningWrites = input.humanLearningWrites ?? { baseline: 0, counterfactual: 0 }; + const baselineReads = await assertBranchHistoryInputs( + input.envelope, + input.branches.baseline.checkoutRoot, + ); + const counterfactualReads = await assertBranchHistoryInputs( + input.envelope, + input.branches.counterfactual.checkoutRoot, + ); + const baselineExecution = replayEnvelopeProductionPath({ envelope: input.envelope, checkoutRoot: input.branches.baseline.checkoutRoot, ablated: new Set(), verifyOriginal: true, }); - const counterfactual = replayEnvelopeProductionPath({ + const counterfactualExecution = replayEnvelopeProductionPath({ envelope: input.envelope, checkoutRoot: input.branches.counterfactual.checkoutRoot, ablated: new Set([input.passId]), verifyOriginal: false, }); if ( - canonicalJson(baseline.raw_response_sha256) !== - canonicalJson(counterfactual.raw_response_sha256) + canonicalJson(baselineExecution.trace.raw_response_sha256) !== + canonicalJson(counterfactualExecution.trace.raw_response_sha256) ) { throw new Error("baseline and counterfactual ordered response hashes differ"); } - return { baseline, counterfactual }; + const baselineOutcomeWrites = await persistBranchPolicyOutcomes({ + envelope: input.envelope, + checkoutRoot: input.branches.baseline.checkoutRoot, + execution: baselineExecution, + }); + const counterfactualOutcomeWrites = await persistBranchPolicyOutcomes({ + envelope: input.envelope, + checkoutRoot: input.branches.counterfactual.checkoutRoot, + execution: counterfactualExecution, + }); + return { + baseline: baselineExecution.trace, + counterfactual: counterfactualExecution.trace, + state: { + baseline: await branchStateEvidence({ + checkoutRoot: input.branches.baseline.checkoutRoot, + historyReads: baselineReads, + historyWrites: humanLearningWrites.baseline + baselineOutcomeWrites, + }), + counterfactual: await branchStateEvidence({ + checkoutRoot: input.branches.counterfactual.checkoutRoot, + historyReads: counterfactualReads, + historyWrites: humanLearningWrites.counterfactual + counterfactualOutcomeWrites, + }), + }, + }; }); } /** One exact baseline/counterfactual pair. No adapter/provider capability enters this API. */ -export function replayPolicyEnvelopePair(input: { +export async function replayPolicyEnvelopePair(input: { sourceRepoRoot: string; envelope: PolicyReplayEnvelope; stateSnapshotRoot: string; passId: PolicyPassId; -}): PolicyReplayPair { +}): Promise { const branches = createReplayBranches({ sourceRepoRoot: input.sourceRepoRoot, sourceCommit: input.envelope.source_commit, @@ -308,7 +542,7 @@ export function replayPolicyEnvelopePair(input: { exactDiff: input.envelope.exact_diff, }); try { - return replayPolicyEnvelopeInBranches({ + return await replayPolicyEnvelopeInBranches({ envelope: input.envelope, passId: input.passId, branches, @@ -319,15 +553,13 @@ export function replayPolicyEnvelopePair(input: { } /** - * Replay an ordered multi-turn sequence in one persistent branch pair. The optional callback is - * the branch-local boundary used by existing production Store APIs; replay itself never invents a - * store transition or interprets a learning schema. + * Replay an ordered multi-turn sequence in one persistent branch pair. Production Store APIs own + * every branch-local read/write; replay neither exposes a mutation callback nor models learning. */ export async function replayPolicyEnvelopeSequence(input: { sourceRepoRoot: string; items: PolicyReplaySequenceItem[]; passId: PolicyPassId; - afterEnvelope?: (step: PolicyReplaySequenceStep) => void | Promise; }): Promise { const first = input.items[0]; if (first === undefined) { @@ -363,13 +595,26 @@ export async function replayPolicyEnvelopeSequence(input: { nextStateSha256: item.envelope.state_sha256, }); } - const pair = replayPolicyEnvelopeInBranches({ + const humanLearningWrites = + previous === undefined + ? { baseline: 0, counterfactual: 0 } + : { + baseline: await applyCapturedHumanLearning( + item.envelope, + branches.baseline.checkoutRoot, + ), + counterfactual: await applyCapturedHumanLearning( + item.envelope, + branches.counterfactual.checkoutRoot, + ), + }; + const pair = await replayPolicyEnvelopeInBranches({ envelope: item.envelope, passId: input.passId, branches, + humanLearningWrites, }); pairs.push(pair); - await input.afterEnvelope?.({ index, item, branches, pair }); } return pairs; } finally { @@ -380,6 +625,7 @@ export async function replayPolicyEnvelopeSequence(input: { export async function replayPolicyAblations(input: { manifestPath: string; sourceRepoRoot: string; + passId?: PolicyPassId; }): Promise { const manifest = RigManifestSchema.parse( JSON.parse(readFileSync(input.manifestPath, "utf8")) as unknown, @@ -400,7 +646,8 @@ export async function replayPolicyAblations(input: { stateSnapshotRoot: stateRoot, })); const rows: RigPolicyAblationRow[] = []; - for (const passId of POLICY_PASS_IDS) { + const requestedPasses = input.passId === undefined ? POLICY_PASS_IDS : [input.passId]; + for (const passId of requestedPasses) { let opportunities = 0; let applied = 0; let wouldApplyWithoutMutation = 0; diff --git a/src/schemas/cassette.ts b/src/schemas/cassette.ts index 9394129..b97ed37 100644 --- a/src/schemas/cassette.ts +++ b/src/schemas/cassette.ts @@ -14,6 +14,19 @@ export const ProviderIdSchema = z.enum([ const ReviewStatusSchema = z.enum(["ok", "error", "abstain", "timeout", "quota-exhausted"]); +export const PolicyReplayCassetteCallSchema = z + .object({ + callId: z.string().regex(/^[0-9a-f]{64}$/), + runId: z.string().min(1).max(128), + iter: z.number().int().positive(), + kind: z.enum(["reviewer", "grounding", "critic"]), + ordinal: z.number().int().nonnegative(), + slot: z.number().int().nonnegative(), + attempt: z.number().int().positive(), + occurrence: z.number().int().nonnegative(), + }) + .strict(); + // zod mirror of ReviewResult (src/providers/adapter-base.ts). rawEventsPath MAY be "" // (several adapters return empty) → plain z.string(), never non-empty. export const ReviewResultSchema = z.object({ @@ -48,6 +61,8 @@ export const CassetteEntrySchema = z key: z.string(), method: z.enum(["review", "complete", "embed"]), promptSha256: z.string(), + /** Additive: absent on legacy/non-authoritative recordings. */ + policyReplayCall: PolicyReplayCassetteCallSchema.optional(), result: z.union([ ReviewResultSchema, z.object({ text: z.string() }), diff --git a/src/schemas/policy-replay.ts b/src/schemas/policy-replay.ts index 00667d0..092c5a5 100644 --- a/src/schemas/policy-replay.ts +++ b/src/schemas/policy-replay.ts @@ -1,3 +1,4 @@ +import { createHash } from "node:crypto"; import { z } from "zod"; import { POLICY_CATALOG_VERSION, @@ -6,6 +7,7 @@ import { } from "../core/policy/catalog.ts"; import { compareCodeUnits } from "../utils/compare.ts"; import { isAuthoritativeThrowableString } from "./bench-result.ts"; +import { ProviderIdSchema } from "./cassette.ts"; import { FindingCategory, FindingSchema } from "./finding.ts"; import { PolicyTraceSchema } from "./policy-trace.ts"; @@ -13,6 +15,102 @@ const Sha256Schema = z.string().regex(/^[0-9a-f]{64}$/); const GitObjectIdSchema = z.string().regex(/^(?:[0-9a-f]{40}|[0-9a-f]{64})$/); const PolicyPassIdSchema = z.enum(POLICY_PASS_IDS); const PolicyInactiveReasonSchema = z.enum(["configured-off", "stage-precondition-miss"]); +const IsoTimestampSchema = z.string().datetime({ offset: true }); + +export interface PolicyReplayCallIdentityInput { + runId: string; + iter: number; + kind: "reviewer" | "grounding" | "critic"; + provider: string; + method: "review" | "complete"; + key: string; + promptSha256: string; + ordinal: number; + slot: number; + attempt: number; + occurrence: number; +} + +/** Stable logical-call identity; physical cassette append order is deliberately absent. */ +export function policyReplayCallId(input: PolicyReplayCallIdentityInput): string { + return createHash("sha256") + .update( + [ + input.runId, + String(input.iter), + input.kind, + input.provider, + input.method, + input.key, + input.promptSha256, + String(input.ordinal), + String(input.slot), + String(input.attempt), + String(input.occurrence), + ].join("\0"), + ) + .digest("hex"); +} + +const ResponseCallSchema = z + .object({ + call_id: Sha256Schema, + kind: z.enum(["reviewer", "grounding", "critic"]), + provider: ProviderIdSchema, + method: z.enum(["review", "complete"]), + key: z.string().min(1), + prompt_sha256: Sha256Schema, + ordinal: z.number().int().nonnegative(), + slot: z.number().int().nonnegative(), + attempt: z.number().int().positive(), + occurrence: z.number().int().nonnegative(), + response_sha256: Sha256Schema, + }) + .strict(); + +const DisabledHistoryStoreSchema = z.object({ enabled: z.literal(false) }).strict(); +const HistoryInputsSchema = z + .object({ + fp_ledger: z.discriminatedUnion("enabled", [ + DisabledHistoryStoreSchema, + z + .object({ + enabled: z.literal(true), + active_at: IsoTimestampSchema, + clusters_at: IsoTimestampSchema, + }) + .strict(), + ]), + reputation: z.discriminatedUnion("enabled", [ + DisabledHistoryStoreSchema, + z + .object({ + enabled: z.literal(true), + observed_at: IsoTimestampSchema, + min_samples: z.number().int().nonnegative(), + trust_floor: z.number().min(0).max(1), + half_life_days: z.number().positive(), + }) + .strict(), + ]), + cycle_state: z + .object({ + source: z.literal("state.json"), + region_rejected_enabled: z.boolean(), + }) + .strict(), + implicit_outcomes: z.discriminatedUnion("enabled", [ + DisabledHistoryStoreSchema, + z + .object({ + enabled: z.literal(true), + cap: z.number().int().positive(), + created_at: IsoTimestampSchema, + }) + .strict(), + ]), + }) + .strict(); const ChangedRangeSchema = z .object({ start: z.number().int().nonnegative(), end: z.number().int().positive() }) @@ -180,6 +278,8 @@ const PolicyReplayEnvelopeBaseSchema = z .strict(), state_sha256: Sha256Schema, raw_response_sha256: z.array(Sha256Schema), + response_calls: z.array(ResponseCallSchema), + history: HistoryInputsSchema, /** Original production trace; replay must reproduce it byte-for-byte before ablation. */ policy_trace: PolicyTraceSchema, lossless: z.boolean(), @@ -251,6 +351,49 @@ export const PolicyReplayEnvelopeSchema = PolicyReplayEnvelopeBaseSchema.superRe message: "ordered response hashes must match the production policy trace", }); } + if ( + value.response_calls.length !== value.raw_response_sha256.length || + value.response_calls.some( + (call, index) => + call.response_sha256 !== value.raw_response_sha256[index] || + (index > 0 && call.ordinal <= (value.response_calls[index - 1]?.ordinal ?? -1)), + ) + ) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + path: ["response_calls"], + message: "response calls must bind every ordered response hash to its logical slot", + }); + } + const seenCallIds = new Set(); + for (const [index, call] of value.response_calls.entries()) { + const expectedMethod = call.kind === "reviewer" ? "review" : "complete"; + const expectedCallId = policyReplayCallId({ + runId: value.run_id, + iter: value.iter, + kind: call.kind, + provider: call.provider, + method: call.method, + key: call.key, + promptSha256: call.prompt_sha256, + ordinal: call.ordinal, + slot: call.slot, + attempt: call.attempt, + occurrence: call.occurrence, + }); + if ( + call.method !== expectedMethod || + call.call_id !== expectedCallId || + seenCallIds.has(call.call_id) + ) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + path: ["response_calls", index], + message: "response call identity, method, ordinal, or call id is invalid", + }); + } + seenCallIds.add(call.call_id); + } for (const row of value.aggregate.policy_inactive) { if (!POLICY_REASON_CODES.includes(row.reason_code)) { ctx.addIssue({ diff --git a/tests/unit/cassette-recording-adapter.test.ts b/tests/unit/cassette-recording-adapter.test.ts index d63c9da..0d3f12b 100644 --- a/tests/unit/cassette-recording-adapter.test.ts +++ b/tests/unit/cassette-recording-adapter.test.ts @@ -6,6 +6,7 @@ import { join } from "node:path"; import { RecordingAdapter } from "../../src/cassette/recording-adapter.ts"; import { loadCassette } from "../../src/cassette/store.ts"; import type { ProviderAdapter, ReviewResult } from "../../src/providers/adapter-base.ts"; +import { policyReplayCallId } from "../../src/schemas/policy-replay.ts"; function realAdapter(): ProviderAdapter & { embed: (t: string, o: unknown) => Promise } { return { @@ -85,4 +86,109 @@ describe("RecordingAdapter", () => { }; expect(typeof rec.embed).toBe("undefined"); }); + + it("binds recorded responses to strict caller identity without persisting prompt text", async () => { + const dir = mkdtempSync(join(tmpdir(), "rg-rec-call-")); + const path = join(dir, "c.jsonl"); + const promptFile = join(dir, "prompt.txt"); + writeFileSync(promptFile, "safe reviewer prompt"); + const rec = new RecordingAdapter(realAdapter(), path); + const context = { + runId: "rig-run", + iter: 2, + kind: "reviewer" as const, + ordinal: 3, + slot: 1, + attempt: 2, + occurrence: 1, + }; + await rec.review({ + promptFile, + workingDir: dir, + findingsPath: join(dir, "findings.json"), + persona: "security", + diffPath: join(dir, "diff.patch"), + cfg: { enabled: true, auth: "oauth", model: "m", timeoutMs: 1000 }, + reviewerId: "openrouter-security", + policyReplayCall: context, + } as Parameters[0]); + + const [entry] = loadCassette(path); + expect(entry?.policyReplayCall).toEqual({ + ...context, + callId: policyReplayCallId({ + ...context, + provider: "openrouter", + method: "review", + key: "openrouter-security", + promptSha256: "c42b75d9fb013066a5f7d895a12bb880ff9b98a45b8abe849eb3401bc0706dcb", + }), + }); + expect(JSON.stringify(entry)).not.toContain("safe reviewer prompt"); + }); + + it("preserves call identities when identical concurrent reviews complete in reverse order", async () => { + const dir = mkdtempSync(join(tmpdir(), "rg-rec-concurrent-")); + const path = join(dir, "c.jsonl"); + const promptFile = join(dir, "prompt.txt"); + writeFileSync(promptFile, "shared safe prompt"); + const real = realAdapter(); + real.review = async (input) => { + if (input.policyReplayCall?.slot === 0) await Bun.sleep(30); + return { + reviewerId: input.reviewerId, + verdict: "PASS", + findings: [], + usage: { inputTokens: 1, outputTokens: 1, costUsd: 0, quotaUsedPct: null }, + durationMs: 1, + exitCode: 0, + rawEventsPath: "", + rawText: `safe slot ${input.policyReplayCall?.slot ?? -1}`, + status: "ok", + }; + }; + const rec = new RecordingAdapter(real, path); + const common = { + promptFile, + workingDir: dir, + findingsPath: join(dir, "findings.json"), + persona: "security", + diffPath: join(dir, "diff.patch"), + cfg: { enabled: true, auth: "oauth" as const, model: "m", timeoutMs: 1000 }, + reviewerId: "openrouter-security", + }; + await Promise.all([ + rec.review({ + ...common, + policyReplayCall: { + runId: "rig-run", + iter: 1, + kind: "reviewer", + ordinal: 0, + slot: 0, + attempt: 1, + occurrence: 0, + }, + }), + rec.review({ + ...common, + policyReplayCall: { + runId: "rig-run", + iter: 1, + kind: "reviewer", + ordinal: 1, + slot: 1, + attempt: 1, + occurrence: 0, + }, + }), + ]); + + const entries = loadCassette(path); + expect(entries.map((entry) => entry.policyReplayCall?.ordinal)).toEqual([1, 0]); + expect( + entries.map((entry) => ("rawText" in entry.result ? entry.result.rawText : undefined)), + ).toEqual(["safe slot 1", "safe slot 0"]); + expect(new Set(entries.map((entry) => entry.policyReplayCall?.callId)).size).toBe(2); + }); }); diff --git a/tests/unit/cassette-schema.test.ts b/tests/unit/cassette-schema.test.ts index d723346..370df25 100644 --- a/tests/unit/cassette-schema.test.ts +++ b/tests/unit/cassette-schema.test.ts @@ -48,6 +48,38 @@ describe("cassette schema", () => { expect((e.result as { vector: number[] }).vector).toHaveLength(3); }); + it("retains strict additive policy replay call identity while legacy entries stay parseable", () => { + const legacy = { + schema: "reviewgate.cassette.entry.v1", + provider: "codex", + key: "codex-security", + method: "review", + promptSha256: "a".repeat(64), + result: reviewResult, + }; + expect(CassetteEntrySchema.parse(legacy)).not.toHaveProperty("policyReplayCall"); + + const policyReplayCall = { + callId: "b".repeat(64), + runId: "rig-run", + iter: 2, + kind: "reviewer" as const, + ordinal: 3, + slot: 1, + attempt: 2, + occurrence: 1, + }; + expect(CassetteEntrySchema.parse({ ...legacy, policyReplayCall }).policyReplayCall).toEqual( + policyReplayCall, + ); + expect(() => + CassetteEntrySchema.parse({ + ...legacy, + policyReplayCall: { ...policyReplayCall, prompt: "must never persist" }, + }), + ).toThrow(); + }); + it("rejects an unknown provider", () => { expect(() => CassetteEntrySchema.parse({ diff --git a/tests/unit/orchestrator-policy-trace.test.ts b/tests/unit/orchestrator-policy-trace.test.ts index a61b5d9..e118dba 100644 --- a/tests/unit/orchestrator-policy-trace.test.ts +++ b/tests/unit/orchestrator-policy-trace.test.ts @@ -1,12 +1,95 @@ import { describe, expect, it } from "bun:test"; -import { readFileSync } from "node:fs"; +import { mkdirSync, mkdtempSync, readFileSync, readdirSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; import { join } from "node:path"; +import { defaultConfig } from "../../src/config/defaults.ts"; +import { Orchestrator } from "../../src/core/orchestrator.ts"; import { EMPTY_POLICY_ABLATIONS, resolvePolicyExecutionOptions, } from "../../src/core/policy/replay.ts"; +import type { ProviderAdapter, ReviewResult } from "../../src/providers/adapter-base.ts"; +import { PolicyReplayEnvelopeSchema } from "../../src/schemas/policy-replay.ts"; +import { initialState } from "../../src/schemas/state.ts"; const REPO_ROOT = join(import.meta.dir, "..", ".."); +const DIFF = "diff --git a/foo.ts b/foo.ts\n--- a/foo.ts\n+++ b/foo.ts\n@@ -1 +1 @@\n-old\n+new\n"; + +function reviewResult( + reviewerId: string, + status: "ok" | "error", + rawText: string, + findings: ReviewResult["findings"] = [], +): ReviewResult { + return { + reviewerId, + verdict: status === "ok" && findings.length === 0 ? "PASS" : status === "ok" ? "FAIL" : "ERROR", + findings, + usage: { inputTokens: 1, outputTokens: 1, costUsd: 0, quotaUsedPct: null }, + durationMs: 1, + exitCode: status === "ok" ? 0 : 1, + rawEventsPath: "", + rawText, + status, + }; +} + +async function capturedEnvelope(input: { + reviewers: Array<{ + provider: "codex" | "gemini"; + persona: "security"; + fallback?: Array<"gemini">; + }>; + adapters: Partial>; + critic?: { provider: "codex"; model: string } | null; + criticMaxAttempts?: number; +}) { + const repoRoot = mkdtempSync(join(tmpdir(), "rg-policy-call-capture-")); + writeFileSync(join(repoRoot, "foo.ts"), "new\n"); + mkdirSync(join(repoRoot, ".reviewgate")); + writeFileSync( + join(repoRoot, ".reviewgate", "state.json"), + JSON.stringify(initialState("policy-call-session")), + { mode: 0o600 }, + ); + const outputRoot = mkdtempSync(join(tmpdir(), "rg-policy-call-output-")); + const sinkDir = join(outputRoot, "policy-replay"); + mkdirSync(sinkDir, { mode: 0o700 }); + const config = { + ...defaultConfig, + phases: { + ...defaultConfig.phases, + review: { ...defaultConfig.phases.review, reviewers: input.reviewers }, + critic: input.critic ?? null, + triage: null, + fpLedger: null, + reputation: { ...defaultConfig.phases.reputation, enabled: false }, + implicitOutcomes: null, + }, + }; + const orchestrator = new Orchestrator({ + repoRoot, + // biome-ignore lint/suspicious/noExplicitAny: focused test config narrows provider tuples + config: config as any, + adapters: input.adapters, + sandboxMode: "off", + hostTier: "opus", + diff: DIFF, + reasonOnFailEnabled: true, + policyExecution: { trace: "memory", policyAblations: new Set(), authoritative: false }, + policyReplayCapture: { sinkDir, sourceCommit: "a".repeat(40) }, + providerAvailable: (provider) => input.adapters[provider as "codex" | "gemini"] !== undefined, + ...(input.criticMaxAttempts === undefined + ? {} + : { criticMaxAttempts: input.criticMaxAttempts }), + }); + await orchestrator.runIteration({ runId: "policy-call-run", iter: 1 }); + const files = readdirSync(sinkDir).filter((name) => name.endsWith(".json")); + expect(files).toHaveLength(1); + return PolicyReplayEnvelopeSchema.parse( + JSON.parse(readFileSync(join(sinkDir, files[0] as string), "utf8")), + ); +} describe("internal policy execution selection", () => { it("keeps legacy direct construction off and defaults the AuditLogger path to persist", () => { @@ -67,3 +150,95 @@ describe("policy ablations stay internal", () => { expect(gate).not.toMatch(/REVIEWGATE_POLICY_PASS/); }); }); + +describe("policy replay response-call capture", () => { + it("binds the settled failover response to the actual provider and attempt", async () => { + const codex: ProviderAdapter = { + id: "codex", + async preflight() { + return { available: true, version: "x", authMode: "oauth", error: null }; + }, + async review(input) { + return reviewResult(input.reviewerId, "error", "safe primary failure"); + }, + }; + const gemini: ProviderAdapter = { + id: "gemini", + async preflight() { + return { available: true, version: "x", authMode: "oauth", error: null }; + }, + async review(input) { + return reviewResult(input.reviewerId, "ok", "safe fallback response"); + }, + }; + const envelope = await capturedEnvelope({ + reviewers: [{ provider: "codex", persona: "security", fallback: ["gemini"] }], + adapters: { codex, gemini }, + }); + + expect(envelope.response_calls).toHaveLength(1); + const responseCall = envelope.response_calls[0]; + if (responseCall === undefined) throw new Error("missing captured fallback response call"); + expect(responseCall).toMatchObject({ + kind: "reviewer", + provider: "gemini", + method: "review", + key: "gemini-security", + slot: 0, + attempt: 2, + occurrence: 0, + ordinal: 0, + }); + expect(envelope.raw_response_sha256).toEqual([responseCall.response_sha256]); + }); + + it("binds critic retries to one prompt identity with ordered attempts and occurrences", async () => { + const finding = { + id: "critic-1", + signature: "critic-signature", + severity: "WARN" as const, + category: "quality" as const, + rule_id: "critic-rule", + file: "foo.ts", + line_start: 1, + line_end: 1, + message: "A concrete issue", + details: "The changed value is not checked.", + reviewer: { provider: "codex", model: "test", persona: "security" }, + confidence: 0.9, + consensus: "singleton" as const, + }; + let completion = 0; + const codex: ProviderAdapter = { + id: "codex", + async preflight() { + return { available: true, version: "x", authMode: "oauth", error: null }; + }, + async review(input) { + return reviewResult(input.reviewerId, "ok", "safe reviewer response", [finding]); + }, + async complete() { + completion += 1; + return completion === 1 + ? "safe invalid critic response" + : JSON.stringify({ + verdicts: [{ signature: finding.signature, verdict: "keep" }], + }); + }, + }; + const envelope = await capturedEnvelope({ + reviewers: [{ provider: "codex", persona: "security" }], + adapters: { codex }, + critic: { provider: "codex", model: "test" }, + criticMaxAttempts: 2, + }); + const criticCalls = envelope.response_calls.filter((call) => call.kind === "critic"); + + expect(criticCalls).toHaveLength(2); + expect(criticCalls.map((call) => call.attempt)).toEqual([1, 2]); + expect(criticCalls.map((call) => call.occurrence)).toEqual([0, 1]); + expect(new Set(criticCalls.map((call) => call.key)).size).toBe(1); + expect(criticCalls.map((call) => call.slot)).toEqual([0, 0]); + expect(criticCalls.map((call) => call.ordinal)).toEqual([2, 3]); + }); +}); diff --git a/tests/unit/policy-replay-capture.test.ts b/tests/unit/policy-replay-capture.test.ts index f03ab5e..5313b5c 100644 --- a/tests/unit/policy-replay-capture.test.ts +++ b/tests/unit/policy-replay-capture.test.ts @@ -28,10 +28,12 @@ import { createPolicyStateSnapshot, createReplayBranches, digestPolicyState, + verifyPolicyStateSnapshot, } from "../../src/rig/policy-replay-state.ts"; import { type PolicyReplayEnvelope, PolicyReplayEnvelopeSchema, + policyReplayCallId, } from "../../src/schemas/policy-replay.ts"; const H = "a".repeat(64); @@ -126,6 +128,64 @@ function envelope(overrides: Partial = {}): PolicyReplayEn pre_policy: { self_refutation_enabled: true, hypothetical_enabled: true }, state_sha256: H, raw_response_sha256: ["b".repeat(64), "c".repeat(64)], + response_calls: [ + { + call_id: policyReplayCallId({ + runId: "rig-run-1", + iter: 1, + kind: "reviewer", + provider: "codex", + method: "review", + key: "codex-correctness", + promptSha256: "d".repeat(64), + ordinal: 0, + slot: 0, + attempt: 1, + occurrence: 0, + }), + kind: "reviewer", + provider: "codex", + method: "review", + key: "codex-correctness", + prompt_sha256: "d".repeat(64), + ordinal: 0, + slot: 0, + attempt: 1, + occurrence: 0, + response_sha256: "b".repeat(64), + }, + { + call_id: policyReplayCallId({ + runId: "rig-run-1", + iter: 1, + kind: "critic", + provider: "openrouter", + method: "complete", + key: `openrouter:complete:${"e".repeat(64)}`, + promptSha256: "e".repeat(64), + ordinal: 1, + slot: 0, + attempt: 1, + occurrence: 0, + }), + kind: "critic", + provider: "openrouter", + method: "complete", + key: `openrouter:complete:${"e".repeat(64)}`, + prompt_sha256: "e".repeat(64), + ordinal: 1, + slot: 0, + attempt: 1, + occurrence: 0, + response_sha256: "c".repeat(64), + }, + ], + history: { + fp_ledger: { enabled: false }, + reputation: { enabled: false }, + cycle_state: { source: "state.json", region_rejected_enabled: false }, + implicit_outcomes: { enabled: false }, + }, policy_trace: policyTrace, lossless: true, ...overrides, @@ -176,6 +236,50 @@ describe("policy replay envelope schema", () => { }), ).toThrow(/ordered response hashes/i); }); + + test("binds response hashes to stable calls and records effective history-store inputs", () => { + const base = envelope(); + const promptSha256 = "d".repeat(64); + const call = { + call_id: policyReplayCallId({ + runId: base.run_id, + iter: base.iter, + kind: "reviewer", + provider: "codex", + method: "review", + key: "codex-correctness", + promptSha256, + ordinal: 0, + slot: 0, + attempt: 1, + occurrence: 0, + }), + kind: "reviewer", + provider: "codex", + method: "review", + key: "codex-correctness", + prompt_sha256: promptSha256, + ordinal: 0, + slot: 0, + attempt: 1, + occurrence: 0, + response_sha256: "b".repeat(64), + }; + expect(() => + PolicyReplayEnvelopeSchema.parse({ + ...base, + response_calls: [call], + raw_response_sha256: ["b".repeat(64)], + policy_trace: { ...base.policy_trace, raw_response_sha256: ["b".repeat(64)] }, + history: { + fp_ledger: { enabled: false }, + reputation: { enabled: false }, + cycle_state: { source: "state.json", region_rejected_enabled: false }, + implicit_outcomes: { enabled: false }, + }, + }), + ).not.toThrow(); + }); }); describe("policy replay capture", () => { @@ -280,6 +384,22 @@ describe("policy replay capture", () => { const traversing = envelope(); traversing.run_id = "../../escape"; traversing.policy_trace.run_id = "../../escape"; + traversing.response_calls = traversing.response_calls.map((call) => ({ + ...call, + call_id: policyReplayCallId({ + runId: traversing.run_id, + iter: traversing.iter, + kind: call.kind, + provider: call.provider, + method: call.method, + key: call.key, + promptSha256: call.prompt_sha256, + ordinal: call.ordinal, + slot: call.slot, + attempt: call.attempt, + occurrence: call.occurrence, + }), + })); expect( capturePolicyReplayEnvelope({ sinkDir: escapedSink, @@ -371,6 +491,64 @@ describe("policy replay state isolation", () => { ).toThrow(/symlink/i); }); + test("refuses a symlinked policy-state ancestor without writing outside the output root", () => { + const sourceRepoRoot = gitRepo(); + const outputRoot = mkdtempSync(join(tmpdir(), "rg-policy-state-output-")); + const outside = mkdtempSync(join(tmpdir(), "rg-policy-state-escape-")); + symlinkSync(outside, join(outputRoot, "policy-state")); + + expect(() => createPolicyStateSnapshot({ sourceRepoRoot, outputRoot })).toThrow( + /symlink|escape|ordinary directory/i, + ); + expect(readdirSync(outside)).toEqual([]); + }); + + test("uses one code-unit order for traversal, digest, manifest, and verification", () => { + const sourceRepoRoot = gitRepo(); + writeFileSync(join(sourceRepoRoot, ".reviewgate", "Z.json"), "upper\n"); + writeFileSync(join(sourceRepoRoot, ".reviewgate", "a.json"), "lower\n"); + const outputRoot = mkdtempSync(join(tmpdir(), "rg-policy-state-output-")); + + const snapshot = createPolicyStateSnapshot({ sourceRepoRoot, outputRoot }); + expect(() => + verifyPolicyStateSnapshot({ + outputRoot, + ref: snapshot.ref, + sha256: snapshot.sha256, + expectedStateSha256: snapshot.stateSha256, + }), + ).not.toThrow(); + const manifest = JSON.parse(readFileSync(join(outputRoot, snapshot.ref), "utf8")) as { + files: Array<{ path: string }>; + }; + expect(manifest.files.map((entry) => entry.path)).toEqual([ + "Z.json", + "a.json", + "fp-ledger.jsonl", + "reputation/events.jsonl", + ]); + }); + + test("rejects mode drift in a persisted state tree while accepting legacy source modes", () => { + const sourceRepoRoot = gitRepo(); + const sourcePath = join(sourceRepoRoot, ".reviewgate", "fp-ledger.jsonl"); + chmodSync(sourcePath, 0o644); + const outputRoot = mkdtempSync(join(tmpdir(), "rg-policy-state-output-")); + const snapshot = createPolicyStateSnapshot({ sourceRepoRoot, outputRoot }); + const persistedPath = join(outputRoot, snapshot.stateRef, "fp-ledger.jsonl"); + expect(lstatSync(persistedPath).mode & 0o7777).toBe(0o600); + chmodSync(persistedPath, 0o644); + + expect(() => + verifyPolicyStateSnapshot({ + outputRoot, + ref: snapshot.ref, + sha256: snapshot.sha256, + expectedStateSha256: snapshot.stateSha256, + }), + ).toThrow(/0600|mode/i); + }); + test("rejects hardlinked and special state entries", () => { const hardlinkedRepo = gitRepo(); linkSync( diff --git a/tests/unit/rig-ablate.test.ts b/tests/unit/rig-ablate.test.ts index 7397039..8b7a4a4 100644 --- a/tests/unit/rig-ablate.test.ts +++ b/tests/unit/rig-ablate.test.ts @@ -1,6 +1,10 @@ import { describe, expect, test } from "bun:test"; +import { mkdtempSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; import { makeMetric, summarizeSpread } from "../../src/bench/metrics.ts"; -import { POLICY_PASS_IDS } from "../../src/core/policy/catalog.ts"; +import { RigLayerSelectorError, runRigAblate } from "../../src/cli/commands/rig.ts"; +import { POLICY_CATALOG_VERSION, POLICY_PASS_IDS } from "../../src/core/policy/catalog.ts"; import { ablate, renderAblationMatrix } from "../../src/rig/ablate.ts"; import { renderPolicyAblationRows } from "../../src/rig/replay.ts"; import type { Finding } from "../../src/schemas/finding.ts"; @@ -88,6 +92,81 @@ function result(turns: RigTurnRecord[], over: Partial = {}): RigResul const NO_TAGS = new Map(); describe("rig ablate", () => { + test("keeps exact catalog selectors and legacy aliases in their own result modes", async () => { + const root = mkdtempSync(join(tmpdir(), "rg-layer-selector-")); + const scriptPath = join(root, "script.json"); + writeFileSync( + scriptPath, + JSON.stringify({ + schema: "reviewgate.rig.turn-script.v1", + id: "selector-script", + turns: [{ index: 1, prompt: "safe", seeded: null }], + }), + ); + const legacyPath = join(root, "legacy.json"); + writeFileSync(legacyPath, JSON.stringify(result([turn({ index: 1 })]))); + await expect( + runRigAblate({ + resultPath: legacyPath, + scriptPath, + layer: "judgment.confidence", + }), + ).rejects.toBeInstanceOf(RigLayerSelectorError); + + const exactPath = join(root, "exact.json"); + writeFileSync( + exactPath, + JSON.stringify( + result([turn({ index: 1 })], { + policyReplay: { + authoritative: true, + catalogVersion: POLICY_CATALOG_VERSION, + sourceCommit: "a".repeat(40), + passIds: [...POLICY_PASS_IDS], + reason: null, + }, + }), + ), + ); + await expect( + runRigAblate({ resultPath: exactPath, scriptPath, layer: "critic" }), + ).rejects.toBeInstanceOf(RigLayerSelectorError); + }); + + test("maps an invalid mode-specific CLI selector to exact exit 2", async () => { + const root = mkdtempSync(join(tmpdir(), "rg-layer-selector-cli-")); + const resultPath = join(root, "legacy.json"); + const scriptPath = join(root, "script.json"); + writeFileSync(resultPath, JSON.stringify(result([turn({ index: 1 })]))); + writeFileSync( + scriptPath, + JSON.stringify({ + schema: "reviewgate.rig.turn-script.v1", + id: "selector-cli", + turns: [{ index: 1, prompt: "safe", seeded: null }], + }), + ); + const child = Bun.spawn( + [ + "bun", + "run", + "src/cli/index.ts", + "rig", + "ablate", + "--result", + resultPath, + "--script", + scriptPath, + "--layer", + "judgment.confidence", + ], + { cwd: join(import.meta.dir, "..", ".."), stdout: "pipe", stderr: "pipe" }, + ); + const [exitCode, stderr] = await Promise.all([child.exited, new Response(child.stderr).text()]); + expect(exitCode).toBe(2); + expect(stderr).toContain("legacy --layer"); + }); + test("exact rows use every closed catalog ID and keep Lore separate", () => { const rendered = renderPolicyAblationRows( POLICY_PASS_IDS.map((passId) => ({ diff --git a/tests/unit/rig-replay.test.ts b/tests/unit/rig-replay.test.ts index db01bc1..263f2c5 100644 --- a/tests/unit/rig-replay.test.ts +++ b/tests/unit/rig-replay.test.ts @@ -1,18 +1,22 @@ -import { describe, expect, test } from "bun:test"; +import { describe, expect, spyOn, test } from "bun:test"; import { execFileSync } from "node:child_process"; import { createHash } from "node:crypto"; import { cpSync, mkdirSync, mkdtempSync, readFileSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; +import { canonicalJson } from "../../src/audit/canonical.ts"; import { aggregate } from "../../src/core/aggregator.ts"; import { validateFindingFacts } from "../../src/core/fact-check.ts"; +import { FpLedgerStore } from "../../src/core/fp-ledger/store.ts"; import { groundFindings } from "../../src/core/grounding.ts"; import { demoteHypotheticalCriticals } from "../../src/core/hypothetical-demote.ts"; +import { ImplicitOutcomeStore } from "../../src/core/learnings/implicit-outcomes.ts"; import { POLICY_CATALOG_VERSION } from "../../src/core/policy/catalog.ts"; import { capturePolicyReplayEnvelope } from "../../src/core/policy/replay-capture.ts"; import { PolicyTraceRecorder } from "../../src/core/policy/trace.ts"; import { ReputationStore } from "../../src/core/reputation/store.ts"; import { demoteSelfRefuting } from "../../src/core/self-refutation.ts"; +import { StateStore } from "../../src/core/state-store.ts"; import { RigAuthorityError, createPolicyStateSnapshot, @@ -23,6 +27,7 @@ import { checkCassette, checkDeterminism, replay, + replayPolicyAblations, replayPolicyEnvelopePair, replayPolicyEnvelopeSequence, runWithReplayProviderCeiling, @@ -30,8 +35,10 @@ import { import { type PolicyReplayEnvelopeInput, PolicyReplayEnvelopeSchema, + policyReplayCallId, } from "../../src/schemas/policy-replay.ts"; import type { RigManifest } from "../../src/schemas/rig-manifest.ts"; +import { initialState } from "../../src/schemas/state.ts"; /** Smallest run that harvests: one turn, one snapshot laid out as a repo root. */ function miniRun(): { manifestPath: string; scriptPath: string; root: string } { @@ -114,6 +121,96 @@ const entry = (key: string, result: Record | "empty") => }, }); +function responseEntry(input: { + provider?: "claude-code" | "codex" | "gemini" | "openrouter" | "opencode" | "ollama"; + method: "review" | "complete"; + key: string; + promptSha256: string; + rawText: string; + call?: ReturnType; + runId?: string; + iter?: number; +}): string { + const provider = input.provider ?? "openrouter"; + return JSON.stringify({ + schema: "reviewgate.cassette.entry.v1", + provider, + method: input.method, + key: input.key, + promptSha256: input.promptSha256, + ...(input.call === undefined + ? {} + : { + policyReplayCall: { + callId: input.call.call_id, + runId: input.runId ?? "exact-run", + iter: input.iter ?? 1, + kind: input.call.kind, + ordinal: input.call.ordinal, + slot: input.call.slot, + attempt: input.call.attempt, + occurrence: input.call.occurrence, + }, + }), + result: + input.method === "complete" + ? { text: input.rawText } + : { + reviewerId: input.key, + verdict: "PASS", + findings: [], + usage: { inputTokens: 10, outputTokens: 5, costUsd: 0, quotaUsedPct: null }, + durationMs: 1, + exitCode: 0, + rawEventsPath: "", + status: "ok", + rawText: input.rawText, + }, + }); +} + +function responseCall(input: { + runId?: string; + iter?: number; + kind: "reviewer" | "grounding" | "critic"; + provider?: "claude-code" | "codex" | "gemini" | "openrouter" | "opencode" | "ollama"; + method: "review" | "complete"; + key: string; + promptSha256: string; + ordinal?: number; + slot: number; + attempt?: number; + occurrence?: number; + rawText: string; +}) { + const identity = { + runId: input.runId ?? "exact-run", + iter: input.iter ?? 1, + kind: input.kind, + provider: input.provider ?? "openrouter", + method: input.method, + key: input.key, + promptSha256: input.promptSha256, + ordinal: input.ordinal ?? input.slot, + slot: input.slot, + attempt: input.attempt ?? 1, + occurrence: input.occurrence ?? 0, + }; + return { + call_id: policyReplayCallId(identity), + kind: identity.kind, + provider: identity.provider, + method: identity.method, + key: identity.key, + prompt_sha256: identity.promptSha256, + ordinal: identity.ordinal, + slot: identity.slot, + attempt: identity.attempt, + occurrence: identity.occurrence, + response_sha256: sha256(input.rawText), + }; +} + function sha256(value: string | Buffer): string { return createHash("sha256").update(value).digest("hex"); } @@ -139,9 +236,9 @@ function emptyPolicyTrace( demoteTestSecurity: true, capDocsSeverity: true, critic: new Map(), - fpActive: new Map(), + fpActive: new Map([["seeded-fp", { id: "FP-001" }]]), fpActiveClusters: new Map(), - repUnreliable: new Set(), + repUnreliable: new Set(["codex:quality"]), protectedReviewers: new Set(), foreignFiles: new Set(), cycleRejected: new Set(), @@ -179,6 +276,72 @@ function exactRun(): { writeFileSync(join(sourceRepoRoot, "src", "x.ts"), "export const x = 1;\n"); mkdirSync(join(sourceRepoRoot, ".reviewgate")); writeFileSync(join(sourceRepoRoot, ".reviewgate", "fp-ledger.jsonl"), ""); + writeFileSync( + join(sourceRepoRoot, ".reviewgate", "state.json"), + JSON.stringify(initialState("exact-session")), + { mode: 0o600 }, + ); + mkdirSync(join(sourceRepoRoot, ".reviewgate", "learnings")); + const observedAt = "2026-08-11T12:00:00.000Z"; + writeFileSync( + join(sourceRepoRoot, ".reviewgate", "learnings", "known_fp.jsonl"), + JSON.stringify({ + schema: "reviewgate.fpledger.v1", + seq: 1, + entries: [ + { + id: "FP-001", + signature: "seeded-fp", + rule_id: "seeded-rule", + category: "quality", + file: "src/x.ts", + symbol: "", + stage: "active", + rejects: [ + { + run_id: "seed-1", + provider: "codex", + ts: "2026-08-01T12:00:00.000Z", + reason: "confirmed false positive one", + }, + { + run_id: "seed-2", + provider: "openrouter", + ts: "2026-08-02T12:00:00.000Z", + reason: "confirmed false positive two", + }, + { + run_id: "seed-3", + provider: "codex", + ts: "2026-08-03T12:00:00.000Z", + reason: "confirmed false positive three", + }, + ], + distinct_providers: ["codex", "openrouter"], + first_seen_at: "2026-08-01T12:00:00.000Z", + last_seen_at: "2026-08-03T12:00:00.000Z", + created_at: "2026-08-01T12:00:00.000Z", + }, + ], + }), + { mode: 0o600 }, + ); + writeFileSync( + join(sourceRepoRoot, ".reviewgate", "reputation.json"), + JSON.stringify({ + schema: "reviewgate.reputation.v1", + reviewers: { + "codex:quality": { + correct: [], + wrong: Array.from({ length: 8 }, (_, index) => ({ + ts: `2026-08-0${index + 1}T12:00:00.000Z`, + eid: `seed-reputation-${index + 1}`, + })), + }, + }, + }), + { mode: 0o600 }, + ); execFileSync("git", ["add", "src/x.ts"], { cwd: sourceRepoRoot }); execFileSync("git", ["commit", "-qm", "source"], { cwd: sourceRepoRoot }); const sourceCommit = execFileSync("git", ["rev-parse", "HEAD"], { @@ -212,9 +375,9 @@ function exactRun(): { demote_test_security: true, cap_docs_severity: true, critic: [], - fp_active: [], + fp_active: [{ signature: "seeded-fp", id: "FP-001" }], fp_active_clusters: [], - rep_unreliable: [], + rep_unreliable: ["codex:quality"], protected_reviewers: [], foreign_files: [], cycle_rejected: [], @@ -227,6 +390,45 @@ function exactRun(): { pre_policy: { self_refutation_enabled: true, hypothetical_enabled: true }, state_sha256: state.stateSha256, raw_response_sha256: [rawHash], + response_calls: [ + { + call_id: policyReplayCallId({ + runId: "exact-run", + iter: 1, + kind: "reviewer", + provider: "openrouter", + method: "review", + key: "openrouter-security", + promptSha256: "a".repeat(64), + ordinal: 0, + slot: 0, + attempt: 1, + occurrence: 0, + }), + kind: "reviewer", + provider: "openrouter", + method: "review", + key: "openrouter-security", + prompt_sha256: "a".repeat(64), + ordinal: 0, + slot: 0, + attempt: 1, + occurrence: 0, + response_sha256: rawHash, + }, + ], + history: { + fp_ledger: { enabled: true, active_at: observedAt, clusters_at: observedAt }, + reputation: { + enabled: true, + observed_at: observedAt, + min_samples: 6, + trust_floor: 0.45, + half_life_days: 45, + }, + cycle_state: { source: "state.json", region_rejected_enabled: false }, + implicit_outcomes: { enabled: false }, + }, policy_trace: policyTrace, lossless: true, }; @@ -238,8 +440,20 @@ function exactRun(): { envelope, }); if (stored.status !== "complete") throw new Error("capture fixture failed"); + const recordedCall = envelope.response_calls[0]; + if (recordedCall === undefined) throw new Error("response call fixture failed"); const cassettePath = join(root, "cassette.jsonl"); - writeFileSync(cassettePath, `${entry("openrouter-security", { rawText })}\n`, { mode: 0o600 }); + writeFileSync( + cassettePath, + `${responseEntry({ + method: "review", + key: "openrouter-security", + promptSha256: "a".repeat(64), + rawText, + call: recordedCall, + })}\n`, + { mode: 0o600 }, + ); const manifestPath = join(root, "manifest.json"); const manifest: RigManifest = { schema: "reviewgate.rig.manifest.v1", @@ -293,6 +507,21 @@ function replaceTrace( turn.policyReplay = { status: "complete", traces: [{ ref: stored.ref, sha256: stored.sha256 }] }; } +function replaceRawTrace( + fixture: ReturnType, + envelope: Record, +): void { + const bytes = canonicalJson(envelope); + const hash = sha256(bytes); + const runId = typeof envelope.run_id === "string" ? envelope.run_id : "exact-run"; + const iter = typeof envelope.iter === "number" ? envelope.iter : 1; + const ref = `${sha256(runId).slice(0, 12)}-i${iter}-${hash.slice(0, 12)}.json`; + writeFileSync(join(fixture.sinkDir, ref), bytes, { mode: 0o600 }); + const turn = fixture.manifest.turns[0]; + if (turn === undefined) throw new Error("turn fixture missing"); + turn.policyReplay = { status: "complete", traces: [{ ref, sha256: hash }] }; +} + function confidenceEnvelope(fixture: ReturnType): PolicyReplayEnvelopeInput { const finding = { id: "confidence-1", @@ -305,7 +534,7 @@ function confidenceEnvelope(fixture: ReturnType): PolicyReplayE line_end: 1, message: "Low-confidence warning", details: "The reviewer explicitly reports uncertainty.", - reviewer: { provider: "codex", model: "gpt-5", persona: "quality" }, + reviewer: { provider: "codex", model: "gpt-5", persona: "correctness" }, confidence: 0.2, consensus: "singleton" as const, }; @@ -337,9 +566,9 @@ function confidenceEnvelope(fixture: ReturnType): PolicyReplayE demoteTestSecurity: true, capDocsSeverity: true, critic: new Map(), - fpActive: new Map(), + fpActive: new Map([["seeded-fp", { id: "FP-001" }]]), fpActiveClusters: new Map(), - repUnreliable: new Set(), + repUnreliable: new Set(["codex:quality"]), protectedReviewers: new Set(), foreignFiles: new Set(), cycleRejected: new Set(), @@ -371,9 +600,9 @@ function confidenceEnvelope(fixture: ReturnType): PolicyReplayE demote_test_security: true, cap_docs_severity: true, critic: [], - fp_active: [], + fp_active: [{ signature: "seeded-fp", id: "FP-001" }], fp_active_clusters: [], - rep_unreliable: [], + rep_unreliable: ["codex:quality"], protected_reviewers: [], foreign_files: [], cycle_rejected: [], @@ -463,6 +692,224 @@ describe("rig replay — cassette integrity", () => { }); describe("rig replay — exact policy authority", () => { + test("rejects captured history inputs that disagree with branch-local production stores", async () => { + const cases: Array<{ + name: string; + mutate: (envelope: PolicyReplayEnvelopeInput) => PolicyReplayEnvelopeInput; + }> = [ + { + name: "fp ledger", + mutate: (envelope) => ({ + ...envelope, + aggregate: { + ...envelope.aggregate, + fp_active: [{ signature: "missing-from-store", id: "FP-999" }], + }, + }), + }, + { + name: "reputation", + mutate: (envelope) => ({ + ...envelope, + aggregate: { ...envelope.aggregate, rep_unreliable: ["gemini:security"] }, + }), + }, + { + name: "cycle state", + mutate: (envelope) => ({ + ...envelope, + aggregate: { ...envelope.aggregate, cycle_rejected: ["not-in-state"] }, + }), + }, + ]; + for (const row of cases) { + const fixture = exactRun(); + const candidate = PolicyReplayEnvelopeSchema.parse(row.mutate(fixture.envelope)); + let caught: unknown; + try { + await replayPolicyEnvelopePair({ + sourceRepoRoot: fixture.sourceRepoRoot, + envelope: candidate, + stateSnapshotRoot: fixture.stateRoot, + passId: "judgment.confidence", + }); + } catch (error) { + caught = error; + } + expect(caught, row.name).toBeInstanceOf(RigAuthorityError); + expect((caught as RigAuthorityError | undefined)?.code, row.name).toBe( + "state-digest-mismatch", + ); + } + }); + + test("uses the real branch-local fp, reputation, and cycle Store APIs", async () => { + const fixture = exactRun(); + const fpRead = spyOn(FpLedgerStore.prototype, "snapshot"); + const reputationRead = spyOn(ReputationStore.prototype, "snapshot"); + const cycleRead = spyOn(StateStore.prototype, "load"); + try { + await replayPolicyEnvelopePair({ + sourceRepoRoot: fixture.sourceRepoRoot, + envelope: PolicyReplayEnvelopeSchema.parse(fixture.envelope), + stateSnapshotRoot: fixture.stateRoot, + passId: "judgment.confidence", + }); + expect(fpRead).toHaveBeenCalled(); + expect(reputationRead).toHaveBeenCalled(); + expect(cycleRead).toHaveBeenCalled(); + } finally { + fpRead.mockRestore(); + reputationRead.mockRestore(); + cycleRead.mockRestore(); + } + }); + + test("matches logical response calls by identity when physical completion order reverses", () => { + const fixture = exactRun(); + const first = "safe slow logical response"; + const second = "safe fast logical response"; + const firstPrompt = "b".repeat(64); + const secondPrompt = "c".repeat(64); + const slowCall = responseCall({ + kind: "reviewer", + method: "review", + key: "openrouter-slow", + promptSha256: firstPrompt, + slot: 0, + rawText: first, + }); + const fastCall = responseCall({ + kind: "reviewer", + method: "review", + key: "openrouter-fast", + promptSha256: secondPrompt, + slot: 1, + rawText: second, + }); + const calls = [slowCall, fastCall]; + const hashes = calls.map((call) => call.response_sha256); + replaceTrace(fixture, { + ...fixture.envelope, + raw_response_sha256: hashes, + response_calls: calls, + policy_trace: emptyPolicyTrace(hashes), + }); + const extraPrompt = "d".repeat(64); + const physical = [ + responseEntry({ + method: "review", + key: "openrouter-fast", + promptSha256: secondPrompt, + rawText: second, + call: fastCall, + }), + responseEntry({ + method: "complete", + key: `openrouter:complete:${extraPrompt}`, + promptSha256: extraPrompt, + rawText: "safe unrelated curator completion", + }), + responseEntry({ + method: "review", + key: "openrouter-slow", + promptSha256: firstPrompt, + rawText: first, + call: slowCall, + }), + ].join("\n"); + const cassetteBytes = `${physical}\n`; + writeFileSync(join(fixture.root, "cassette.jsonl"), cassetteBytes, { mode: 0o600 }); + if (fixture.manifest.policyReplay) { + fixture.manifest.policyReplay.cassetteSha256 = sha256(cassetteBytes); + } + + expect(() => + validateRigPolicyReplayArtifacts({ + manifest: fixture.manifest, + manifestPath: fixture.manifestPath, + }), + ).not.toThrow(); + }); + + test("matches identical concurrent calls by call id when their completions reverse", () => { + const fixture = exactRun(); + const promptSha256 = "b".repeat(64); + const slow = responseCall({ + kind: "reviewer", + method: "review", + key: "openrouter-security", + promptSha256, + slot: 0, + occurrence: 0, + rawText: "safe slow duplicate response", + }); + const fast = responseCall({ + kind: "reviewer", + method: "review", + key: "openrouter-security", + promptSha256, + slot: 1, + occurrence: 1, + rawText: "safe fast duplicate response", + }); + const calls = [slow, fast]; + const hashes = calls.map((call) => call.response_sha256); + replaceTrace(fixture, { + ...fixture.envelope, + raw_response_sha256: hashes, + response_calls: calls, + policy_trace: emptyPolicyTrace(hashes), + }); + const physical = `${responseEntry({ method: "review", key: fast.key, promptSha256, rawText: "safe fast duplicate response", call: fast })}\n${responseEntry({ method: "review", key: slow.key, promptSha256, rawText: "safe slow duplicate response", call: slow })}\n`; + writeFileSync(join(fixture.root, "cassette.jsonl"), physical, { mode: 0o600 }); + if (fixture.manifest.policyReplay) + fixture.manifest.policyReplay.cassetteSha256 = sha256(physical); + + expect(() => + validateRigPolicyReplayArtifacts({ + manifest: fixture.manifest, + manifestPath: fixture.manifestPath, + }), + ).not.toThrow(); + }); + + test("rejects authoritative cassette responses whose call metadata is missing", () => { + const fixture = exactRun(); + const cassettePath = join(fixture.root, "cassette.jsonl"); + const rawText = "safe recorded review response"; + const legacy = `${entry("openrouter-security", { rawText })}\n`; + writeFileSync(cassettePath, legacy, { mode: 0o600 }); + if (fixture.manifest.policyReplay) + fixture.manifest.policyReplay.cassetteSha256 = sha256(legacy); + expect(() => + validateRigPolicyReplayArtifacts({ + manifest: fixture.manifest, + manifestPath: fixture.manifestPath, + }), + ).toThrow( + expect.objectContaining({ + code: "response-hash-mismatch", + exitCode: 4, + }), + ); + }); + + test("filters authoritative ablation to one exact closed-catalog selector", async () => { + const fixture = exactRun(); + const selectedReplay = replayPolicyAblations as unknown as (input: { + manifestPath: string; + sourceRepoRoot: string; + passId: string; + }) => Promise>; + const rows = await selectedReplay({ + manifestPath: fixture.manifestPath, + sourceRepoRoot: fixture.sourceRepoRoot, + passId: "judgment.confidence", + }); + expect(rows.map((row) => row.passId)).toEqual(["judgment.confidence"]); + }); + test("turns any attempted network or provider subprocess call into authority exit 4", () => { const attempts: Array<() => unknown> = [ () => fetch("https://example.invalid"), @@ -480,7 +927,7 @@ describe("rig replay — exact policy authority", () => { } }); - test("replays production policy in isolated branches without a live provider capability", () => { + test("replays production policy in isolated branches without a live provider capability", async () => { const fixture = exactRun(); const validated = validateRigPolicyReplayArtifacts({ manifest: fixture.manifest, @@ -489,7 +936,7 @@ describe("rig replay — exact policy authority", () => { expect(validated).not.toBeNull(); const item = validated?.turns.get(1)?.[0]; if (item === undefined) throw new Error("validated trace missing"); - const pair = replayPolicyEnvelopePair({ + const pair = await replayPolicyEnvelopePair({ sourceRepoRoot: fixture.sourceRepoRoot, envelope: item.envelope, stateSnapshotRoot: item.stateRoot, @@ -499,10 +946,10 @@ describe("rig replay — exact policy authority", () => { expect(pair.counterfactual.ablated).toEqual(["judgment.confidence"]); }); - test("allows the ablated production pass to change output after reproducing the baseline", () => { + test("allows the ablated production pass to change output after reproducing the baseline", async () => { const fixture = exactRun(); const candidate = PolicyReplayEnvelopeSchema.parse(confidenceEnvelope(fixture)); - const pair = replayPolicyEnvelopePair({ + const pair = await replayPolicyEnvelopePair({ sourceRepoRoot: fixture.sourceRepoRoot, envelope: candidate, stateSnapshotRoot: fixture.stateRoot, @@ -512,29 +959,41 @@ describe("rig replay — exact policy authority", () => { expect(pair.counterfactual.final.counts.warn).toBe(1); }); - test("keeps one branch pair and branch-local Store writes across a multi-envelope sequence", async () => { + test("persists real branch-local policy outcomes across a multi-envelope sequence", async () => { const fixture = exactRun(); const sourceStateBefore = digestPolicyState(join(fixture.sourceRepoRoot, ".reviewgate")); const sourceFileBefore = readFileSync(join(fixture.sourceRepoRoot, "src", "x.ts"), "utf8"); - const expectedRepo = mkdtempSync(join(tmpdir(), "rg-policy-sequence-expected-")); - cpSync(fixture.stateRoot, join(expectedRepo, ".reviewgate"), { recursive: true }); - const event = { - reviewerKey: "codex:quality", - eid: "exact-run:1:confidence-sig:codex:quality", - ts: "2026-08-11T12:00:00.000Z", - }; - await new ReputationStore(expectedRepo).record([{ ...event, outcome: "correct" }], { - now: new Date(event.ts), - }); - const expectedOutput = mkdtempSync(join(tmpdir(), "rg-policy-sequence-state-")); - const secondState = createPolicyStateSnapshot({ - sourceRepoRoot: expectedRepo, - outputRoot: expectedOutput, + const firstEnvelope = PolicyReplayEnvelopeSchema.parse({ + ...confidenceEnvelope(fixture), + history: { + ...fixture.envelope.history, + implicit_outcomes: { + enabled: true, + cap: 100, + created_at: "2026-08-11T12:00:00.000Z", + }, + }, }); const secondTrace = emptyPolicyTrace([...fixture.envelope.raw_response_sha256], { runId: "exact-run", iter: 2, }); + const secondCalls = fixture.envelope.response_calls.map((call) => ({ + ...call, + call_id: policyReplayCallId({ + runId: "exact-run", + iter: 2, + kind: call.kind, + provider: call.provider, + method: call.method, + key: call.key, + promptSha256: call.prompt_sha256, + ordinal: call.ordinal, + slot: call.slot, + attempt: call.attempt, + occurrence: call.occurrence, + }), + })); const secondEnvelope = PolicyReplayEnvelopeSchema.parse({ ...fixture.envelope, iter: 2, @@ -547,63 +1006,185 @@ describe("rig replay — exact policy authority", () => { "+export const x = 2;", "", ].join("\n"), - state_sha256: secondState.stateSha256, + response_calls: secondCalls, + history: { + ...fixture.envelope.history, + implicit_outcomes: { + enabled: true, + cap: 100, + created_at: "2026-08-11T12:00:01.000Z", + }, + }, policy_trace: secondTrace, }); - const branchRoots: Array<{ baseline: string; counterfactual: string }> = []; - - await replayPolicyEnvelopeSequence({ + const implicitAppend = spyOn(ImplicitOutcomeStore.prototype, "append"); + const pairs = await replayPolicyEnvelopeSequence({ sourceRepoRoot: fixture.sourceRepoRoot, passId: "judgment.confidence", items: [ { - envelope: PolicyReplayEnvelopeSchema.parse(fixture.envelope), + envelope: firstEnvelope, stateSnapshotRoot: fixture.stateRoot, }, { envelope: secondEnvelope, - stateSnapshotRoot: join(expectedOutput, secondState.stateRef), + stateSnapshotRoot: fixture.stateRoot, }, ], - afterEnvelope: async ({ index, branches }) => { - branchRoots.push({ - baseline: branches.baseline.checkoutRoot, - counterfactual: branches.counterfactual.checkoutRoot, - }); - if (index === 0) { - await new ReputationStore(branches.baseline.checkoutRoot).record( - [{ ...event, outcome: "correct" }], - { now: new Date(event.ts) }, - ); - await new ReputationStore(branches.counterfactual.checkoutRoot).record( - [{ ...event, outcome: "wrong" }], - { now: new Date(event.ts) }, - ); - return; - } - expect(readFileSync(join(branches.baseline.checkoutRoot, "src", "x.ts"), "utf8")).toBe( - "export const x = 2;\n", - ); - const baselineRep = await new ReputationStore(branches.baseline.checkoutRoot).snapshot(); - const counterfactualRep = await new ReputationStore( - branches.counterfactual.checkoutRoot, - ).snapshot(); - expect(baselineRep.reviewers["codex:quality"]?.correct).toHaveLength(1); - expect(counterfactualRep.reviewers["codex:quality"]?.wrong).toHaveLength(1); - expect(digestPolicyState(join(branches.baseline.checkoutRoot, ".reviewgate"))).not.toBe( - digestPolicyState(join(branches.counterfactual.checkoutRoot, ".reviewgate")), - ); - }, }); - - expect(branchRoots).toHaveLength(2); - expect(branchRoots[1]).toEqual(branchRoots[0]); + const stateful = pairs as unknown as Array<{ + state: { + baseline: { digest: string; implicit_outcomes: number; history_reads: number }; + counterfactual: { digest: string; implicit_outcomes: number; history_reads: number }; + }; + }>; + expect(implicitAppend).toHaveBeenCalled(); + expect(stateful[0]?.state.baseline.implicit_outcomes).toBe(1); + expect(stateful[0]?.state.counterfactual.implicit_outcomes).toBe(0); + expect(stateful[1]?.state.baseline.implicit_outcomes).toBe(1); + expect(stateful[1]?.state.counterfactual.implicit_outcomes).toBe(0); + expect(stateful[1]?.state.baseline.digest).not.toBe(stateful[1]?.state.counterfactual.digest); + expect(stateful[1]?.state.baseline.history_reads).toBeGreaterThan(0); + expect(stateful[1]?.state.counterfactual.history_reads).toBeGreaterThan(0); + implicitAppend.mockRestore(); expect(digestPolicyState(join(fixture.sourceRepoRoot, ".reviewgate"))).toBe(sourceStateBefore); expect(readFileSync(join(fixture.sourceRepoRoot, "src", "x.ts"), "utf8")).toBe( sourceFileBefore, ); }); + test("re-applies only captured human decisions through production learning stores", async () => { + const fixture = exactRun(); + const observedAt = "2026-08-11T12:00:01.000Z"; + const expectedRepo = mkdtempSync(join(tmpdir(), "rg-policy-human-learning-")); + cpSync(fixture.stateRoot, join(expectedRepo, ".reviewgate"), { recursive: true }); + const statePath = join(expectedRepo, ".reviewgate", "state.json"); + const state = JSON.parse(readFileSync(statePath, "utf8")) as { iteration: number }; + state.iteration = 1; + writeFileSync(statePath, JSON.stringify(state), { mode: 0o600 }); + const learningFinding = { + id: "F-001", + signature: "captured-human-decision", + severity: "WARN", + category: "quality", + rule_id: "captured-decision-rule", + file: "src/x.ts", + line_start: 1, + line_end: 1, + message: "Captured human decision finding", + details: "The captured operator accepted this finding.", + reviewer: { provider: "codex", model: "test", persona: "correctness" }, + confidence: 0.9, + consensus: "singleton", + }; + writeFileSync( + join(expectedRepo, ".reviewgate", "pending.json"), + JSON.stringify({ findings: [learningFinding] }), + { mode: 0o600 }, + ); + mkdirSync(join(expectedRepo, ".reviewgate", "decisions")); + writeFileSync( + join(expectedRepo, ".reviewgate", "decisions", "1.jsonl"), + `${JSON.stringify({ + schema: "reviewgate.decision.v1", + finding_id: "F-001", + verdict: "accepted", + action: "fixed", + })}\n`, + { mode: 0o600 }, + ); + await new ReputationStore(expectedRepo).record( + [ + { + reviewerKey: "codex:correctness", + outcome: "correct", + eid: "exact-session:0:1:F-001:codex:correctness", + ts: observedAt, + }, + ], + { now: new Date(observedAt), halfLifeDays: 45 }, + ); + await new FpLedgerStore(expectedRepo).decayPass(observedAt); + const expectedOutput = mkdtempSync(join(tmpdir(), "rg-policy-human-output-")); + const nextState = createPolicyStateSnapshot({ + sourceRepoRoot: expectedRepo, + outputRoot: expectedOutput, + }); + const secondTrace = emptyPolicyTrace([...fixture.envelope.raw_response_sha256], { + runId: "exact-run", + iter: 2, + }); + const secondCalls = fixture.envelope.response_calls.map((call) => ({ + ...call, + call_id: policyReplayCallId({ + runId: "exact-run", + iter: 2, + kind: call.kind, + provider: call.provider, + method: call.method, + key: call.key, + promptSha256: call.prompt_sha256, + ordinal: call.ordinal, + slot: call.slot, + attempt: call.attempt, + occurrence: call.occurrence, + }), + })); + const secondEnvelope = PolicyReplayEnvelopeSchema.parse({ + ...fixture.envelope, + iter: 2, + state_sha256: nextState.stateSha256, + response_calls: secondCalls, + history: { + ...fixture.envelope.history, + fp_ledger: { + enabled: true, + active_at: observedAt, + clusters_at: observedAt, + }, + reputation: { + ...fixture.envelope.history.reputation, + observed_at: observedAt, + }, + }, + policy_trace: secondTrace, + }); + const fpWrite = spyOn(FpLedgerStore.prototype, "decayPass"); + const reputationWrite = spyOn(ReputationStore.prototype, "record"); + try { + await replayPolicyEnvelopePair({ + sourceRepoRoot: fixture.sourceRepoRoot, + envelope: secondEnvelope, + stateSnapshotRoot: join(expectedOutput, nextState.stateRef), + passId: "judgment.confidence", + }); + expect(fpWrite).not.toHaveBeenCalled(); + expect(reputationWrite).not.toHaveBeenCalled(); + + const pairs = await replayPolicyEnvelopeSequence({ + sourceRepoRoot: fixture.sourceRepoRoot, + passId: "judgment.confidence", + items: [ + { + envelope: PolicyReplayEnvelopeSchema.parse(fixture.envelope), + stateSnapshotRoot: fixture.stateRoot, + }, + { + envelope: secondEnvelope, + stateSnapshotRoot: join(expectedOutput, nextState.stateRef), + }, + ], + }); + expect(fpWrite).toHaveBeenCalled(); + expect(reputationWrite).toHaveBeenCalled(); + expect(pairs[1]?.state.baseline.history_writes).toBeGreaterThanOrEqual(2); + expect(pairs[1]?.state.counterfactual.history_writes).toBeGreaterThanOrEqual(2); + } finally { + fpWrite.mockRestore(); + reputationWrite.mockRestore(); + } + }); + test("rejects the authoritative invalidity matrix before metrics", () => { const cases: Array<{ name: string; @@ -663,23 +1244,96 @@ describe("rig replay — exact policy authority", () => { replaceTrace(fixture, { ...fixture.envelope, raw_response_sha256: [unknown], + response_calls: fixture.envelope.response_calls.map((call) => ({ + ...call, + response_sha256: unknown, + })), policy_trace: policyTrace, }); }, }, { - name: "response order", + name: "response call attempt", + code: "invalid-trace", + mutate: (fixture) => { + const first = fixture.envelope.response_calls[0]; + if (first === undefined) throw new Error("response call fixture missing"); + replaceRawTrace(fixture, { + ...fixture.envelope, + response_calls: [ + { + ...first, + attempt: first.attempt + 1, + // Deliberately retain call_id: attempt is part of stable identity. + }, + ], + }); + }, + }, + { + name: "recomputed response call attempt", + code: "response-hash-mismatch", + mutate: (fixture) => { + const first = fixture.envelope.response_calls[0]; + if (first === undefined) throw new Error("response call fixture missing"); + const attempt = first.attempt + 1; + replaceTrace(fixture, { + ...fixture.envelope, + response_calls: [ + { + ...first, + attempt, + call_id: policyReplayCallId({ + runId: fixture.envelope.run_id, + iter: fixture.envelope.iter, + kind: first.kind, + provider: first.provider, + method: first.method, + key: first.key, + promptSha256: first.prompt_sha256, + ordinal: first.ordinal, + slot: first.slot, + attempt, + occurrence: first.occurrence, + }), + }, + ], + }); + }, + }, + { + name: "response call identity", code: "response-hash-mismatch", mutate: (fixture) => { const first = "safe first recorded response"; const second = "safe second recorded response"; const hashes = [sha256(first), sha256(second)]; + const firstPrompt = "b".repeat(64); + const secondPrompt = "c".repeat(64); replaceTrace(fixture, { ...fixture.envelope, raw_response_sha256: hashes, + response_calls: [ + responseCall({ + kind: "reviewer", + method: "review", + key: "openrouter-first", + promptSha256: firstPrompt, + slot: 0, + rawText: first, + }), + responseCall({ + kind: "reviewer", + method: "review", + key: "openrouter-second", + promptSha256: secondPrompt, + slot: 1, + rawText: second, + }), + ], policy_trace: emptyPolicyTrace(hashes), }); - const reversed = `${entry("openrouter-second", { rawText: second })}\n${entry("openrouter-first", { rawText: first })}\n`; + const reversed = `${responseEntry({ method: "review", key: "openrouter-first", promptSha256: firstPrompt, rawText: second })}\n${responseEntry({ method: "review", key: "openrouter-second", promptSha256: secondPrompt, rawText: first })}\n`; const cassettePath = join(fixture.root, "cassette.jsonl"); writeFileSync(cassettePath, reversed, { mode: 0o600 }); if (fixture.manifest.policyReplay) { From 49d000ea700a6263782d6f912d96f16333c2e4d7 Mon Sep 17 00:00:00 2001 From: Codevena Date: Tue, 11 Aug 2026 18:02:51 +0200 Subject: [PATCH 47/55] fix(rig): preserve counterfactual state ownership --- src/rig/policy-replay-state.ts | 165 ++++++++++++++++++----- src/rig/replay.ts | 44 ++++-- tests/unit/policy-replay-capture.test.ts | 74 ++++++++++ tests/unit/rig-replay.test.ts | 44 +++++- 4 files changed, 281 insertions(+), 46 deletions(-) diff --git a/src/rig/policy-replay-state.ts b/src/rig/policy-replay-state.ts index 2198eb2..f7f25c2 100644 --- a/src/rig/policy-replay-state.ts +++ b/src/rig/policy-replay-state.ts @@ -96,6 +96,8 @@ export interface PolicyStateSnapshot { export interface ReplayBranch { checkoutRoot: string; startingStateSha256: string; + /** Files whose current presence or absence is causally owned by this replay branch. */ + ownedStatePaths: Map; } export interface ReplayBranches { @@ -297,15 +299,48 @@ function mkdirIfMissing(path: string): void { } } +function validateDirectoryComponentName(name: string): void { + if (name.length === 0 || name === "." || name === ".." || name.includes(sep)) { + throw new Error(`invalid policy state directory component: ${name}`); + } +} + +/** Validate every already-existing directory component without following an ancestor symlink. */ +function exactContainedDirectoryChain(rootPath: string, components: string[]): string { + const rootReal = exactDirectory(rootPath); + let current = rootReal; + for (const component of components) { + validateDirectoryComponentName(component); + const candidate = join(current, component); + const before = lstatSync(candidate); + if (before.isSymbolicLink() || !before.isDirectory()) { + throw new Error(`expected an ordinary directory: ${candidate}`); + } + const candidateReal = realpathSync(candidate); + const after = lstatSync(candidate); + if ( + after.isSymbolicLink() || + !after.isDirectory() || + before.dev !== after.dev || + before.ino !== after.ino + ) { + throw new Error(`policy state directory changed while validating: ${candidate}`); + } + if (!isContained(rootReal, candidateReal)) { + throw new Error(`policy state directory escapes real root: ${candidate}`); + } + current = candidateReal; + } + return current; +} + function ensureContainedDirectory( rootPath: string, rootReal: string, parent: string, name: string, ): string { - if (name.length === 0 || name === "." || name === ".." || name.includes(sep)) { - throw new Error(`invalid policy state directory component: ${name}`); - } + validateDirectoryComponentName(name); const parentReal = exactDirectory(parent); if (!isContained(rootReal, parentReal)) { throw new Error(`policy state directory parent escapes root: ${parent}`); @@ -355,19 +390,41 @@ function copyEntries(entries: StateEntry[], destinationRoot: string): void { } } -function sameStateEntry(left: StateEntry | undefined, right: StateEntry | undefined): boolean { - return left?.path === right?.path && left?.size === right?.size && left?.sha256 === right?.sha256; +/** Record one real Store decision, including an absent result as an explicit tombstone. */ +export function recordReplayBranchStateDecision(branch: ReplayBranch, stateFilePath: string): void { + const stateRootPath = resolve(branch.checkoutRoot, ".reviewgate"); + const stateRoot = exactDirectory(stateRootPath); + const unresolved = resolve(stateFilePath); + if (!isContained(stateRootPath, unresolved)) { + throw new Error("replay Store decision escapes branch-local policy state"); + } + const relativePath = relative(stateRootPath, unresolved).split(sep).join("/"); + validateRelativeStatePath(relativePath); + let status: "present" | "absent" = "absent"; + try { + const entry = lstatSync(unresolved); + if (entry.isSymbolicLink() || !entry.isFile() || entry.nlink !== 1) { + throw new Error(`replay Store decision produced a non-private file: ${relativePath}`); + } + const targetReal = realpathSync(unresolved); + if (!isContained(stateRoot, targetReal)) { + throw new Error(`replay Store decision escapes branch-local policy state: ${relativePath}`); + } + readStableFile(unresolved); + status = "present"; + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; + } + branch.ownedStatePaths.set(relativePath, status); } /** - * Apply the recorded production-state transition without erasing branch-local writes. - * - * This is a file-level three-way merge: previous capture is the base, next capture supplies - * exogenous production changes, and a branch-local change wins a same-file conflict. Store APIs - * still own the bytes and schema; replay does not model or reinterpret any learning format. + * Advance one branch by importing only exogenous captured state. A Store-owned path retains the + * branch's current bytes, while an owned absence is a tombstone that prevents baseline capture + * bytes from leaking into the counterfactual. Untouched paths always advance to the next capture. */ function advanceBranchState(input: { - checkoutRoot: string; + branch: ReplayBranch; previousStateSnapshotRoot: string; nextStateSnapshotRoot: string; }): void { @@ -377,28 +434,32 @@ function advanceBranchState(input: { const next = new Map( collectStateEntries(input.nextStateSnapshotRoot).map((entry) => [entry.path, entry]), ); - const branchStateRoot = join(input.checkoutRoot, ".reviewgate"); + const branchStateRoot = join(input.branch.checkoutRoot, ".reviewgate"); const branch = new Map(collectStateEntries(branchStateRoot).map((entry) => [entry.path, entry])); - const paths = [...new Set([...previous.keys(), ...next.keys(), ...branch.keys()])].sort( - compareCodeUnits, - ); + const paths = [ + ...new Set([ + ...previous.keys(), + ...next.keys(), + ...branch.keys(), + ...input.branch.ownedStatePaths.keys(), + ]), + ].sort(compareCodeUnits); const merged: StateEntry[] = []; for (const path of paths) { - const baseEntry = previous.get(path); const nextEntry = next.get(path); const branchEntry = branch.get(path); - const branchChanged = !sameStateEntry(branchEntry, baseEntry); - const productionChanged = !sameStateEntry(nextEntry, baseEntry); - const selected = - branchChanged && productionChanged && !sameStateEntry(branchEntry, nextEntry) - ? branchEntry - : branchChanged - ? branchEntry - : nextEntry; + const ownership = input.branch.ownedStatePaths.get(path); + if (ownership !== undefined) { + const actual = branchEntry === undefined ? "absent" : "present"; + if (actual !== ownership) { + throw new Error(`replay Store ownership drifted for ${path}`); + } + } + const selected = ownership === undefined ? nextEntry : branchEntry; if (selected !== undefined) merged.push(selected); } rmSync(branchStateRoot, { recursive: true, force: true }); - ensureDirectoryChain(input.checkoutRoot, [".reviewgate"]); + ensureDirectoryChain(input.branch.checkoutRoot, [".reviewgate"]); copyEntries(merged, branchStateRoot); } @@ -442,13 +503,27 @@ export function createPolicyStateSnapshot(input: { throw new Error("policy state output escapes root"); if (!existsSync(stateDestination)) { ensureDirectoryChain(outputReal, ["policy-state", stateSha256, ".reviewgate"]); - copyEntries(entries, stateDestination); - } else { - exactDirectory(stateDestination); + const createdStateRoot = exactContainedDirectoryChain(outputReal, [ + "policy-state", + stateSha256, + ".reviewgate", + ]); + copyEntries(entries, createdStateRoot); } - if (stateDigest(collectStateEntries(stateDestination, true)) !== stateSha256) { + const stateDestinationReal = exactContainedDirectoryChain(outputReal, [ + "policy-state", + stateSha256, + ".reviewgate", + ]); + if (stateDigest(collectStateEntries(stateDestinationReal, true)) !== stateSha256) { throw new Error("policy state snapshot digest mismatch"); } + if ( + exactContainedDirectoryChain(outputReal, ["policy-state", stateSha256, ".reviewgate"]) !== + stateDestinationReal + ) { + throw new Error("policy state snapshot directory changed while reading"); + } const manifest: PolicyStateManifest = { schema: "reviewgate.policy-state-snapshot.v1", @@ -464,7 +539,8 @@ export function createPolicyStateSnapshot(input: { const manifestSha256 = sha256(bytes); const ref = `policy-state/${manifestSha256}.json`; if (!STATE_MANIFEST_REF.test(ref)) throw new Error("invalid policy state manifest reference"); - const destination = resolve(outputReal, ref); + const policyStateReal = exactContainedDirectoryChain(outputReal, ["policy-state"]); + const destination = resolve(policyStateReal, `${manifestSha256}.json`); if (!isContained(outputReal, destination)) throw new Error("policy state manifest escapes root"); ensureDirectoryChain(outputReal, ["policy-state"]); if (!writeFileIfAbsent(destination, bytes, { mode: 0o600 })) { @@ -490,7 +566,8 @@ export function verifyPolicyStateSnapshot(input: { throw new Error("invalid policy state snapshot reference"); } const outputReal = exactDirectory(input.outputRoot); - const manifestPath = resolve(outputReal, input.ref); + const policyStateReal = exactContainedDirectoryChain(outputReal, ["policy-state"]); + const manifestPath = resolve(policyStateReal, basename(input.ref)); if (!isContained(outputReal, manifestPath)) throw new Error("policy state manifest escapes root"); const bytes = readStableFile(manifestPath, STATE_MAX_FILE_BYTES, true); if (sha256(bytes) !== input.sha256 || input.ref !== `policy-state/${input.sha256}.json`) { @@ -520,9 +597,23 @@ export function verifyPolicyStateSnapshot(input: { ) { throw new Error("policy state tree escapes root"); } - const entries = collectStateEntries(stateRoot, true); + const stateRootReal = exactContainedDirectoryChain(outputReal, [ + "policy-state", + manifest.state_sha256, + ".reviewgate", + ]); + const entries = collectStateEntries(stateRootReal, true); if (stateDigest(entries) !== manifest.state_sha256) throw new Error("policy state tree digest mismatch"); + if ( + exactContainedDirectoryChain(outputReal, [ + "policy-state", + manifest.state_sha256, + ".reviewgate", + ]) !== stateRootReal + ) { + throw new Error("policy state tree changed while reading"); + } const actualFiles = entries.map(({ path, size, sha256: contentSha256 }) => ({ path, size, @@ -531,7 +622,7 @@ export function verifyPolicyStateSnapshot(input: { if (canonicalJson(actualFiles) !== canonicalJson(manifest.files)) { throw new Error("policy state tree does not match manifest"); } - return { stateRoot, stateSha256: manifest.state_sha256 }; + return { stateRoot: stateRootReal, stateSha256: manifest.state_sha256 }; } function authority(code: RigAuthorityInvalidity, message: string): never { @@ -841,7 +932,11 @@ function prepareBranch(input: { reverse: false, label: basename(input.destination), }); - return { checkoutRoot: input.destination, startingStateSha256: input.expectedStateSha256 }; + return { + checkoutRoot: input.destination, + startingStateSha256: input.expectedStateSha256, + ownedStatePaths: new Map(), + }; } export function createReplayBranches(input: { @@ -963,7 +1058,7 @@ export function advanceReplayBranches(input: { label: `${label}-next`, }); advanceBranchState({ - checkoutRoot: branch.checkoutRoot, + branch, previousStateSnapshotRoot: input.previousStateSnapshotRoot, nextStateSnapshotRoot: input.nextStateSnapshotRoot, }); diff --git a/src/rig/replay.ts b/src/rig/replay.ts index a29d1e1..3631887 100644 --- a/src/rig/replay.ts +++ b/src/rig/replay.ts @@ -31,6 +31,7 @@ import type { PolicyTrace } from "../schemas/policy-trace.ts"; import { RigManifestSchema } from "../schemas/rig-manifest.ts"; import type { RigResult } from "../schemas/rig-result.ts"; import { compareCodeUnits } from "../utils/compare.ts"; +import { implicitOutcomesPath, knownFpPath, reputationJsonPath } from "../utils/paths.ts"; import { type RigAblation, SUPPRESSION_LAYERS, ablate, seededTagsFromScript } from "./ablate.ts"; import { harvest } from "./harvest.ts"; import { @@ -40,6 +41,7 @@ import { cleanupReplayBranches, createReplayBranches, digestPolicyState, + recordReplayBranchStateDecision, validateRigPolicyReplayArtifacts, } from "./policy-replay-state.ts"; @@ -230,12 +232,15 @@ async function assertBranchHistoryInputs( async function applyCapturedHumanLearning( envelope: PolicyReplayEnvelope, - checkoutRoot: string, + branch: ReplayBranches["baseline"], ): Promise { + const checkoutRoot = branch.checkoutRoot; const state = await new StateStore(checkoutRoot).load(); if (state.iteration < 1) return 0; let writes = 0; if (envelope.history.fp_ledger.enabled) { + const path = knownFpPath(checkoutRoot); + const before = readOptionalStateFile(path); const store = new FpLedgerStore(checkoutRoot); await learnFromDecisions({ repoRoot: checkoutRoot, @@ -246,9 +251,14 @@ async function applyCapturedHumanLearning( nowIso: envelope.history.fp_ledger.active_at, }); await store.decayPass(envelope.history.fp_ledger.active_at); + if (!sameOptionalBytes(before, readOptionalStateFile(path))) { + recordReplayBranchStateDecision(branch, path); + } writes += 1; } if (envelope.history.reputation.enabled) { + const path = reputationJsonPath(checkoutRoot); + const before = readOptionalStateFile(path); await learnReputationFromDecisions({ repoRoot: checkoutRoot, iter: state.iteration, @@ -258,11 +268,27 @@ async function applyCapturedHumanLearning( nowIso: envelope.history.reputation.observed_at, halfLifeDays: envelope.history.reputation.half_life_days, }); + if (!sameOptionalBytes(before, readOptionalStateFile(path))) { + recordReplayBranchStateDecision(branch, path); + } writes += 1; } return writes; } +function readOptionalStateFile(path: string): Buffer | null { + try { + return readFileSync(path); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") return null; + throw error; + } +} + +function sameOptionalBytes(left: Buffer | null, right: Buffer | null): boolean { + return left === null ? right === null : right !== null && left.equals(right); +} + function aggregateInputFromEnvelope( envelope: PolicyReplayEnvelope, findings: PolicyReplayEnvelope["aggregate"]["findings"], @@ -431,7 +457,7 @@ function replayEnvelopeProductionPath(input: { async function persistBranchPolicyOutcomes(input: { envelope: PolicyReplayEnvelope; - checkoutRoot: string; + branch: ReplayBranches["baseline"]; execution: ReplayPolicyExecution; }): Promise { if (!input.envelope.history.implicit_outcomes.enabled) return 0; @@ -444,10 +470,11 @@ async function persistBranchPolicyOutcomes(input: { nowIso: input.envelope.history.implicit_outcomes.created_at, }, ); - await new ImplicitOutcomeStore(input.checkoutRoot).append( + await new ImplicitOutcomeStore(input.branch.checkoutRoot).append( outcomes, input.envelope.history.implicit_outcomes.cap, ); + recordReplayBranchStateDecision(input.branch, implicitOutcomesPath(input.branch.checkoutRoot)); return outcomes.length; } @@ -500,12 +527,12 @@ async function replayPolicyEnvelopeInBranches(input: { } const baselineOutcomeWrites = await persistBranchPolicyOutcomes({ envelope: input.envelope, - checkoutRoot: input.branches.baseline.checkoutRoot, + branch: input.branches.baseline, execution: baselineExecution, }); const counterfactualOutcomeWrites = await persistBranchPolicyOutcomes({ envelope: input.envelope, - checkoutRoot: input.branches.counterfactual.checkoutRoot, + branch: input.branches.counterfactual, execution: counterfactualExecution, }); return { @@ -599,13 +626,10 @@ export async function replayPolicyEnvelopeSequence(input: { previous === undefined ? { baseline: 0, counterfactual: 0 } : { - baseline: await applyCapturedHumanLearning( - item.envelope, - branches.baseline.checkoutRoot, - ), + baseline: await applyCapturedHumanLearning(item.envelope, branches.baseline), counterfactual: await applyCapturedHumanLearning( item.envelope, - branches.counterfactual.checkoutRoot, + branches.counterfactual, ), }; const pair = await replayPolicyEnvelopeInBranches({ diff --git a/tests/unit/policy-replay-capture.test.ts b/tests/unit/policy-replay-capture.test.ts index 5313b5c..fc6631e 100644 --- a/tests/unit/policy-replay-capture.test.ts +++ b/tests/unit/policy-replay-capture.test.ts @@ -2,6 +2,7 @@ import { describe, expect, test } from "bun:test"; import { execFileSync } from "node:child_process"; import { chmodSync, + cpSync, existsSync, linkSync, lstatSync, @@ -10,6 +11,7 @@ import { readFileSync, readdirSync, realpathSync, + renameSync, symlinkSync, writeFileSync, } from "node:fs"; @@ -503,6 +505,78 @@ describe("policy replay state isolation", () => { expect(readdirSync(outside)).toEqual([]); }); + test("create rejects existing digest-tree symlink ancestors even when the external tree is identical", () => { + for (const ancestor of ["state-digest", "state-root"] as const) { + const sourceRepoRoot = gitRepo(); + const seedOutput = mkdtempSync(join(tmpdir(), "rg-policy-state-seed-")); + const seed = createPolicyStateSnapshot({ sourceRepoRoot, outputRoot: seedOutput }); + const outside = mkdtempSync(join(tmpdir(), "rg-policy-state-identical-")); + const externalDigestTree = join(outside, seed.stateSha256); + cpSync(join(seedOutput, "policy-state", seed.stateSha256), externalDigestTree, { + recursive: true, + }); + const outsideDigestBefore = digestPolicyState(join(externalDigestTree, ".reviewgate")); + const outputRoot = mkdtempSync(join(tmpdir(), "rg-policy-state-output-")); + mkdirSync(join(outputRoot, "policy-state")); + if (ancestor === "state-digest") { + symlinkSync(externalDigestTree, join(outputRoot, "policy-state", seed.stateSha256)); + } else { + mkdirSync(join(outputRoot, "policy-state", seed.stateSha256)); + symlinkSync( + join(externalDigestTree, ".reviewgate"), + join(outputRoot, "policy-state", seed.stateSha256, ".reviewgate"), + ); + } + + expect(() => createPolicyStateSnapshot({ sourceRepoRoot, outputRoot }), ancestor).toThrow( + /symlink|escape|ordinary directory/i, + ); + expect(digestPolicyState(join(externalDigestTree, ".reviewgate")), ancestor).toBe( + outsideDigestBefore, + ); + } + }); + + test("verify rejects digest-tree symlink ancestors even when manifest and external bytes match", () => { + for (const ancestor of ["state-digest", "state-root"] as const) { + const sourceRepoRoot = gitRepo(); + const outputRoot = mkdtempSync(join(tmpdir(), "rg-policy-state-output-")); + const snapshot = createPolicyStateSnapshot({ sourceRepoRoot, outputRoot }); + const persistedDigestTree = join(outputRoot, "policy-state", snapshot.stateSha256); + const outside = mkdtempSync(join(tmpdir(), "rg-policy-state-verify-identical-")); + const externalDigestTree = join(outside, snapshot.stateSha256); + if (ancestor === "state-digest") { + renameSync(persistedDigestTree, externalDigestTree); + symlinkSync(externalDigestTree, persistedDigestTree); + } else { + mkdirSync(externalDigestTree); + renameSync( + join(persistedDigestTree, ".reviewgate"), + join(externalDigestTree, ".reviewgate"), + ); + symlinkSync( + join(externalDigestTree, ".reviewgate"), + join(persistedDigestTree, ".reviewgate"), + ); + } + const outsideDigestBefore = digestPolicyState(join(externalDigestTree, ".reviewgate")); + + expect( + () => + verifyPolicyStateSnapshot({ + outputRoot, + ref: snapshot.ref, + sha256: snapshot.sha256, + expectedStateSha256: snapshot.stateSha256, + }), + ancestor, + ).toThrow(/symlink|escape|ordinary directory/i); + expect(digestPolicyState(join(externalDigestTree, ".reviewgate")), ancestor).toBe( + outsideDigestBefore, + ); + } + }); + test("uses one code-unit order for traversal, digest, manifest, and verification", () => { const sourceRepoRoot = gitRepo(); writeFileSync(join(sourceRepoRoot, ".reviewgate", "Z.json"), "upper\n"); diff --git a/tests/unit/rig-replay.test.ts b/tests/unit/rig-replay.test.ts index 263f2c5..6936d7f 100644 --- a/tests/unit/rig-replay.test.ts +++ b/tests/unit/rig-replay.test.ts @@ -974,6 +974,45 @@ describe("rig replay — exact policy authority", () => { }, }, }); + const capturedBaselineRepo = mkdtempSync(join(tmpdir(), "rg-policy-baseline-capture-")); + cpSync(fixture.stateRoot, join(capturedBaselineRepo, ".reviewgate"), { recursive: true }); + await new ImplicitOutcomeStore(capturedBaselineRepo).append( + [ + { + schema: "reviewgate.implicit_outcome.v1", + signature: "confidence-sig", + reviewer_key: "codex:correctness", + category: "quality", + demote_reason: "low_confidence", + run_id: "exact-run", + iter: 1, + created_at: "2026-08-11T12:00:00.000Z", + }, + ], + 100, + ); + writeFileSync( + join(capturedBaselineRepo, ".reviewgate", "exogenous.json"), + '{"source":"captured-iteration-2"}\n', + { mode: 0o600 }, + ); + const nextOutput = mkdtempSync(join(tmpdir(), "rg-policy-next-state-")); + const nextState = createPolicyStateSnapshot({ + sourceRepoRoot: capturedBaselineRepo, + outputRoot: nextOutput, + }); + const expectedCounterfactualRepo = mkdtempSync(join(tmpdir(), "rg-policy-cf-expected-")); + cpSync(fixture.stateRoot, join(expectedCounterfactualRepo, ".reviewgate"), { + recursive: true, + }); + writeFileSync( + join(expectedCounterfactualRepo, ".reviewgate", "exogenous.json"), + '{"source":"captured-iteration-2"}\n', + { mode: 0o600 }, + ); + const expectedCounterfactualDigest = digestPolicyState( + join(expectedCounterfactualRepo, ".reviewgate"), + ); const secondTrace = emptyPolicyTrace([...fixture.envelope.raw_response_sha256], { runId: "exact-run", iter: 2, @@ -997,6 +1036,7 @@ describe("rig replay — exact policy authority", () => { const secondEnvelope = PolicyReplayEnvelopeSchema.parse({ ...fixture.envelope, iter: 2, + state_sha256: nextState.stateSha256, exact_diff: [ "diff --git a/src/x.ts b/src/x.ts", "--- a/src/x.ts", @@ -1028,7 +1068,7 @@ describe("rig replay — exact policy authority", () => { }, { envelope: secondEnvelope, - stateSnapshotRoot: fixture.stateRoot, + stateSnapshotRoot: join(nextOutput, nextState.stateRef), }, ], }); @@ -1043,6 +1083,8 @@ describe("rig replay — exact policy authority", () => { expect(stateful[0]?.state.counterfactual.implicit_outcomes).toBe(0); expect(stateful[1]?.state.baseline.implicit_outcomes).toBe(1); expect(stateful[1]?.state.counterfactual.implicit_outcomes).toBe(0); + expect(stateful[1]?.state.baseline.digest).toBe(nextState.stateSha256); + expect(stateful[1]?.state.counterfactual.digest).toBe(expectedCounterfactualDigest); expect(stateful[1]?.state.baseline.digest).not.toBe(stateful[1]?.state.counterfactual.digest); expect(stateful[1]?.state.baseline.history_reads).toBeGreaterThan(0); expect(stateful[1]?.state.counterfactual.history_reads).toBeGreaterThan(0); From 9fdc09fee17934a12bef2920abfc6c8684b5c3ba Mon Sep 17 00:00:00 2001 From: Codevena Date: Tue, 11 Aug 2026 18:16:03 +0200 Subject: [PATCH 48/55] fix(rig): contain envelope replay state --- src/rig/policy-replay-state.ts | 55 +++++++++++--------------- tests/unit/rig-replay.test.ts | 71 +++++++++++++++++++++++++++++++++- 2 files changed, 92 insertions(+), 34 deletions(-) diff --git a/src/rig/policy-replay-state.ts b/src/rig/policy-replay-state.ts index f7f25c2..a268d06 100644 --- a/src/rig/policy-replay-state.ts +++ b/src/rig/policy-replay-state.ts @@ -334,6 +334,19 @@ function exactContainedDirectoryChain(rootPath: string, components: string[]): s return current; } +function readPersistedPolicyStateTree( + outputRoot: string, + stateSha256: string, +): { stateRoot: string; entries: StateEntry[] } { + const components = ["policy-state", stateSha256, ".reviewgate"]; + const stateRoot = exactContainedDirectoryChain(outputRoot, components); + const entries = collectStateEntries(stateRoot, true); + if (exactContainedDirectoryChain(outputRoot, components) !== stateRoot) { + throw new Error("policy state tree changed while reading"); + } + return { stateRoot, entries }; +} + function ensureContainedDirectory( rootPath: string, rootReal: string, @@ -510,20 +523,10 @@ export function createPolicyStateSnapshot(input: { ]); copyEntries(entries, createdStateRoot); } - const stateDestinationReal = exactContainedDirectoryChain(outputReal, [ - "policy-state", - stateSha256, - ".reviewgate", - ]); - if (stateDigest(collectStateEntries(stateDestinationReal, true)) !== stateSha256) { + const persistedState = readPersistedPolicyStateTree(outputReal, stateSha256); + if (stateDigest(persistedState.entries) !== stateSha256) { throw new Error("policy state snapshot digest mismatch"); } - if ( - exactContainedDirectoryChain(outputReal, ["policy-state", stateSha256, ".reviewgate"]) !== - stateDestinationReal - ) { - throw new Error("policy state snapshot directory changed while reading"); - } const manifest: PolicyStateManifest = { schema: "reviewgate.policy-state-snapshot.v1", @@ -597,23 +600,10 @@ export function verifyPolicyStateSnapshot(input: { ) { throw new Error("policy state tree escapes root"); } - const stateRootReal = exactContainedDirectoryChain(outputReal, [ - "policy-state", - manifest.state_sha256, - ".reviewgate", - ]); - const entries = collectStateEntries(stateRootReal, true); + const persistedState = readPersistedPolicyStateTree(outputReal, manifest.state_sha256); + const entries = persistedState.entries; if (stateDigest(entries) !== manifest.state_sha256) throw new Error("policy state tree digest mismatch"); - if ( - exactContainedDirectoryChain(outputReal, [ - "policy-state", - manifest.state_sha256, - ".reviewgate", - ]) !== stateRootReal - ) { - throw new Error("policy state tree changed while reading"); - } const actualFiles = entries.map(({ path, size, sha256: contentSha256 }) => ({ path, size, @@ -622,7 +612,7 @@ export function verifyPolicyStateSnapshot(input: { if (canonicalJson(actualFiles) !== canonicalJson(manifest.files)) { throw new Error("policy state tree does not match manifest"); } - return { stateRoot: stateRootReal, stateSha256: manifest.state_sha256 }; + return { stateRoot: persistedState.stateRoot, stateSha256: manifest.state_sha256 }; } function authority(code: RigAuthorityInvalidity, message: string): never { @@ -805,12 +795,11 @@ export function validateRigPolicyReplayArtifacts(input: { return authority("invalid-trace", `duplicate replay identity ${identity}`); } identities.add(identity); - const stateRoot = resolve(outputRoot, `policy-state/${envelope.state_sha256}/.reviewgate`); + let stateRoot: string; try { - if ( - !isContained(outputRoot, stateRoot) || - stateDigest(collectStateEntries(stateRoot, true)) !== envelope.state_sha256 - ) { + const persistedState = readPersistedPolicyStateTree(outputRoot, envelope.state_sha256); + stateRoot = persistedState.stateRoot; + if (stateDigest(persistedState.entries) !== envelope.state_sha256) { return authority( "state-digest-mismatch", `turn ${turn.index} trace ${trace.ref} state does not match`, diff --git a/tests/unit/rig-replay.test.ts b/tests/unit/rig-replay.test.ts index 6936d7f..e1f0cda 100644 --- a/tests/unit/rig-replay.test.ts +++ b/tests/unit/rig-replay.test.ts @@ -1,7 +1,15 @@ import { describe, expect, spyOn, test } from "bun:test"; import { execFileSync } from "node:child_process"; import { createHash } from "node:crypto"; -import { cpSync, mkdirSync, mkdtempSync, readFileSync, writeFileSync } from "node:fs"; +import { + cpSync, + mkdirSync, + mkdtempSync, + readFileSync, + renameSync, + symlinkSync, + writeFileSync, +} from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { canonicalJson } from "../../src/audit/canonical.ts"; @@ -522,6 +530,20 @@ function replaceRawTrace( turn.policyReplay = { status: "complete", traces: [{ ref, sha256: hash }] }; } +function captureDistinctEnvelopeState(fixture: ReturnType) { + const capturedRepo = mkdtempSync(join(tmpdir(), "rg-policy-envelope-state-")); + cpSync(fixture.stateRoot, join(capturedRepo, ".reviewgate"), { recursive: true }); + writeFileSync(join(capturedRepo, ".reviewgate", "envelope-only.json"), '{"iter":1}\n', { + mode: 0o600, + }); + const state = createPolicyStateSnapshot({ + sourceRepoRoot: capturedRepo, + outputRoot: fixture.root, + }); + replaceTrace(fixture, { ...fixture.envelope, state_sha256: state.stateSha256 }); + return state; +} + function confidenceEnvelope(fixture: ReturnType): PolicyReplayEnvelopeInput { const finding = { id: "confidence-1", @@ -765,6 +787,53 @@ describe("rig replay — exact policy authority", () => { } }); + test("rejects a symlinked digest ancestor for a separately captured envelope state", () => { + const fixture = exactRun(); + const state = captureDistinctEnvelopeState(fixture); + expect(state.stateSha256).not.toBe(fixture.manifest.policyReplay?.initialStateDigest); + expect(() => + validateRigPolicyReplayArtifacts({ + manifest: fixture.manifest, + manifestPath: fixture.manifestPath, + }), + ).not.toThrow(); + + const persistedDigestTree = join(fixture.root, "policy-state", state.stateSha256); + const outside = mkdtempSync(join(tmpdir(), "rg-policy-envelope-state-outside-")); + const externalDigestTree = join(outside, state.stateSha256); + renameSync(persistedDigestTree, externalDigestTree); + const outsideDigestBefore = digestPolicyState(join(externalDigestTree, ".reviewgate")); + symlinkSync(externalDigestTree, persistedDigestTree); + + expect(() => + validateRigPolicyReplayArtifacts({ + manifest: fixture.manifest, + manifestPath: fixture.manifestPath, + }), + ).toThrow(expect.objectContaining({ code: "state-digest-mismatch", exitCode: 4 })); + expect(digestPolicyState(join(externalDigestTree, ".reviewgate"))).toBe(outsideDigestBefore); + }); + + test("rejects a direct state-root symlink for a separately captured envelope state", () => { + const fixture = exactRun(); + const state = captureDistinctEnvelopeState(fixture); + const persistedDigestTree = join(fixture.root, "policy-state", state.stateSha256); + const persistedStateRoot = join(persistedDigestTree, ".reviewgate"); + const outside = mkdtempSync(join(tmpdir(), "rg-policy-envelope-root-outside-")); + const externalStateRoot = join(outside, ".reviewgate"); + renameSync(persistedStateRoot, externalStateRoot); + const outsideDigestBefore = digestPolicyState(externalStateRoot); + symlinkSync(externalStateRoot, persistedStateRoot); + + expect(() => + validateRigPolicyReplayArtifacts({ + manifest: fixture.manifest, + manifestPath: fixture.manifestPath, + }), + ).toThrow(expect.objectContaining({ code: "state-digest-mismatch", exitCode: 4 })); + expect(digestPolicyState(externalStateRoot)).toBe(outsideDigestBefore); + }); + test("matches logical response calls by identity when physical completion order reverses", () => { const fixture = exactRun(); const first = "safe slow logical response"; From ade84226130fe5f21184c5aa443d6a395d0f9252 Mon Sep 17 00:00:00 2001 From: Codevena Date: Tue, 11 Aug 2026 18:39:18 +0200 Subject: [PATCH 49/55] fix(rig): replay pre-aggregation ablations --- src/rig/replay.ts | 5 +- tests/unit/rig-replay.test.ts | 210 ++++++++++++++++++++++++++++++++++ 2 files changed, 214 insertions(+), 1 deletion(-) diff --git a/src/rig/replay.ts b/src/rig/replay.ts index 3631887..e2faa61 100644 --- a/src/rig/replay.ts +++ b/src/rig/replay.ts @@ -401,7 +401,10 @@ function replayEnvelopeProductionPath(input: { runtime, ); } - if (canonicalJson(grounded) !== canonicalJson(envelope.aggregate.findings)) { + if ( + input.verifyOriginal && + canonicalJson(grounded) !== canonicalJson(envelope.aggregate.findings) + ) { throw new Error("captured aggregate findings do not match production pre-policy replay"); } const result = aggregate(aggregateInputFromEnvelope(envelope, grounded, runtime)); diff --git a/tests/unit/rig-replay.test.ts b/tests/unit/rig-replay.test.ts index e1f0cda..3ebf10f 100644 --- a/tests/unit/rig-replay.test.ts +++ b/tests/unit/rig-replay.test.ts @@ -40,6 +40,7 @@ import { replayPolicyEnvelopeSequence, runWithReplayProviderCeiling, } from "../../src/rig/replay.ts"; +import type { Finding } from "../../src/schemas/finding.ts"; import { type PolicyReplayEnvelopeInput, PolicyReplayEnvelopeSchema, @@ -643,6 +644,122 @@ function confidenceEnvelope(fixture: ReturnType): PolicyReplayE }; } +function preAggregationEnvelope( + fixture: ReturnType, + finding: Finding, +): PolicyReplayEnvelopeInput { + const runtime = PolicyTraceRecorder.start({ + runId: "exact-run", + iter: 1, + ablated: new Set(), + }); + const factChecked = validateFindingFacts([finding], fixture.sourceRepoRoot, new Set(), runtime); + const selfScreened = demoteSelfRefuting(factChecked, true, runtime); + const hypothetical = demoteHypotheticalCriticals(selfScreened, true, runtime); + const grounded = groundFindings(hypothetical, "", runtime); + runtime.markInactive("judgment.grounding-llm", "configured-off"); + const policyInactive = { + "judgment.critic": "configured-off" as const, + "scope.diff": "configured-off" as const, + "scope.delta": "stage-precondition-miss" as const, + "scope.session": "stage-precondition-miss" as const, + }; + const result = aggregate({ + findings: grounded, + reviewersTotal: 1, + changedRanges: new Map(), + scopeToDiff: false, + outOfDiffBlocking: [], + confidenceFloor: 0, + demoteCorrectness: true, + corroborateCritical: true, + demoteTestSecurity: true, + capDocsSeverity: true, + critic: new Map(), + fpActive: new Map(), + fpActiveClusters: new Map(), + repUnreliable: new Set(), + protectedReviewers: new Set(), + foreignFiles: new Set(), + cycleRejected: new Set(), + claimedFixed: new Map(), + deltaScope: new Set(), + rejectedRegions: [], + policyRuntime: runtime, + policyInactive, + }); + const policyTrace = runtime.finalize({ + rawResponseSha256: [...fixture.envelope.raw_response_sha256], + verdict: result.verdict, + finalFindings: result.dedupedFindings, + }); + if (policyTrace === null) throw new Error("pre-aggregation trace fixture failed"); + return { + ...fixture.envelope, + pre_policy_findings: [finding], + grounding: { corpus: "", verdicts: [], llm_status: "not-run" }, + history: { + ...fixture.envelope.history, + fp_ledger: { enabled: false }, + reputation: { enabled: false }, + }, + aggregate: { + findings: grounded, + reviewers_total: 1, + changed_ranges: [], + scope_to_diff: false, + out_of_diff_blocking: [], + confidence_floor: 0, + demote_correctness: true, + corroborate_critical: true, + demote_test_security: true, + cap_docs_severity: true, + critic: [], + fp_active: [], + fp_active_clusters: [], + rep_unreliable: [], + protected_reviewers: [], + foreign_files: [], + cycle_rejected: [], + claimed_fixed: [], + delta_scope: [], + rejected_regions: [], + policy_inactive: Object.entries(policyInactive) + .map(([pass_id, reason_code]) => ({ + pass_id: pass_id as keyof typeof policyInactive, + reason_code, + })) + .sort((left, right) => left.pass_id.localeCompare(right.pass_id)), + }, + policy_final_findings: result.dedupedFindings, + policy_trace: policyTrace, + }; +} + +function preAggregationFinding(input: { + signature: string; + severity: "CRITICAL" | "WARN"; + line: number; + message: string; + details: string; +}): Finding { + return { + id: input.signature, + signature: input.signature, + severity: input.severity, + category: "quality", + rule_id: `${input.signature}-rule`, + file: "src/x.ts", + line_start: input.line, + line_end: input.line, + message: input.message, + details: input.details, + reviewer: { provider: "codex", model: "gpt-5", persona: "correctness" }, + confidence: 0.9, + consensus: "singleton", + }; +} + describe("rig replay — determinism self-check", () => { test("a run whose metrics re-derive identically is DETERMINISTIC", () => { const { manifestPath, scriptPath } = miniRun(); @@ -1028,6 +1145,99 @@ describe("rig replay — exact policy authority", () => { expect(pair.counterfactual.final.counts.warn).toBe(1); }); + test("replays fact-location ablation from immutable raw findings into aggregate", async () => { + const fixture = exactRun(); + const envelope = PolicyReplayEnvelopeSchema.parse( + preAggregationEnvelope( + fixture, + preAggregationFinding({ + signature: "fact-location-counterfactual", + severity: "WARN", + line: 99, + message: "Finding cites a source location outside the file", + details: "The reported location is not present in the one-line fixture.", + }), + ), + ); + + const pair = await replayPolicyEnvelopePair({ + sourceRepoRoot: fixture.sourceRepoRoot, + envelope, + stateSnapshotRoot: fixture.stateRoot, + passId: "evidence.fact-location", + }); + + expect(pair.baseline.final.counts.info).toBe(1); + expect(pair.counterfactual.final.counts.warn).toBe(1); + expect(pair.counterfactual.ablated).toEqual(["evidence.fact-location"]); + expect(pair.baseline.raw_response_sha256).toEqual(pair.counterfactual.raw_response_sha256); + }); + + test("replays self-refutation ablation before aggregate without trusting captured output", async () => { + const fixture = exactRun(); + const envelope = PolicyReplayEnvelopeSchema.parse( + preAggregationEnvelope( + fixture, + preAggregationFinding({ + signature: "self-refutation-counterfactual", + severity: "WARN", + line: 1, + message: "Potential maintainability concern", + details: "The implementation was inspected. In conclusion, no issue.", + }), + ), + ); + + const pair = await replayPolicyEnvelopePair({ + sourceRepoRoot: fixture.sourceRepoRoot, + envelope, + stateSnapshotRoot: fixture.stateRoot, + passId: "evidence.self-refutation", + }); + + expect(pair.baseline.final.counts.info).toBe(1); + expect(pair.counterfactual.final.counts.warn).toBe(1); + expect(pair.counterfactual.ablated).toEqual(["evidence.self-refutation"]); + }); + + test("keeps baseline aggregate and response-hash authority for pre-aggregation replay", async () => { + const fixture = exactRun(); + const original = PolicyReplayEnvelopeSchema.parse( + preAggregationEnvelope( + fixture, + preAggregationFinding({ + signature: "pre-aggregate-authority", + severity: "WARN", + line: 99, + message: "Finding cites a source location outside the file", + details: "The captured raw finding must remain immutable.", + }), + ), + ); + const mismatchedAggregate = { + ...original, + aggregate: { + ...original.aggregate, + findings: original.pre_policy_findings, + }, + }; + const mismatchedResponseHash = { + ...original, + raw_response_sha256: ["f".repeat(64)], + }; + + for (const envelope of [mismatchedAggregate, mismatchedResponseHash]) { + await expect( + replayPolicyEnvelopePair({ + sourceRepoRoot: fixture.sourceRepoRoot, + envelope, + stateSnapshotRoot: fixture.stateRoot, + passId: "evidence.fact-location", + }), + ).rejects.toThrow(/captured aggregate findings|baseline replay/); + } + }); + test("persists real branch-local policy outcomes across a multi-envelope sequence", async () => { const fixture = exactRun(); const sourceStateBefore = digestPolicyState(join(fixture.sourceRepoRoot, ".reviewgate")); From 68320ef119bb162a73139ef7bb607a010ab4b41f Mon Sep 17 00:00:00 2001 From: Codevena Date: Tue, 11 Aug 2026 19:42:42 +0200 Subject: [PATCH 50/55] test(policy): prove all pass contracts and mutations --- ...26-08-10-policy-trace-mutation-evidence.md | 38 + tests/fixtures/policy-pass-contracts.ts | 1023 +++++++++++++++++ .../policy-trace-offline-replay.test.ts | 516 +++++++++ .../unit/policy-pass-contract-matrix.test.ts | 90 ++ 4 files changed, 1667 insertions(+) create mode 100644 docs/dev/2026-08-10-policy-trace-mutation-evidence.md create mode 100644 tests/fixtures/policy-pass-contracts.ts create mode 100644 tests/integration/policy-trace-offline-replay.test.ts create mode 100644 tests/unit/policy-pass-contract-matrix.test.ts diff --git a/docs/dev/2026-08-10-policy-trace-mutation-evidence.md b/docs/dev/2026-08-10-policy-trace-mutation-evidence.md new file mode 100644 index 0000000..eb91db7 --- /dev/null +++ b/docs/dev/2026-08-10-policy-trace-mutation-evidence.md @@ -0,0 +1,38 @@ +# Policy trace contract mutation evidence + +Date: 2026-08-11 +Source commit: `ade84226130fe5f21184c5aa443d6a395d0f9252` + +The mutations below were performed one at a time in a disposable no-hardlink clone. The three +Task-10 test files were copied into that clone, and its baseline was validated before mutation: +`147 pass, 0 fail` across the Task-10 matrix/offline replay plus the relevant aggregation, Bench, +Rig, artifact-store, and persistence regressions. The combined SHA-256 ledger for the seven +production files touched by the mutations was +`b124c4395f08755b417843ef3cb5be5bd25523f056e324a7429e81cc1e089669`. + +## Killed mutations + +| # | Deliberate mutation | Named failing command | Exact red evidence | Restored production SHA-256 | +|---:|---|---|---|---| +| 1 | Removed `evidence.fact-location` from the closed catalog inventory. | `bun test tests/unit/policy-pass-contract-matrix.test.ts -t "contains one literal contract"` | `0 pass, 1 fail`; literal fixture inventory had the missing catalog row. | `catalog.ts`: `8343fe3bbf1aae538c25ba70e2a3aea65968f02d9766af422acbef9b83cade6c` | +| 2 | Returned the production mutation without appending its `PolicyEffect`. | `bun test tests/unit/policy-pass-contract-matrix.test.ts -t "evidence.self-refutation"` | `1 pass, 1 fail`; active effect length was `0`, expected `1`. | `trace.ts`: `9f5c972a8f5eebe082c352a13c77b9b1c45cb3cd5bbb4772d5256bcd3e09c2a5` | +| 3 | Omitted every non-no-opportunity `opportunities` increment. | `bun test tests/unit/policy-pass-contract-matrix.test.ts -t "evidence.self-refutation"` | `0 pass, 2 fail`; no-match opportunity was `0`, expected `1`, and telemetry could not retain the material effect. | `trace.ts`: `9f5c972a8f5eebe082c352a13c77b9b1c45cb3cd5bbb4772d5256bcd3e09c2a5` | +| 4 | Reversed policy-effect order in `mergePolicyEffects`. | `bun test tests/unit/policy-pass-contract-matrix.test.ts -t "accepts both explanatory stages"` | `0 pass, 1 fail`; the explanatory-stage trace could not finalize with reversed material effects. | `trace.ts`: `9f5c972a8f5eebe082c352a13c77b9b1c45cb3cd5bbb4772d5256bcd3e09c2a5` | +| 5 | Returned the proposed severity mutation even when the pass was ablated. | `bun test tests/unit/policy-pass-contract-matrix.test.ts -t "evidence.fact-location"` | `1 pass, 1 fail`; ablated blocking count was `0`, expected `1`. | `trace.ts`: `9f5c972a8f5eebe082c352a13c77b9b1c45cb3cd5bbb4772d5256bcd3e09c2a5` | +| 6 | Dropped a merged cluster member's effects. | `bun test tests/unit/policy-aggregator-first-half.test.ts -t "propagates a demoted member effect"` | `0 pass, 1 fail`; final `policy_effects` was absent instead of carrying the member's order-60 demotion. | `aggregator.ts`: `b960229f9eae79fbbace76643eace513c873aab7daba2f40f7914b437c95b4d4` | +| 7 | Skipped the authoritative envelope-to-cassette response-call comparison, including ordered raw-response identity. | `bun test tests/unit/rig-replay.test.ts -t "rejects the authoritative invalidity matrix"` | `0 pass, 1 fail`; the `response hash` corruption unexpectedly passed instead of raising `RigAuthorityError`. | `policy-replay-state.ts`: `f8c59f6a7a8f36ef7037fcbabab866b054795d7af36074d9a3bc3451d3cc0239` | +| 8 | Accepted a caller-supplied artifact hash that did not match the stored bytes. | `bun test tests/unit/policy-trace-store.test.ts -t "rejects missing, absolute, traversing, wrong-hash, tampered, and symlink-escaping refs"` | `0 pass, 1 fail`; verification returned `true` for the tampered hash, expected `false`. | `policy-trace-store.ts`: `66d1c28ac972879152d144d1e8d02cebb311871e02167653d89398c7c00194f6` | +| 9 | Coerced a missing trace counter to zero before validation. | `bun test tests/unit/bench-matrix.test.ts -t "rejects every non-authoritative trace-pair boundary"` | `0 pass, 1 fail`; the `missing counters` case returned `ok: true`, expected `false`. | `runner.ts`: `6df3356e443d263323ab2d70c4921d614dcb149fe3fe73e86387469b037b1fec` | +| 10 | Removed report findings when policy-trace persistence returned `error`. | `bun test tests/unit/report-writer.test.ts -t "keeps the production verdict/findings when trace persistence fails"` | `0 pass, 1 fail`; persisted findings length was `0`, expected `1`. | `orchestrator.ts`: `a8876c064c4d467cab2ee8aa6b3edcad7184ebb1eff61bf9958e84e64cd9ae59` | + +## Restore and cleanup proof + +After every red run, only that mutation's production file was restored from the disposable clone's +immutable `HEAD`; its SHA-256 was checked against the value above and `git diff --exit-code -- ` +returned zero. After mutation 10, `git diff --exit-code -- src` was clean and the combined production +ledger was again exactly +`b124c4395f08755b417843ef3cb5be5bd25523f056e324a7429e81cc1e089669`. + +The final disposable baseline rerun was `147 pass, 0 fail, 1294 expect() calls` across seven files. +The disposable tree was moved intact, rather than deleted, to the recoverable location +`~/.Trash/reviewgate-task10-mutations-rPRQzq`. diff --git a/tests/fixtures/policy-pass-contracts.ts b/tests/fixtures/policy-pass-contracts.ts new file mode 100644 index 0000000..de8f4b5 --- /dev/null +++ b/tests/fixtures/policy-pass-contracts.ts @@ -0,0 +1,1023 @@ +import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { type AggregateInput, aggregate } from "../../src/core/aggregator.ts"; +import { validateFindingFacts } from "../../src/core/fact-check.ts"; +import { + type GroundingVerdict, + applyGroundingJudgeVerdicts, + groundFindings, +} from "../../src/core/grounding.ts"; +import { demoteHypotheticalCriticals } from "../../src/core/hypothetical-demote.ts"; +import { + type PolicyPassId, + type PolicyReasonCode, + type PolicyStageId, +} from "../../src/core/policy/catalog.ts"; +import { PolicyTraceRecorder } from "../../src/core/policy/trace.ts"; +import { demoteSelfRefuting } from "../../src/core/self-refutation.ts"; +import type { Finding } from "../../src/schemas/finding.ts"; +import type { + PolicyEffect, + PolicyEvaluation, + PolicyPassSummary, + PolicyStageEvaluation, +} from "../../src/schemas/policy-trace.ts"; + +export type PolicyNumericTuple = readonly [ + considered: number, + opportunities: number, + wouldApply: number, + applied: number, + protectedCount: number, + blockingRemoved: number, + blockingPreserved: number, + dropped: number, +]; + +export interface PolicyContractScenario { + tuple: PolicyNumericTuple; + blocking: number; + severities: Finding["severity"][]; + effects: PolicyEffect[]; + evaluations: PolicyEvaluation[]; +} + +export interface PolicyPassContractActual { + noOpportunity: PolicyContractScenario; + noMatch: PolicyContractScenario; + active: PolicyContractScenario; + ablated: PolicyContractScenario; + protected?: PolicyContractScenario; + inactive: PolicyPassSummary; + variant?: PolicyContractScenario; +} + +export interface PolicyPassContractExpected { + noOpportunity: PolicyNumericTuple; + noMatch: PolicyNumericTuple; + active: PolicyNumericTuple; + ablated: PolicyNumericTuple; + protected?: PolicyNumericTuple; + inactiveReason: Extract; + activeBlocking: number; + ablatedBlocking: number; + protectedBlocking?: number; + activeSeverities: Finding["severity"][]; + ablatedSeverities: Finding["severity"][]; + protectedSeverities?: Finding["severity"][]; + variant?: { + tuple: PolicyNumericTuple; + blocking: number; + severities: Finding["severity"][]; + }; +} + +export interface PolicyPassContract { + passId: PolicyPassId; + expected: PolicyPassContractExpected; + run(): PolicyPassContractActual; +} + +function finding(overrides: Partial = {}): Finding { + return { + id: "F-001", + signature: "sig-policy", + severity: "WARN", + category: "quality", + rule_id: "policy-contract", + file: "src/a.ts", + line_start: 10, + line_end: 10, + message: "A concrete policy finding", + details: "The implementation has a concrete defect.", + reviewer: { provider: "codex", model: "m", persona: "quality" }, + confidence: 0.9, + consensus: "singleton", + ...overrides, + }; +} + +function runtime(runId: string, ablated: readonly PolicyPassId[] = []): PolicyTraceRecorder { + return PolicyTraceRecorder.start({ runId, iter: 1, ablated }); +} + +function tuple(recorder: PolicyTraceRecorder, passId: PolicyPassId): PolicyNumericTuple { + const summary = recorder.summary(passId); + if (summary.status !== "ran") throw new Error(`${passId} did not run`); + return [ + summary.considered, + summary.opportunities, + summary.would_apply, + summary.applied, + summary.protected, + summary.blocking_removed, + summary.blocking_preserved, + summary.dropped, + ]; +} + +function scenario( + recorder: PolicyTraceRecorder, + passId: PolicyPassId, + findings: readonly Finding[], +): PolicyContractScenario { + return { + tuple: tuple(recorder, passId), + blocking: findings.filter(({ severity }) => severity === "CRITICAL" || severity === "WARN") + .length, + severities: findings.map(({ severity }) => severity), + effects: findings.flatMap(({ policy_effects }) => policy_effects ?? []), + evaluations: recorder.evaluations().filter(({ pass_id }) => pass_id === passId), + }; +} + +function inactive( + passId: PolicyPassId, + reasonCode: Extract, +): PolicyPassSummary { + const recorder = runtime(`${passId}-inactive`); + recorder.markInactive(passId, reasonCode); + return recorder.summary(passId); +} + +function runPrePass( + passId: PolicyPassId, + runId: string, + invoke: (recorder: PolicyTraceRecorder) => Finding[], + ablated: readonly PolicyPassId[] = [], +): PolicyContractScenario { + const recorder = runtime(runId, ablated); + return scenario(recorder, passId, invoke(recorder)); +} + +function runAggregatePass( + passId: PolicyPassId, + runId: string, + input: AggregateInput, + ablated: readonly PolicyPassId[] = [], +): PolicyContractScenario { + const recorder = runtime(runId, ablated); + const result = aggregate({ ...input, policyRuntime: recorder }); + return scenario(recorder, passId, result.dedupedFindings); +} + +function aggregateContract( + passId: PolicyPassId, + expected: PolicyPassContractExpected, + inputs: { + noOpportunity: AggregateInput; + noMatch: AggregateInput; + active: AggregateInput; + protected?: AggregateInput; + variant?: AggregateInput; + }, +): PolicyPassContract { + return { + passId, + expected, + run: () => ({ + noOpportunity: runAggregatePass(passId, `${passId}-no-opportunity`, inputs.noOpportunity), + noMatch: runAggregatePass(passId, `${passId}-no-match`, inputs.noMatch), + active: runAggregatePass(passId, `${passId}-active`, inputs.active), + ablated: runAggregatePass(passId, `${passId}-ablated`, inputs.active, [passId]), + ...(inputs.protected === undefined + ? {} + : { + protected: runAggregatePass( + passId, + `${passId}-protected`, + inputs.protected, + ), + }), + inactive: inactive(passId, expected.inactiveReason), + ...(inputs.variant === undefined + ? {} + : { variant: runAggregatePass(passId, `${passId}-variant`, inputs.variant) }), + }), + }; +} + +function factLocationContract(): PolicyPassContract { + const passId = "evidence.fact-location" as const; + const expected: PolicyPassContractExpected = { + noOpportunity: [1, 0, 0, 0, 0, 0, 0, 0], + noMatch: [1, 1, 0, 0, 0, 0, 0, 0], + active: [1, 1, 1, 1, 0, 1, 0, 0], + ablated: [1, 1, 1, 0, 0, 0, 1, 0], + inactiveReason: "stage-precondition-miss", + activeBlocking: 0, + ablatedBlocking: 1, + activeSeverities: ["INFO"], + ablatedSeverities: ["WARN"], + variant: { tuple: [1, 1, 1, 1, 0, 0, 1, 0], blocking: 1, severities: ["CRITICAL"] }, + }; + return { + passId, + expected, + run: () => { + const repoRoot = mkdtempSync(join(tmpdir(), "reviewgate-policy-contract-")); + writeFileSync(join(repoRoot, "one-line.ts"), "const present = true;\n"); + try { + const activeFinding = finding({ file: "one-line.ts", line_start: 9, line_end: 9 }); + const reanchorFinding = finding({ + severity: "CRITICAL", + file: "one-line.ts", + line_start: 99, + line_end: 99, + evidence_line: "const present = true;", + }); + return { + noOpportunity: runPrePass(passId, "fact-no-opportunity", (recorder) => + validateFindingFacts( + [finding({ file: "absent.ts", line_start: 9, line_end: 9 })], + repoRoot, + new Set(), + recorder, + ), + ), + noMatch: runPrePass(passId, "fact-no-match", (recorder) => + validateFindingFacts( + [finding({ file: "one-line.ts", line_start: 1, line_end: 1 })], + repoRoot, + new Set(), + recorder, + ), + ), + active: runPrePass(passId, "fact-active", (recorder) => + validateFindingFacts([activeFinding], repoRoot, new Set(), recorder), + ), + ablated: runPrePass( + passId, + "fact-ablated", + (recorder) => validateFindingFacts([activeFinding], repoRoot, new Set(), recorder), + [passId], + ), + inactive: inactive(passId, expected.inactiveReason), + variant: runPrePass(passId, "fact-reanchor", (recorder) => + validateFindingFacts([reanchorFinding], repoRoot, new Set(), recorder), + ), + }; + } finally { + rmSync(repoRoot, { recursive: true, force: true }); + } + }, + }; +} + +function preAggregationContract( + passId: PolicyPassId, + expected: PolicyPassContractExpected, + inputs: { + noOpportunity: (recorder: PolicyTraceRecorder) => Finding[]; + noMatch: (recorder: PolicyTraceRecorder) => Finding[]; + active: (recorder: PolicyTraceRecorder) => Finding[]; + protected?: (recorder: PolicyTraceRecorder) => Finding[]; + }, +): PolicyPassContract { + return { + passId, + expected, + run: () => ({ + noOpportunity: runPrePass(passId, `${passId}-no-opportunity`, inputs.noOpportunity), + noMatch: runPrePass(passId, `${passId}-no-match`, inputs.noMatch), + active: runPrePass(passId, `${passId}-active`, inputs.active), + ablated: runPrePass(passId, `${passId}-ablated`, inputs.active, [passId]), + ...(inputs.protected === undefined + ? {} + : { protected: runPrePass(passId, `${passId}-protected`, inputs.protected) }), + inactive: inactive(passId, expected.inactiveReason), + }), + }; +} + +function sameFindingRunner( + value: Finding, + operation: (values: Finding[], recorder: PolicyTraceRecorder) => Finding[], +): (recorder: PolicyTraceRecorder) => Finding[] { + return (recorder) => operation([value], recorder); +} + +const selfActive = finding({ details: "Checked carefully. No issue." }); +const hypotheticalActive = finding({ + severity: "CRITICAL", + details: "This is currently safe, but a future change could break it.", +}); +const tokenActive = finding({ + severity: "CRITICAL", + details: "The --absent-token breaks the theme.", +}); +const llmActive = finding({ severity: "CRITICAL" }); +const llmUngrounded = new Map([ + [llmActive.signature, { grounded: false, reason: "not present" }], +]); + +const redactionActive = finding({ message: "undefined variable " }); +const criticActive = finding({ signature: "sig-critic" }); +const diffActive = finding({ line_start: 50, line_end: 50 }); +const ranges = new Map([["src/a.ts", [[10, 14]] as Array<[number, number]>]]); +const fpInput = { + findings: [finding()], + reviewersTotal: 1, + fpActive: new Map([["sig-policy", { id: "FP-001" }]]), +}; +const cycleInput = { + findings: [finding()], + reviewersTotal: 1, + cycleRejected: new Set(["sig-policy"]), +}; +const activeCluster = new Map([ + ["policy@src/a.ts", { key: "policy@src/a.ts", member_ids: ["FP-001"] }], +]); +const clusterInput = { + findings: [finding()], + reviewersTotal: 1, + fpActiveClusters: activeCluster, +}; +const lowConfidence = finding({ confidence: 0.2 }); +const reputationInput = { + findings: [finding()], + reviewersTotal: 1, + repUnreliable: new Set(["codex:quality"]), +}; +const rejectedRegion = { + file: "src/a.ts", + start_line: 8, + end_line: 12, + severity: "WARN" as const, + categories: ["quality" as const], + reason: "this exact region was already disproven twice", + distinct_count: 2, +}; +const regionInput = { + findings: [finding()], + reviewersTotal: 1, + rejectedRegions: [rejectedRegion], +}; +const testSecurity = finding({ category: "security", file: "src/a.test.ts" }); +const docsCritical = finding({ severity: "CRITICAL", file: "README.md" }); + +export const POLICY_PASS_CONTRACTS: readonly PolicyPassContract[] = [ + factLocationContract(), + preAggregationContract( + "evidence.self-refutation", + { + noOpportunity: [1, 0, 0, 0, 0, 0, 0, 0], + noMatch: [1, 1, 0, 0, 0, 0, 0, 0], + active: [1, 1, 1, 1, 0, 1, 0, 0], + ablated: [1, 1, 1, 0, 0, 0, 1, 0], + protected: [1, 1, 1, 0, 1, 0, 1, 0], + inactiveReason: "configured-off", + activeBlocking: 0, + ablatedBlocking: 1, + protectedBlocking: 1, + activeSeverities: ["INFO"], + ablatedSeverities: ["WARN"], + protectedSeverities: ["WARN"], + }, + { + noOpportunity: sameFindingRunner( + finding({ severity: "INFO", details: "No issue." }), + (values, recorder) => demoteSelfRefuting(values, true, recorder), + ), + noMatch: sameFindingRunner(finding(), (values, recorder) => + demoteSelfRefuting(values, true, recorder), + ), + active: sameFindingRunner(selfActive, (values, recorder) => + demoteSelfRefuting(values, true, recorder), + ), + protected: sameFindingRunner( + { ...selfActive, category: "correctness" }, + (values, recorder) => demoteSelfRefuting(values, true, recorder), + ), + }, + ), + preAggregationContract( + "judgment.hypothetical", + { + noOpportunity: [1, 0, 0, 0, 0, 0, 0, 0], + noMatch: [1, 1, 0, 0, 0, 0, 0, 0], + active: [1, 1, 1, 1, 0, 0, 1, 0], + ablated: [1, 1, 1, 0, 0, 0, 1, 0], + protected: [1, 1, 1, 0, 1, 0, 1, 0], + inactiveReason: "configured-off", + activeBlocking: 1, + ablatedBlocking: 1, + protectedBlocking: 1, + activeSeverities: ["WARN"], + ablatedSeverities: ["CRITICAL"], + protectedSeverities: ["CRITICAL"], + }, + { + noOpportunity: sameFindingRunner( + finding({ severity: "WARN", details: "Currently safe; future change." }), + (values, recorder) => demoteHypotheticalCriticals(values, true, recorder), + ), + noMatch: sameFindingRunner( + finding({ + severity: "CRITICAL", + details: "Currently safe in theory, but this already fails right now.", + }), + (values, recorder) => demoteHypotheticalCriticals(values, true, recorder), + ), + active: sameFindingRunner(hypotheticalActive, (values, recorder) => + demoteHypotheticalCriticals(values, true, recorder), + ), + protected: sameFindingRunner( + { ...hypotheticalActive, category: "security" }, + (values, recorder) => demoteHypotheticalCriticals(values, true, recorder), + ), + }, + ), + preAggregationContract( + "evidence.grounding-token", + { + noOpportunity: [1, 0, 0, 0, 0, 0, 0, 0], + noMatch: [1, 1, 0, 0, 0, 0, 0, 0], + active: [1, 1, 1, 1, 0, 0, 1, 0], + ablated: [1, 1, 1, 0, 0, 0, 1, 0], + protected: [1, 1, 1, 0, 1, 0, 1, 0], + inactiveReason: "stage-precondition-miss", + activeBlocking: 1, + ablatedBlocking: 1, + protectedBlocking: 1, + activeSeverities: ["WARN"], + ablatedSeverities: ["CRITICAL"], + protectedSeverities: ["CRITICAL"], + }, + { + noOpportunity: sameFindingRunner( + finding({ severity: "WARN", details: "Missing --absent-token." }), + (values, recorder) => groundFindings(values, ":root { --present-token: #fff; }", recorder), + ), + noMatch: sameFindingRunner( + finding({ severity: "CRITICAL", details: "The --present-token is wrong." }), + (values, recorder) => groundFindings(values, ":root { --present-token: #fff; }", recorder), + ), + active: sameFindingRunner(tokenActive, (values, recorder) => + groundFindings(values, "const present = true;", recorder), + ), + protected: sameFindingRunner( + { ...tokenActive, category: "security" }, + (values, recorder) => groundFindings(values, "const present = true;", recorder), + ), + }, + ), + preAggregationContract( + "judgment.grounding-llm", + { + noOpportunity: [1, 0, 0, 0, 0, 0, 0, 0], + noMatch: [1, 1, 0, 0, 0, 0, 0, 0], + active: [1, 1, 1, 1, 0, 0, 1, 0], + ablated: [1, 1, 1, 0, 0, 0, 1, 0], + protected: [1, 1, 1, 0, 1, 0, 1, 0], + inactiveReason: "configured-off", + activeBlocking: 1, + ablatedBlocking: 1, + protectedBlocking: 1, + activeSeverities: ["WARN"], + ablatedSeverities: ["CRITICAL"], + protectedSeverities: ["CRITICAL"], + }, + { + noOpportunity: sameFindingRunner(llmActive, (values, recorder) => + applyGroundingJudgeVerdicts(values, new Map(), recorder), + ), + noMatch: sameFindingRunner(llmActive, (values, recorder) => + applyGroundingJudgeVerdicts( + values, + new Map([[llmActive.signature, { grounded: true }]]), + recorder, + ), + ), + active: sameFindingRunner(llmActive, (values, recorder) => + applyGroundingJudgeVerdicts(values, llmUngrounded, recorder), + ), + protected: sameFindingRunner( + { ...llmActive, category: "correctness" }, + (values, recorder) => + applyGroundingJudgeVerdicts( + values, + new Map([[llmActive.signature, { grounded: false }]]), + recorder, + ), + ), + }, + ), + aggregateContract( + "evidence.redaction-placeholder", + { + noOpportunity: [1, 0, 0, 0, 0, 0, 0, 0], + noMatch: [1, 1, 0, 0, 0, 0, 0, 0], + active: [1, 1, 1, 1, 0, 1, 0, 0], + ablated: [1, 1, 1, 0, 0, 0, 1, 0], + protected: [1, 1, 1, 0, 1, 0, 1, 0], + inactiveReason: "stage-precondition-miss", + activeBlocking: 0, + ablatedBlocking: 1, + protectedBlocking: 1, + activeSeverities: ["INFO"], + ablatedSeverities: ["WARN"], + protectedSeverities: ["WARN"], + }, + { + noOpportunity: { + findings: [{ ...redactionActive, severity: "INFO" }], + reviewersTotal: 1, + }, + noMatch: { + findings: [finding({ message: "exposed value " })], + reviewersTotal: 1, + }, + active: { findings: [redactionActive], reviewersTotal: 1 }, + protected: { + findings: [{ ...redactionActive, category: "security" }], + reviewersTotal: 1, + }, + }, + ), + aggregateContract( + "judgment.critic", + { + noOpportunity: [1, 0, 0, 0, 0, 0, 0, 0], + noMatch: [1, 1, 0, 0, 0, 0, 0, 0], + active: [1, 1, 1, 1, 0, 1, 0, 0], + ablated: [1, 1, 1, 0, 0, 0, 1, 0], + protected: [1, 1, 1, 0, 1, 0, 1, 0], + inactiveReason: "configured-off", + activeBlocking: 0, + ablatedBlocking: 1, + protectedBlocking: 1, + activeSeverities: ["INFO"], + ablatedSeverities: ["WARN"], + protectedSeverities: ["WARN"], + variant: { tuple: [1, 1, 1, 1, 0, 0, 0, 1], blocking: 0, severities: [] }, + }, + { + noOpportunity: { findings: [criticActive], reviewersTotal: 1, critic: new Map() }, + noMatch: { + findings: [criticActive], + reviewersTotal: 1, + critic: new Map([[criticActive.signature, { verdict: "keep" }]]), + }, + active: { + findings: [criticActive], + reviewersTotal: 1, + critic: new Map([[criticActive.signature, { verdict: "likely_fp" }]]), + }, + protected: { + findings: [ + finding({ signature: "sig-critic-majority-a" }), + finding({ + signature: "sig-critic-majority-b", + reviewer: { provider: "gemini", model: "m", persona: "quality" }, + }), + ], + reviewersTotal: 3, + critic: new Map([["sig-critic-majority-b", { verdict: "likely_fp" }]]), + }, + variant: { + findings: [{ ...criticActive, severity: "INFO" }], + reviewersTotal: 1, + critic: new Map([[criticActive.signature, { verdict: "likely_fp" }]]), + }, + }, + ), + aggregateContract( + "scope.diff", + { + noOpportunity: [1, 0, 0, 0, 0, 0, 0, 0], + noMatch: [1, 1, 0, 0, 0, 0, 0, 0], + active: [1, 1, 1, 1, 0, 1, 0, 0], + ablated: [1, 1, 1, 0, 0, 0, 1, 0], + protected: [1, 1, 1, 0, 1, 0, 1, 0], + inactiveReason: "configured-off", + activeBlocking: 0, + ablatedBlocking: 1, + protectedBlocking: 1, + activeSeverities: ["INFO"], + ablatedSeverities: ["WARN"], + protectedSeverities: ["WARN"], + }, + { + noOpportunity: { + findings: [{ ...diffActive, line_start: 0, line_end: 0 }], + reviewersTotal: 1, + changedRanges: ranges, + scopeToDiff: true, + }, + noMatch: { + findings: [{ ...diffActive, line_start: 11, line_end: 11 }], + reviewersTotal: 1, + changedRanges: ranges, + scopeToDiff: true, + }, + active: { + findings: [diffActive], + reviewersTotal: 1, + changedRanges: ranges, + scopeToDiff: true, + }, + protected: { + findings: [{ ...diffActive, category: "security" }], + reviewersTotal: 1, + changedRanges: ranges, + scopeToDiff: true, + outOfDiffBlocking: ["security"], + }, + }, + ), + aggregateContract( + "scope.delta", + { + noOpportunity: [1, 0, 0, 0, 0, 0, 0, 0], + noMatch: [1, 1, 0, 0, 0, 0, 0, 0], + active: [1, 1, 1, 1, 0, 1, 0, 0], + ablated: [1, 1, 1, 0, 0, 0, 1, 0], + protected: [1, 1, 1, 0, 1, 0, 1, 0], + inactiveReason: "stage-precondition-miss", + activeBlocking: 0, + ablatedBlocking: 1, + protectedBlocking: 1, + activeSeverities: ["INFO"], + ablatedSeverities: ["WARN"], + protectedSeverities: ["WARN"], + }, + { + noOpportunity: { + findings: [finding({ severity: "INFO" })], + reviewersTotal: 1, + deltaScope: new Set(["src/a.ts"]), + }, + noMatch: { findings: [finding()], reviewersTotal: 1, deltaScope: new Set(["src/a.ts"]) }, + active: { + findings: [finding()], + reviewersTotal: 1, + deltaScope: new Set(["src/other.ts"]), + }, + protected: { + findings: [finding({ category: "correctness" })], + reviewersTotal: 1, + deltaScope: new Set(["src/other.ts"]), + }, + }, + ), + aggregateContract( + "scope.session", + { + noOpportunity: [1, 0, 0, 0, 0, 0, 0, 0], + noMatch: [1, 1, 0, 0, 0, 0, 0, 0], + active: [1, 1, 1, 1, 0, 1, 0, 0], + ablated: [1, 1, 1, 0, 0, 0, 1, 0], + protected: [1, 1, 1, 0, 1, 0, 1, 0], + inactiveReason: "stage-precondition-miss", + activeBlocking: 0, + ablatedBlocking: 1, + protectedBlocking: 1, + activeSeverities: ["INFO"], + ablatedSeverities: ["WARN"], + protectedSeverities: ["WARN"], + }, + { + noOpportunity: { + findings: [finding({ severity: "INFO" })], + reviewersTotal: 1, + foreignFiles: new Set(["src/a.ts"]), + }, + noMatch: { + findings: [finding()], + reviewersTotal: 1, + foreignFiles: new Set(["src/foreign.ts"]), + }, + active: { + findings: [finding()], + reviewersTotal: 1, + foreignFiles: new Set(["src/a.ts"]), + }, + protected: { + findings: [finding({ category: "security" })], + reviewersTotal: 1, + foreignFiles: new Set(["src/a.ts"]), + outOfDiffBlocking: ["security"], + }, + }, + ), + aggregateContract( + "history.fp-signature", + { + noOpportunity: [1, 0, 0, 0, 0, 0, 0, 0], + noMatch: [1, 1, 0, 0, 0, 0, 0, 0], + active: [1, 1, 1, 1, 0, 1, 0, 0], + ablated: [1, 1, 1, 0, 0, 0, 1, 0], + inactiveReason: "stage-precondition-miss", + activeBlocking: 0, + ablatedBlocking: 1, + activeSeverities: ["INFO"], + ablatedSeverities: ["WARN"], + }, + { + noOpportunity: { + findings: [finding({ severity: "INFO" })], + reviewersTotal: 1, + fpActive: fpInput.fpActive, + }, + noMatch: { + findings: [finding()], + reviewersTotal: 1, + fpActive: new Map([["other", { id: "FP-001" }]]), + }, + active: fpInput, + }, + ), + aggregateContract( + "history.cycle-rejected", + { + noOpportunity: [1, 0, 0, 0, 0, 0, 0, 0], + noMatch: [1, 1, 0, 0, 0, 0, 0, 0], + active: [1, 1, 1, 1, 0, 1, 0, 0], + ablated: [1, 1, 1, 0, 0, 0, 1, 0], + protected: [1, 1, 1, 0, 1, 0, 1, 0], + inactiveReason: "stage-precondition-miss", + activeBlocking: 0, + ablatedBlocking: 1, + protectedBlocking: 1, + activeSeverities: ["INFO"], + ablatedSeverities: ["WARN"], + protectedSeverities: ["WARN"], + }, + { + noOpportunity: { + findings: [finding({ severity: "INFO" })], + reviewersTotal: 1, + cycleRejected: cycleInput.cycleRejected, + }, + noMatch: { + findings: [finding()], + reviewersTotal: 1, + cycleRejected: new Set(["other"]), + }, + active: cycleInput, + protected: { + findings: [finding({ category: "correctness" })], + reviewersTotal: 1, + cycleRejected: cycleInput.cycleRejected, + }, + }, + ), + aggregateContract( + "history.fp-cluster", + { + noOpportunity: [1, 0, 0, 0, 0, 0, 0, 0], + noMatch: [1, 1, 0, 0, 0, 0, 0, 0], + active: [1, 1, 1, 1, 0, 1, 0, 0], + ablated: [1, 1, 1, 0, 0, 0, 1, 0], + inactiveReason: "stage-precondition-miss", + activeBlocking: 0, + ablatedBlocking: 1, + activeSeverities: ["INFO"], + ablatedSeverities: ["WARN"], + }, + { + noOpportunity: { + findings: [finding({ severity: "INFO" })], + reviewersTotal: 1, + fpActiveClusters: activeCluster, + }, + noMatch: { + findings: [finding({ rule_id: "other-contract" })], + reviewersTotal: 1, + fpActiveClusters: activeCluster, + }, + active: clusterInput, + }, + ), + aggregateContract( + "judgment.confidence", + { + noOpportunity: [1, 0, 0, 0, 0, 0, 0, 0], + noMatch: [1, 1, 0, 0, 0, 0, 0, 0], + active: [1, 1, 1, 1, 0, 1, 0, 0], + ablated: [1, 1, 1, 0, 0, 0, 1, 0], + protected: [1, 1, 1, 0, 1, 0, 1, 0], + inactiveReason: "configured-off", + activeBlocking: 0, + ablatedBlocking: 1, + protectedBlocking: 1, + activeSeverities: ["INFO"], + ablatedSeverities: ["WARN"], + protectedSeverities: ["WARN"], + }, + { + noOpportunity: { + findings: [ + lowConfidence, + { + ...lowConfidence, + signature: "sig-majority", + reviewer: { provider: "gemini", model: "m", persona: "quality" }, + }, + ], + reviewersTotal: 2, + confidenceFloor: 0.5, + }, + noMatch: { + findings: [finding({ confidence: 0.5 })], + reviewersTotal: 1, + confidenceFloor: 0.5, + }, + active: { findings: [lowConfidence], reviewersTotal: 1, confidenceFloor: 0.5 }, + protected: { + findings: [lowConfidence], + reviewersTotal: 1, + confidenceFloor: 0.5, + protectedReviewers: new Set(["codex"]), + }, + }, + ), + aggregateContract( + "judgment.reputation", + { + noOpportunity: [1, 0, 0, 0, 0, 0, 0, 0], + noMatch: [1, 1, 0, 0, 0, 0, 0, 0], + active: [1, 1, 1, 1, 0, 1, 0, 0], + ablated: [1, 1, 1, 0, 0, 0, 1, 0], + protected: [1, 1, 1, 0, 1, 0, 1, 0], + inactiveReason: "stage-precondition-miss", + activeBlocking: 0, + ablatedBlocking: 1, + protectedBlocking: 1, + activeSeverities: ["INFO"], + ablatedSeverities: ["WARN"], + protectedSeverities: ["WARN"], + }, + { + noOpportunity: { + findings: [ + finding(), + finding({ + signature: "sig-reputation-majority", + reviewer: { provider: "gemini", model: "m", persona: "quality" }, + }), + ], + reviewersTotal: 2, + repUnreliable: new Set(["codex:quality", "gemini:quality"]), + }, + noMatch: { + findings: [finding()], + reviewersTotal: 1, + repUnreliable: new Set(["gemini:quality"]), + }, + active: reputationInput, + protected: { + findings: [finding({ category: "security" })], + reviewersTotal: 1, + repUnreliable: new Set(["codex:quality"]), + }, + }, + ), + aggregateContract( + "history.region-rejected", + { + noOpportunity: [1, 0, 0, 0, 0, 0, 0, 0], + noMatch: [1, 1, 0, 0, 0, 0, 0, 0], + active: [1, 1, 1, 1, 0, 1, 0, 0], + ablated: [1, 1, 1, 0, 0, 0, 1, 0], + protected: [1, 1, 1, 0, 1, 0, 1, 0], + inactiveReason: "stage-precondition-miss", + activeBlocking: 0, + ablatedBlocking: 1, + protectedBlocking: 1, + activeSeverities: ["INFO"], + ablatedSeverities: ["WARN"], + protectedSeverities: ["WARN"], + }, + { + noOpportunity: { + findings: [finding({ line_start: 0, line_end: 0 })], + reviewersTotal: 1, + rejectedRegions: [rejectedRegion], + }, + noMatch: { + findings: [finding({ line_start: 40, line_end: 40 })], + reviewersTotal: 1, + rejectedRegions: [rejectedRegion], + }, + active: regionInput, + protected: { + findings: [finding()], + reviewersTotal: 1, + rejectedRegions: [{ ...rejectedRegion, distinct_count: 1 }], + }, + }, + ), + aggregateContract( + "judgment.test-security", + { + noOpportunity: [1, 0, 0, 0, 0, 0, 0, 0], + noMatch: [1, 1, 0, 0, 0, 0, 0, 0], + active: [1, 1, 1, 1, 0, 1, 0, 0], + ablated: [1, 1, 1, 0, 0, 0, 1, 0], + protected: [1, 1, 1, 0, 1, 0, 1, 0], + inactiveReason: "configured-off", + activeBlocking: 0, + ablatedBlocking: 1, + protectedBlocking: 1, + activeSeverities: ["INFO"], + ablatedSeverities: ["WARN"], + protectedSeverities: ["WARN"], + }, + { + noOpportunity: { + findings: [{ ...testSecurity, severity: "INFO" }], + reviewersTotal: 1, + demoteTestSecurity: true, + }, + noMatch: { + findings: [finding({ file: "src/a.test.ts" })], + reviewersTotal: 1, + demoteTestSecurity: true, + }, + active: { findings: [testSecurity], reviewersTotal: 1, demoteTestSecurity: true }, + protected: { + findings: [ + { ...testSecurity, signature: "sig-test-security", message: "same test issue" }, + finding({ + signature: "sig-test-correctness", + category: "correctness", + file: "src/a.test.ts", + message: "same test issue", + }), + ], + reviewersTotal: 1, + demoteTestSecurity: true, + }, + }, + ), + aggregateContract( + "judgment.docs-cap", + { + noOpportunity: [1, 0, 0, 0, 0, 0, 0, 0], + noMatch: [1, 1, 0, 0, 0, 0, 0, 0], + active: [1, 1, 1, 1, 0, 0, 1, 0], + ablated: [1, 1, 1, 0, 0, 0, 1, 0], + protected: [1, 1, 1, 0, 1, 0, 1, 0], + inactiveReason: "configured-off", + activeBlocking: 1, + ablatedBlocking: 1, + protectedBlocking: 1, + activeSeverities: ["WARN"], + ablatedSeverities: ["CRITICAL"], + protectedSeverities: ["CRITICAL"], + }, + { + noOpportunity: { + findings: [finding({ file: "README.md" })], + reviewersTotal: 1, + capDocsSeverity: true, + }, + noMatch: { + findings: [finding({ severity: "CRITICAL" })], + reviewersTotal: 2, + capDocsSeverity: true, + }, + active: { findings: [docsCritical], reviewersTotal: 1, capDocsSeverity: true }, + protected: { + findings: [{ ...docsCritical, category: "correctness" }], + reviewersTotal: 1, + capDocsSeverity: true, + }, + }, + ), +]; + +export function runExplanatoryStageContract(): { + stages: PolicyStageEvaluation[]; + effects: PolicyEffect[]; +} { + const protectedFinding = finding({ + category: "security", + message: "undefined variable ", + }); + const recorder = runtime("explanatory-stages"); + const result = aggregate({ + findings: [protectedFinding], + reviewersTotal: 1, + critic: new Map([[protectedFinding.signature, { verdict: "likely_fp" }]]), + policyRuntime: recorder, + }); + const trace = recorder.finalize({ + rawResponseSha256: [], + verdict: result.verdict, + finalFindings: result.dedupedFindings, + }); + if (trace === null) throw new Error("explanatory stage trace did not finalize"); + return { + stages: trace.stages, + effects: result.dedupedFindings.flatMap(({ policy_effects }) => policy_effects ?? []), + }; +} + +export const EXPLANATORY_STAGE_IDS: readonly PolicyStageId[] = [ + "aggregation.cluster", + "verdict.compute", +]; diff --git a/tests/integration/policy-trace-offline-replay.test.ts b/tests/integration/policy-trace-offline-replay.test.ts new file mode 100644 index 0000000..9651754 --- /dev/null +++ b/tests/integration/policy-trace-offline-replay.test.ts @@ -0,0 +1,516 @@ +import { describe, expect, it, spyOn } from "bun:test"; +import { execFileSync } from "node:child_process"; +import { createHash } from "node:crypto"; +import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { canonicalJson } from "../../src/audit/canonical.ts"; +import { type AggregateInput, aggregate } from "../../src/core/aggregator.ts"; +import { validateFindingFacts } from "../../src/core/fact-check.ts"; +import { groundFindings } from "../../src/core/grounding.ts"; +import { demoteHypotheticalCriticals } from "../../src/core/hypothetical-demote.ts"; +import { + POLICY_CATALOG_VERSION, + POLICY_PASS_IDS, + type PolicyPassId, +} from "../../src/core/policy/catalog.ts"; +import { + capturePolicyReplayEnvelope, + serializePolicyReplayAggregateInputs, +} from "../../src/core/policy/replay-capture.ts"; +import { PolicyTraceRecorder } from "../../src/core/policy/trace.ts"; +import { demoteSelfRefuting } from "../../src/core/self-refutation.ts"; +import { + createPolicyStateSnapshot, + digestPolicyState, + validateRigPolicyReplayArtifacts, +} from "../../src/rig/policy-replay-state.ts"; +import { replayPolicyEnvelopePair } from "../../src/rig/replay.ts"; +import type { Finding } from "../../src/schemas/finding.ts"; +import { + type PolicyReplayEnvelope, + type PolicyReplayEnvelopeInput, + PolicyReplayEnvelopeSchema, + policyReplayCallId, +} from "../../src/schemas/policy-replay.ts"; +import type { PolicyTrace } from "../../src/schemas/policy-trace.ts"; +import type { RigManifest } from "../../src/schemas/rig-manifest.ts"; +import { initialState } from "../../src/schemas/state.ts"; + +type ReplayClass = "evidence" | "value-judgment" | "scope" | "history"; + +interface ReplayCase { + className: ReplayClass; + passId: PolicyPassId; + finding: Finding; + aggregateInput(findings: Finding[]): AggregateInput; +} + +function sha256(value: string | Buffer): string { + return createHash("sha256").update(value).digest("hex"); +} + +function finding(overrides: Partial = {}): Finding { + return { + id: "F-001", + signature: "offline-policy", + severity: "WARN", + category: "quality", + rule_id: "offline-policy", + file: "src/x.ts", + line_start: 1, + line_end: 1, + message: "A concrete offline replay finding", + details: "The recorded finding exercises one production policy pass.", + reviewer: { provider: "codex", model: "m", persona: "quality" }, + confidence: 0.9, + consensus: "singleton", + ...overrides, + }; +} + +function baseAggregateInput(findings: Finding[]): AggregateInput { + return { + findings, + reviewersTotal: 1, + changedRanges: new Map(), + scopeToDiff: false, + outOfDiffBlocking: [], + confidenceFloor: 0, + demoteCorrectness: true, + corroborateCritical: true, + demoteTestSecurity: false, + capDocsSeverity: false, + critic: new Map(), + fpActive: new Map(), + fpActiveClusters: new Map(), + repUnreliable: new Set(), + protectedReviewers: new Set(), + foreignFiles: new Set(), + cycleRejected: new Set(), + claimedFixed: new Map(), + deltaScope: new Set(), + rejectedRegions: [], + policyInactive: { + "judgment.critic": "configured-off", + "scope.diff": "configured-off", + "scope.delta": "stage-precondition-miss", + "scope.session": "stage-precondition-miss", + }, + }; +} + +function replayCases(): ReplayCase[] { + return [ + { + className: "evidence", + passId: "evidence.fact-location", + finding: finding({ signature: "offline-fact", line_start: 99, line_end: 99 }), + aggregateInput: baseAggregateInput, + }, + { + className: "value-judgment", + passId: "judgment.confidence", + finding: finding({ signature: "offline-confidence", confidence: 0.2 }), + aggregateInput: (findings) => ({ + ...baseAggregateInput(findings), + confidenceFloor: 0.8, + }), + }, + { + className: "scope", + passId: "scope.diff", + finding: finding({ signature: "offline-scope", line_start: 10, line_end: 10 }), + aggregateInput: (findings) => { + const input = baseAggregateInput(findings); + return { + ...input, + changedRanges: new Map([["src/x.ts", [[1, 2]]]]), + scopeToDiff: true, + policyInactive: { + "judgment.critic": "configured-off", + "scope.delta": "stage-precondition-miss", + "scope.session": "stage-precondition-miss", + }, + }; + }, + }, + { + className: "history", + passId: "history.fp-signature", + finding: finding({ signature: "seeded-fp" }), + aggregateInput: (findings) => ({ + ...baseAggregateInput(findings), + fpActive: new Map([["seeded-fp", { id: "FP-001" }]]), + }), + }, + ]; +} + +function createSourceRepo(): { root: string; commit: string } { + const root = mkdtempSync(join(tmpdir(), "reviewgate-offline-source-")); + execFileSync("git", ["init", "-q", "."], { cwd: root }); + execFileSync("git", ["config", "user.email", "offline@example.invalid"], { cwd: root }); + execFileSync("git", ["config", "user.name", "offline"], { cwd: root }); + mkdirSync(join(root, "src")); + writeFileSync( + join(root, "src", "x.ts"), + `${Array.from( + { length: 20 }, + (_, index) => `export const line${index + 1} = ${index + 1};`, + ).join("\n")}\n`, + ); + execFileSync("git", ["add", "src/x.ts"], { cwd: root }); + execFileSync("git", ["commit", "-qm", "source"], { cwd: root }); + + mkdirSync(join(root, ".reviewgate", "learnings"), { recursive: true }); + writeFileSync(join(root, ".reviewgate", "state.json"), JSON.stringify(initialState("offline")), { + mode: 0o600, + }); + writeFileSync( + join(root, ".reviewgate", "learnings", "known_fp.jsonl"), + JSON.stringify({ + schema: "reviewgate.fpledger.v1", + seq: 1, + entries: [ + { + id: "FP-001", + signature: "seeded-fp", + rule_id: "offline-policy", + category: "quality", + file: "src/x.ts", + symbol: "", + stage: "active", + rejects: [ + { + run_id: "seed-1", + provider: "codex", + ts: "2026-08-01T12:00:00.000Z", + reason: "confirmed false positive one", + }, + { + run_id: "seed-2", + provider: "openrouter", + ts: "2026-08-02T12:00:00.000Z", + reason: "confirmed false positive two", + }, + { + run_id: "seed-3", + provider: "codex", + ts: "2026-08-03T12:00:00.000Z", + reason: "confirmed false positive three", + }, + ], + distinct_providers: ["codex", "openrouter"], + first_seen_at: "2026-08-01T12:00:00.000Z", + last_seen_at: "2026-08-03T12:00:00.000Z", + created_at: "2026-08-01T12:00:00.000Z", + }, + ], + }), + { mode: 0o600 }, + ); + const commit = execFileSync("git", ["rev-parse", "HEAD"], { + cwd: root, + encoding: "utf8", + }).trim(); + return { root, commit }; +} + +function executeProductionBaseline(input: { + sourceRepoRoot: string; + runId: string; + rawResponseSha256: string[]; + replayCase: ReplayCase; +}): { + aggregateInput: AggregateInput; + aggregateFindings: Finding[]; + finalFindings: Finding[]; + trace: PolicyTrace; +} { + const recorder = PolicyTraceRecorder.start({ runId: input.runId, iter: 1, ablated: [] }); + const factChecked = validateFindingFacts( + [input.replayCase.finding], + input.sourceRepoRoot, + new Set(), + recorder, + ); + const selfScreened = demoteSelfRefuting(factChecked, true, recorder); + const hypothetical = demoteHypotheticalCriticals(selfScreened, true, recorder); + const grounded = groundFindings(hypothetical, "", recorder); + recorder.markInactive("judgment.grounding-llm", "configured-off"); + const aggregateInput = input.replayCase.aggregateInput(grounded); + const result = aggregate({ ...aggregateInput, policyRuntime: recorder }); + const trace = recorder.finalize({ + rawResponseSha256: input.rawResponseSha256, + verdict: result.verdict, + finalFindings: result.dedupedFindings, + }); + if (trace === null) throw new Error("offline production trace did not finalize"); + return { + aggregateInput, + aggregateFindings: grounded, + finalFindings: result.dedupedFindings, + trace, + }; +} + +function responseCall(runId: string, rawText: string) { + const identity = { + runId, + iter: 1, + kind: "reviewer" as const, + provider: "openrouter" as const, + method: "review" as const, + key: "openrouter-quality", + promptSha256: "a".repeat(64), + ordinal: 0, + slot: 0, + attempt: 1, + occurrence: 0, + }; + return { + call_id: policyReplayCallId(identity), + kind: identity.kind, + provider: identity.provider, + method: identity.method, + key: identity.key, + prompt_sha256: identity.promptSha256, + ordinal: identity.ordinal, + slot: identity.slot, + attempt: identity.attempt, + occurrence: identity.occurrence, + response_sha256: sha256(rawText), + }; +} + +function cassetteEntry(runId: string, rawText: string, call: ReturnType) { + return { + schema: "reviewgate.cassette.entry.v1", + provider: call.provider, + method: call.method, + key: call.key, + promptSha256: call.prompt_sha256, + policyReplayCall: { + callId: call.call_id, + runId, + iter: 1, + kind: call.kind, + ordinal: call.ordinal, + slot: call.slot, + attempt: call.attempt, + occurrence: call.occurrence, + }, + result: { + reviewerId: call.key, + verdict: "PASS", + findings: [], + usage: { inputTokens: 1, outputTokens: 1, costUsd: 0, quotaUsedPct: null }, + durationMs: 1, + exitCode: 0, + rawEventsPath: "", + status: "ok", + rawText, + }, + }; +} + +function persistAuthoritativeFixture(input: { + sourceRepoRoot: string; + sourceCommit: string; + outputRoot: string; + replayCase: ReplayCase; +}): { + envelope: PolicyReplayEnvelope; + stateSnapshotRoot: string; + callsBefore: string; +} { + const runId = `offline-${input.replayCase.className}`; + const rawText = `safe recorded response for ${input.replayCase.className}`; + const rawHash = sha256(rawText); + const call = responseCall(runId, rawText); + const production = executeProductionBaseline({ + sourceRepoRoot: input.sourceRepoRoot, + runId, + rawResponseSha256: [rawHash], + replayCase: input.replayCase, + }); + const state = createPolicyStateSnapshot({ + sourceRepoRoot: input.sourceRepoRoot, + outputRoot: input.outputRoot, + }); + const envelopeInput: PolicyReplayEnvelopeInput = { + schema: "reviewgate.policy-replay-envelope.v1", + catalog_version: POLICY_CATALOG_VERSION, + run_id: runId, + iter: 1, + source_commit: input.sourceCommit, + exact_diff: "", + pre_policy_findings: [input.replayCase.finding], + grounding: { corpus: "", verdicts: [], llm_status: "not-run" }, + aggregate: serializePolicyReplayAggregateInputs(production.aggregateInput), + policy_final_findings: production.finalFindings, + pre_policy: { self_refutation_enabled: true, hypothetical_enabled: true }, + state_sha256: state.stateSha256, + raw_response_sha256: [rawHash], + response_calls: [call], + history: { + fp_ledger: + input.replayCase.className === "history" + ? { + enabled: true, + active_at: "2026-08-11T12:00:00.000Z", + clusters_at: "2026-08-11T12:00:00.000Z", + } + : { enabled: false }, + reputation: { enabled: false }, + cycle_state: { source: "state.json", region_rejected_enabled: false }, + implicit_outcomes: { + enabled: true, + created_at: "2026-08-11T12:00:00.000Z", + cap: 100, + }, + }, + policy_trace: production.trace, + lossless: true, + }; + const envelope = PolicyReplayEnvelopeSchema.parse(envelopeInput); + const sinkDir = join(input.outputRoot, "policy-replay"); + mkdirSync(sinkDir, { mode: 0o700 }); + const stored = capturePolicyReplayEnvelope({ + sinkDir, + measuredRepoRoot: input.sourceRepoRoot, + envelope, + }); + if (stored.status !== "complete") throw new Error("offline replay envelope did not persist"); + const cassettePath = join(input.outputRoot, "cassette.jsonl"); + writeFileSync(cassettePath, `${JSON.stringify(cassetteEntry(runId, rawText, call))}\n`, { + mode: 0o600, + }); + const manifestPath = join(input.outputRoot, "manifest.json"); + const manifest: RigManifest = { + schema: "reviewgate.rig.manifest.v1", + runId, + scriptId: `script-${input.replayCase.className}`, + outDir: input.outputRoot, + turns: [ + { + index: 1, + snapshotDir: join(input.outputRoot, "turns", "1"), + agentExitCode: 0, + wallMs: 1, + policyReplay: { status: "complete", traces: [{ ref: stored.ref, sha256: stored.sha256 }] }, + }, + ], + policyReplay: { + catalogVersion: POLICY_CATALOG_VERSION, + sourceCommit: input.sourceCommit, + initialStateRef: state.ref, + initialStateSha256: state.sha256, + initialStateDigest: state.stateSha256, + cassetteSha256: sha256(readFileSync(cassettePath)), + cassetteRef: "cassette.jsonl", + captureDir: "policy-replay", + }, + }; + writeFileSync(manifestPath, JSON.stringify(manifest)); + const validated = validateRigPolicyReplayArtifacts({ manifest, manifestPath }); + if (validated === null) throw new Error("offline replay fixture was treated as legacy"); + const authoritative = validated.turns.get(1)?.[0]; + if (authoritative === undefined) throw new Error("offline replay envelope was not authoritative"); + return { + envelope: authoritative.envelope, + stateSnapshotRoot: authoritative.stateRoot, + callsBefore: canonicalJson(authoritative.envelope.response_calls), + }; +} + +describe("policy trace offline replay", () => { + it("replays evidence, judgment, scope, and stateful history without live providers", async () => { + const source = createSourceRepo(); + const outputRoots: string[] = []; + const sourceFileBefore = readFileSync(join(source.root, "src", "x.ts")); + const stateBefore = digestPolicyState(join(source.root, ".reviewgate")); + const statusBefore = execFileSync("git", ["status", "--porcelain=v1"], { + cwd: source.root, + encoding: "utf8", + }); + const fetchSpy = spyOn(globalThis, "fetch"); + const spawnSpy = spyOn(Bun, "spawn"); + try { + const observed: Array<{ + className: ReplayClass; + baselineVerdict: PolicyTrace["final"]["verdict"]; + counterfactualVerdict: PolicyTrace["final"]["verdict"]; + }> = []; + for (const replayCase of replayCases()) { + const outputRoot = mkdtempSync(join(tmpdir(), "reviewgate-offline-run-")); + outputRoots.push(outputRoot); + const fixture = persistAuthoritativeFixture({ + sourceRepoRoot: source.root, + sourceCommit: source.commit, + outputRoot, + replayCase, + }); + const pair = await replayPolicyEnvelopePair({ + sourceRepoRoot: source.root, + envelope: fixture.envelope, + stateSnapshotRoot: fixture.stateSnapshotRoot, + passId: replayCase.passId, + }); + + expect(pair.baseline.passes).toHaveLength(POLICY_PASS_IDS.length); + expect(pair.counterfactual.passes).toHaveLength(POLICY_PASS_IDS.length); + expect(pair.baseline.stages.map(({ stage_id }) => stage_id)).toContain( + "aggregation.cluster", + ); + expect(pair.baseline.stages.map(({ stage_id }) => stage_id)).toContain("verdict.compute"); + expect(pair.baseline.raw_response_sha256).toEqual(pair.counterfactual.raw_response_sha256); + expect(canonicalJson(fixture.envelope.response_calls)).toBe(fixture.callsBefore); + expect(pair.baseline.final.finding_severities.map(({ severity }) => severity)).toEqual([ + "INFO", + ]); + expect( + pair.counterfactual.final.finding_severities.map(({ severity }) => severity), + ).toEqual(["WARN"]); + expect(pair.baseline.final.verdict).toBe("PASS"); + expect(pair.counterfactual.final.verdict).toBe("SOFT-PASS"); + expect(pair.state.baseline.history_reads).toBeGreaterThan(0); + expect(pair.state.baseline.history_writes).toBe(1); + expect(pair.state.counterfactual.history_writes).toBe(0); + expect(pair.state.baseline.digest).not.toBe(pair.state.counterfactual.digest); + observed.push({ + className: replayCase.className, + baselineVerdict: pair.baseline.final.verdict, + counterfactualVerdict: pair.counterfactual.final.verdict, + }); + } + + expect(observed).toEqual([ + { className: "evidence", baselineVerdict: "PASS", counterfactualVerdict: "SOFT-PASS" }, + { + className: "value-judgment", + baselineVerdict: "PASS", + counterfactualVerdict: "SOFT-PASS", + }, + { className: "scope", baselineVerdict: "PASS", counterfactualVerdict: "SOFT-PASS" }, + { className: "history", baselineVerdict: "PASS", counterfactualVerdict: "SOFT-PASS" }, + ]); + expect(fetchSpy).not.toHaveBeenCalled(); + expect(spawnSpy).not.toHaveBeenCalled(); + expect(readFileSync(join(source.root, "src", "x.ts"))).toEqual(sourceFileBefore); + expect(digestPolicyState(join(source.root, ".reviewgate"))).toBe(stateBefore); + expect( + execFileSync("git", ["status", "--porcelain=v1"], { + cwd: source.root, + encoding: "utf8", + }), + ).toBe(statusBefore); + } finally { + fetchSpy.mockRestore(); + spawnSpy.mockRestore(); + for (const root of outputRoots) rmSync(root, { recursive: true, force: true }); + rmSync(source.root, { recursive: true, force: true }); + } + }); +}); diff --git a/tests/unit/policy-pass-contract-matrix.test.ts b/tests/unit/policy-pass-contract-matrix.test.ts new file mode 100644 index 0000000..4fc4240 --- /dev/null +++ b/tests/unit/policy-pass-contract-matrix.test.ts @@ -0,0 +1,90 @@ +import { describe, expect, it } from "bun:test"; +import { POLICY_PASS_IDS, POLICY_STAGES } from "../../src/core/policy/catalog.ts"; +import { + EXPLANATORY_STAGE_IDS, + POLICY_PASS_CONTRACTS, + runExplanatoryStageContract, +} from "../fixtures/policy-pass-contracts.ts"; + +describe("policy pass contract matrix", () => { + it("contains one literal contract in catalog order for every policy pass", () => { + expect(POLICY_PASS_CONTRACTS.map(({ passId }) => passId)).toEqual([...POLICY_PASS_IDS]); + }); + + it("accepts both explanatory stages and preserves material effect order", () => { + const result = runExplanatoryStageContract(); + expect(EXPLANATORY_STAGE_IDS).toEqual(POLICY_STAGES.map(({ id }) => id)); + expect(result.stages.map(({ stage_id }) => stage_id)).toEqual([...EXPLANATORY_STAGE_IDS]); + expect(result.effects.map(({ pass_id }) => pass_id)).toEqual([ + "evidence.redaction-placeholder", + "judgment.critic", + ]); + expect(result.effects.map(({ order }) => order)).toEqual([60, 70]); + }); +}); + +for (const contract of POLICY_PASS_CONTRACTS) { + describe(contract.passId, () => { + it("matches the literal numeric, severity, blocking, and inactive contract", () => { + const actual = contract.run(); + const { expected } = contract; + + expect(actual.noOpportunity.tuple).toEqual(expected.noOpportunity); + expect(actual.noMatch.tuple).toEqual(expected.noMatch); + expect(actual.active.tuple).toEqual(expected.active); + expect(actual.ablated.tuple).toEqual(expected.ablated); + expect(actual.active.blocking).toBe(expected.activeBlocking); + expect(actual.ablated.blocking).toBe(expected.ablatedBlocking); + expect(actual.active.severities).toEqual(expected.activeSeverities); + expect(actual.ablated.severities).toEqual(expected.ablatedSeverities); + + if (expected.protected === undefined) { + expect(actual.protected).toBeUndefined(); + } else { + expect(actual.protected?.tuple).toEqual(expected.protected); + expect(actual.protected?.blocking).toBe(expected.protectedBlocking); + expect(actual.protected?.severities).toEqual(expected.protectedSeverities); + } + + expect(actual.inactive).toEqual({ + pass_id: contract.passId, + status: "not-run", + reason_code: expected.inactiveReason, + }); + expect(Object.keys(actual.inactive).sort()).toEqual(["pass_id", "reason_code", "status"]); + + if (expected.variant === undefined) { + expect(actual.variant).toBeUndefined(); + } else { + expect(actual.variant?.tuple).toEqual(expected.variant.tuple); + expect(actual.variant?.blocking).toBe(expected.variant.blocking); + expect(actual.variant?.severities).toEqual(expected.variant.severities); + } + }); + + it("records the production transition and suppresses it under ablation", () => { + const actual = contract.run(); + const activeEffects = actual.active.effects.filter( + ({ pass_id }) => pass_id === contract.passId, + ); + const ablatedEffects = actual.ablated.effects.filter( + ({ pass_id }) => pass_id === contract.passId, + ); + expect(activeEffects).toHaveLength(1); + expect(actual.active.evaluations.map(({ result }) => result)).toEqual(["applied"]); + expect(ablatedEffects).toEqual([]); + expect(actual.ablated.evaluations.map(({ result }) => result)).toEqual(["would-apply"]); + expect(actual.noOpportunity.evaluations.map(({ result }) => result)).toEqual([ + "no-opportunity", + ]); + expect(actual.noMatch.evaluations.map(({ result }) => result)).toEqual(["no-match"]); + + if (actual.protected !== undefined) { + expect( + actual.protected.effects.filter(({ pass_id }) => pass_id === contract.passId), + ).toHaveLength(1); + expect(actual.protected.evaluations.map(({ result }) => result)).toEqual(["protected"]); + } + }); + }); +} From 2355ac0a4e2af53941220374d82469da6445cf57 Mon Sep 17 00:00:00 2001 From: Codevena Date: Tue, 11 Aug 2026 20:00:06 +0200 Subject: [PATCH 51/55] fix(test): bind inactive pass contracts --- ...26-08-10-policy-trace-mutation-evidence.md | 27 +++ tests/fixtures/policy-pass-contracts.ts | 159 ++++++++++++++---- .../unit/policy-pass-contract-matrix.test.ts | 42 ++++- 3 files changed, 188 insertions(+), 40 deletions(-) diff --git a/docs/dev/2026-08-10-policy-trace-mutation-evidence.md b/docs/dev/2026-08-10-policy-trace-mutation-evidence.md index eb91db7..1aaabbc 100644 --- a/docs/dev/2026-08-10-policy-trace-mutation-evidence.md +++ b/docs/dev/2026-08-10-policy-trace-mutation-evidence.md @@ -36,3 +36,30 @@ ledger was again exactly The final disposable baseline rerun was `147 pass, 0 fail, 1294 expect() calls` across seven files. The disposable tree was moved intact, rather than deleted, to the recoverable location `~/.Trash/reviewgate-task10-mutations-rPRQzq`. + +## Review Fix Round 1 — production lifecycle binding + +Independent review found that the original inactive-row fixture called +`PolicyTraceRecorder.markInactive()` directly for every pass. A disposable mutation first disabled +the production `judgment.docs-cap` inactive branch while leaving that fixture unchanged. The +pre-fix matrix stayed green (`38 pass, 0 fail, 393 expect() calls`), proving that its inactive +assertion did not observe production lifecycle behavior. A separate test-first probe then expected +the three always-on passes not to fabricate `not-run`; it failed on `evidence.fact-location` with +the fixture-created `stage-precondition-miss` row (`0 pass, 1 fail`). + +The replacement runs one real Orchestrator iteration with policy tracing in memory. Literal fixture +expectations classify `evidence.fact-location`, `evidence.grounding-token`, and +`evidence.redaction-placeholder` as empty `ran` rows; the other 15 passes are expected to be actual +production `not-run` rows with their literal `configured-off` or `stage-precondition-miss` reason. +No fixture calls `markInactive()`. + +Two post-fix mutations were applied independently in the same disposable copy: + +| Deliberate lifecycle mutation | Named failing command | Exact red evidence | +|---|---|---| +| Bypassed the production `judgment.docs-cap` configured-off transition. | `bun test tests/unit/policy-pass-contract-matrix.test.ts -t "binds every literal lifecycle"` | `0 pass, 1 fail`; production returned empty `ran`, expected `not-run/configured-off`. | +| Injected a fake production `stage-precondition-miss` for always-on `evidence.fact-location`. | Same command. | `0 pass, 1 fail`; production returned `not-run`, expected the literal empty `ran` counters. | + +Both production mutations were restored, `git diff --exit-code -- src` returned zero, and the same +lifecycle test returned `1 pass, 0 fail, 37 expect() calls`. The disposable copy was moved intact to +`~/.Trash/reviewgate-task10-r1-mutation.K9lQ5B`. diff --git a/tests/fixtures/policy-pass-contracts.ts b/tests/fixtures/policy-pass-contracts.ts index de8f4b5..08d8b6f 100644 --- a/tests/fixtures/policy-pass-contracts.ts +++ b/tests/fixtures/policy-pass-contracts.ts @@ -1,6 +1,8 @@ import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; +import { defaultConfig } from "../../src/config/defaults.ts"; +import type { ReviewgateConfig } from "../../src/config/define-config.ts"; import { type AggregateInput, aggregate } from "../../src/core/aggregator.ts"; import { validateFindingFacts } from "../../src/core/fact-check.ts"; import { @@ -15,7 +17,9 @@ import { type PolicyStageId, } from "../../src/core/policy/catalog.ts"; import { PolicyTraceRecorder } from "../../src/core/policy/trace.ts"; +import { Orchestrator } from "../../src/core/orchestrator.ts"; import { demoteSelfRefuting } from "../../src/core/self-refutation.ts"; +import type { ProviderAdapter, ReviewResult } from "../../src/providers/adapter-base.ts"; import type { Finding } from "../../src/schemas/finding.ts"; import type { PolicyEffect, @@ -49,17 +53,23 @@ export interface PolicyPassContractActual { active: PolicyContractScenario; ablated: PolicyContractScenario; protected?: PolicyContractScenario; - inactive: PolicyPassSummary; variant?: PolicyContractScenario; } +export type PolicyLifecycleExpected = + | { kind: "ran-empty" } + | { + kind: "not-run"; + reasonCode: Extract; + }; + export interface PolicyPassContractExpected { noOpportunity: PolicyNumericTuple; noMatch: PolicyNumericTuple; active: PolicyNumericTuple; ablated: PolicyNumericTuple; protected?: PolicyNumericTuple; - inactiveReason: Extract; + lifecycle: PolicyLifecycleExpected; activeBlocking: number; ablatedBlocking: number; protectedBlocking?: number; @@ -79,6 +89,103 @@ export interface PolicyPassContract { run(): PolicyPassContractActual; } +export interface PolicyLifecycleActual { + passId: PolicyPassId; + summary: PolicyPassSummary; + evaluationCount: number; +} + +const LIFECYCLE_DIFF = [ + "diff --git a/a.ts b/a.ts", + "--- a/a.ts", + "+++ b/a.ts", + "@@ -1 +1 @@", + "-export const value = 0;", + "+export const value = 1;", + "", +].join("\n"); + +function lifecycleAdapter(): ProviderAdapter { + return { + id: "codex", + async preflight() { + return { available: true, version: "fixture", authMode: "oauth", error: null }; + }, + async review(input) { + return { + reviewerId: input.reviewerId, + verdict: "PASS", + findings: [], + usage: { inputTokens: 1, outputTokens: 1, costUsd: 0, quotaUsedPct: null }, + durationMs: 1, + exitCode: 0, + rawEventsPath: "", + rawText: '{"verdict":"PASS","findings":[]}', + status: "ok", + } satisfies ReviewResult; + }, + }; +} + +export async function runProductionLifecycleContracts(): Promise { + const repoRoot = mkdtempSync(join(tmpdir(), "reviewgate-policy-lifecycle-")); + writeFileSync(join(repoRoot, "a.ts"), "export const value = 1;\n"); + try { + const config: ReviewgateConfig = { + ...defaultConfig, + cache: { enabled: false, reviewTtlDays: 7 }, + phases: { + ...defaultConfig.phases, + review: { + ...defaultConfig.phases.review, + reviewers: [{ provider: "codex", persona: "quality" }], + selfRefutationFilter: false, + hypotheticalSeverityGuard: false, + scopeToDiff: false, + scopeToSession: false, + deltaReview: false, + confidenceFloor: 0, + demoteTestSecurity: false, + capDocsSeverity: false, + providerPrecisionContext: false, + }, + brain: null, + critic: null, + fpLedger: null, + grounding: null, + implicitOutcomes: null, + lore: null, + reputation: { ...defaultConfig.phases.reputation, enabled: false }, + triage: null, + }, + }; + const result = await new Orchestrator({ + repoRoot, + config, + adapters: { codex: lifecycleAdapter() }, + sandboxMode: "off", + hostTier: "opus", + agentHost: "codex", + diff: LIFECYCLE_DIFF, + gitInfo: { sha: "a".repeat(40), branch: "fixture", dirtyFiles: ["a.ts"] }, + reasonOnFailEnabled: true, + disableLastResortFailover: true, + policyExecution: { trace: "memory", policyAblations: new Set(), authoritative: false }, + providerAvailable: (provider) => provider === "codex", + }).runIteration({ runId: "policy-lifecycle-contract", iter: 1 }); + const trace = result.policyTrace; + if (trace === undefined) throw new Error("production lifecycle trace was not returned"); + return trace.passes.map((summary) => ({ + passId: summary.pass_id, + summary, + evaluationCount: trace.evaluations.filter(({ pass_id }) => pass_id === summary.pass_id) + .length, + })); + } finally { + rmSync(repoRoot, { recursive: true, force: true }); + } +} + function finding(overrides: Partial = {}): Finding { return { id: "F-001", @@ -132,15 +239,6 @@ function scenario( }; } -function inactive( - passId: PolicyPassId, - reasonCode: Extract, -): PolicyPassSummary { - const recorder = runtime(`${passId}-inactive`); - recorder.markInactive(passId, reasonCode); - return recorder.summary(passId); -} - function runPrePass( passId: PolicyPassId, runId: string, @@ -190,7 +288,6 @@ function aggregateContract( inputs.protected, ), }), - inactive: inactive(passId, expected.inactiveReason), ...(inputs.variant === undefined ? {} : { variant: runAggregatePass(passId, `${passId}-variant`, inputs.variant) }), @@ -205,7 +302,7 @@ function factLocationContract(): PolicyPassContract { noMatch: [1, 1, 0, 0, 0, 0, 0, 0], active: [1, 1, 1, 1, 0, 1, 0, 0], ablated: [1, 1, 1, 0, 0, 0, 1, 0], - inactiveReason: "stage-precondition-miss", + lifecycle: { kind: "ran-empty" }, activeBlocking: 0, ablatedBlocking: 1, activeSeverities: ["INFO"], @@ -253,7 +350,6 @@ function factLocationContract(): PolicyPassContract { (recorder) => validateFindingFacts([activeFinding], repoRoot, new Set(), recorder), [passId], ), - inactive: inactive(passId, expected.inactiveReason), variant: runPrePass(passId, "fact-reanchor", (recorder) => validateFindingFacts([reanchorFinding], repoRoot, new Set(), recorder), ), @@ -286,7 +382,6 @@ function preAggregationContract( ...(inputs.protected === undefined ? {} : { protected: runPrePass(passId, `${passId}-protected`, inputs.protected) }), - inactive: inactive(passId, expected.inactiveReason), }), }; } @@ -367,7 +462,7 @@ export const POLICY_PASS_CONTRACTS: readonly PolicyPassContract[] = [ active: [1, 1, 1, 1, 0, 1, 0, 0], ablated: [1, 1, 1, 0, 0, 0, 1, 0], protected: [1, 1, 1, 0, 1, 0, 1, 0], - inactiveReason: "configured-off", + lifecycle: { kind: "not-run", reasonCode: "configured-off" }, activeBlocking: 0, ablatedBlocking: 1, protectedBlocking: 1, @@ -400,7 +495,7 @@ export const POLICY_PASS_CONTRACTS: readonly PolicyPassContract[] = [ active: [1, 1, 1, 1, 0, 0, 1, 0], ablated: [1, 1, 1, 0, 0, 0, 1, 0], protected: [1, 1, 1, 0, 1, 0, 1, 0], - inactiveReason: "configured-off", + lifecycle: { kind: "not-run", reasonCode: "configured-off" }, activeBlocking: 1, ablatedBlocking: 1, protectedBlocking: 1, @@ -437,7 +532,7 @@ export const POLICY_PASS_CONTRACTS: readonly PolicyPassContract[] = [ active: [1, 1, 1, 1, 0, 0, 1, 0], ablated: [1, 1, 1, 0, 0, 0, 1, 0], protected: [1, 1, 1, 0, 1, 0, 1, 0], - inactiveReason: "stage-precondition-miss", + lifecycle: { kind: "ran-empty" }, activeBlocking: 1, ablatedBlocking: 1, protectedBlocking: 1, @@ -471,7 +566,7 @@ export const POLICY_PASS_CONTRACTS: readonly PolicyPassContract[] = [ active: [1, 1, 1, 1, 0, 0, 1, 0], ablated: [1, 1, 1, 0, 0, 0, 1, 0], protected: [1, 1, 1, 0, 1, 0, 1, 0], - inactiveReason: "configured-off", + lifecycle: { kind: "not-run", reasonCode: "configured-off" }, activeBlocking: 1, ablatedBlocking: 1, protectedBlocking: 1, @@ -512,7 +607,7 @@ export const POLICY_PASS_CONTRACTS: readonly PolicyPassContract[] = [ active: [1, 1, 1, 1, 0, 1, 0, 0], ablated: [1, 1, 1, 0, 0, 0, 1, 0], protected: [1, 1, 1, 0, 1, 0, 1, 0], - inactiveReason: "stage-precondition-miss", + lifecycle: { kind: "ran-empty" }, activeBlocking: 0, ablatedBlocking: 1, protectedBlocking: 1, @@ -544,7 +639,7 @@ export const POLICY_PASS_CONTRACTS: readonly PolicyPassContract[] = [ active: [1, 1, 1, 1, 0, 1, 0, 0], ablated: [1, 1, 1, 0, 0, 0, 1, 0], protected: [1, 1, 1, 0, 1, 0, 1, 0], - inactiveReason: "configured-off", + lifecycle: { kind: "not-run", reasonCode: "configured-off" }, activeBlocking: 0, ablatedBlocking: 1, protectedBlocking: 1, @@ -591,7 +686,7 @@ export const POLICY_PASS_CONTRACTS: readonly PolicyPassContract[] = [ active: [1, 1, 1, 1, 0, 1, 0, 0], ablated: [1, 1, 1, 0, 0, 0, 1, 0], protected: [1, 1, 1, 0, 1, 0, 1, 0], - inactiveReason: "configured-off", + lifecycle: { kind: "not-run", reasonCode: "configured-off" }, activeBlocking: 0, ablatedBlocking: 1, protectedBlocking: 1, @@ -635,7 +730,7 @@ export const POLICY_PASS_CONTRACTS: readonly PolicyPassContract[] = [ active: [1, 1, 1, 1, 0, 1, 0, 0], ablated: [1, 1, 1, 0, 0, 0, 1, 0], protected: [1, 1, 1, 0, 1, 0, 1, 0], - inactiveReason: "stage-precondition-miss", + lifecycle: { kind: "not-run", reasonCode: "configured-off" }, activeBlocking: 0, ablatedBlocking: 1, protectedBlocking: 1, @@ -670,7 +765,7 @@ export const POLICY_PASS_CONTRACTS: readonly PolicyPassContract[] = [ active: [1, 1, 1, 1, 0, 1, 0, 0], ablated: [1, 1, 1, 0, 0, 0, 1, 0], protected: [1, 1, 1, 0, 1, 0, 1, 0], - inactiveReason: "stage-precondition-miss", + lifecycle: { kind: "not-run", reasonCode: "configured-off" }, activeBlocking: 0, ablatedBlocking: 1, protectedBlocking: 1, @@ -709,7 +804,7 @@ export const POLICY_PASS_CONTRACTS: readonly PolicyPassContract[] = [ noMatch: [1, 1, 0, 0, 0, 0, 0, 0], active: [1, 1, 1, 1, 0, 1, 0, 0], ablated: [1, 1, 1, 0, 0, 0, 1, 0], - inactiveReason: "stage-precondition-miss", + lifecycle: { kind: "not-run", reasonCode: "stage-precondition-miss" }, activeBlocking: 0, ablatedBlocking: 1, activeSeverities: ["INFO"], @@ -737,7 +832,7 @@ export const POLICY_PASS_CONTRACTS: readonly PolicyPassContract[] = [ active: [1, 1, 1, 1, 0, 1, 0, 0], ablated: [1, 1, 1, 0, 0, 0, 1, 0], protected: [1, 1, 1, 0, 1, 0, 1, 0], - inactiveReason: "stage-precondition-miss", + lifecycle: { kind: "not-run", reasonCode: "stage-precondition-miss" }, activeBlocking: 0, ablatedBlocking: 1, protectedBlocking: 1, @@ -771,7 +866,7 @@ export const POLICY_PASS_CONTRACTS: readonly PolicyPassContract[] = [ noMatch: [1, 1, 0, 0, 0, 0, 0, 0], active: [1, 1, 1, 1, 0, 1, 0, 0], ablated: [1, 1, 1, 0, 0, 0, 1, 0], - inactiveReason: "stage-precondition-miss", + lifecycle: { kind: "not-run", reasonCode: "stage-precondition-miss" }, activeBlocking: 0, ablatedBlocking: 1, activeSeverities: ["INFO"], @@ -799,7 +894,7 @@ export const POLICY_PASS_CONTRACTS: readonly PolicyPassContract[] = [ active: [1, 1, 1, 1, 0, 1, 0, 0], ablated: [1, 1, 1, 0, 0, 0, 1, 0], protected: [1, 1, 1, 0, 1, 0, 1, 0], - inactiveReason: "configured-off", + lifecycle: { kind: "not-run", reasonCode: "configured-off" }, activeBlocking: 0, ablatedBlocking: 1, protectedBlocking: 1, @@ -842,7 +937,7 @@ export const POLICY_PASS_CONTRACTS: readonly PolicyPassContract[] = [ active: [1, 1, 1, 1, 0, 1, 0, 0], ablated: [1, 1, 1, 0, 0, 0, 1, 0], protected: [1, 1, 1, 0, 1, 0, 1, 0], - inactiveReason: "stage-precondition-miss", + lifecycle: { kind: "not-run", reasonCode: "stage-precondition-miss" }, activeBlocking: 0, ablatedBlocking: 1, protectedBlocking: 1, @@ -883,7 +978,7 @@ export const POLICY_PASS_CONTRACTS: readonly PolicyPassContract[] = [ active: [1, 1, 1, 1, 0, 1, 0, 0], ablated: [1, 1, 1, 0, 0, 0, 1, 0], protected: [1, 1, 1, 0, 1, 0, 1, 0], - inactiveReason: "stage-precondition-miss", + lifecycle: { kind: "not-run", reasonCode: "stage-precondition-miss" }, activeBlocking: 0, ablatedBlocking: 1, protectedBlocking: 1, @@ -918,7 +1013,7 @@ export const POLICY_PASS_CONTRACTS: readonly PolicyPassContract[] = [ active: [1, 1, 1, 1, 0, 1, 0, 0], ablated: [1, 1, 1, 0, 0, 0, 1, 0], protected: [1, 1, 1, 0, 1, 0, 1, 0], - inactiveReason: "configured-off", + lifecycle: { kind: "not-run", reasonCode: "configured-off" }, activeBlocking: 0, ablatedBlocking: 1, protectedBlocking: 1, @@ -961,7 +1056,7 @@ export const POLICY_PASS_CONTRACTS: readonly PolicyPassContract[] = [ active: [1, 1, 1, 1, 0, 0, 1, 0], ablated: [1, 1, 1, 0, 0, 0, 1, 0], protected: [1, 1, 1, 0, 1, 0, 1, 0], - inactiveReason: "configured-off", + lifecycle: { kind: "not-run", reasonCode: "configured-off" }, activeBlocking: 1, ablatedBlocking: 1, protectedBlocking: 1, diff --git a/tests/unit/policy-pass-contract-matrix.test.ts b/tests/unit/policy-pass-contract-matrix.test.ts index 4fc4240..3f0188a 100644 --- a/tests/unit/policy-pass-contract-matrix.test.ts +++ b/tests/unit/policy-pass-contract-matrix.test.ts @@ -4,6 +4,7 @@ import { EXPLANATORY_STAGE_IDS, POLICY_PASS_CONTRACTS, runExplanatoryStageContract, + runProductionLifecycleContracts, } from "../fixtures/policy-pass-contracts.ts"; describe("policy pass contract matrix", () => { @@ -21,11 +22,43 @@ describe("policy pass contract matrix", () => { ]); expect(result.effects.map(({ order }) => order)).toEqual([60, 70]); }); + + it("binds every literal lifecycle classification to the production orchestrator path", async () => { + const actual = await runProductionLifecycleContracts(); + expect(actual.map(({ passId }) => passId)).toEqual( + POLICY_PASS_CONTRACTS.map(({ passId }) => passId), + ); + + for (const [index, contract] of POLICY_PASS_CONTRACTS.entries()) { + const observed = actual[index]; + expect(observed?.evaluationCount, contract.passId).toBe(0); + if (contract.expected.lifecycle.kind === "ran-empty") { + expect(observed?.summary, contract.passId).toEqual({ + pass_id: contract.passId, + status: "ran", + considered: 0, + opportunities: 0, + would_apply: 0, + applied: 0, + protected: 0, + blocking_removed: 0, + blocking_preserved: 0, + dropped: 0, + }); + } else { + expect(observed?.summary, contract.passId).toEqual({ + pass_id: contract.passId, + status: "not-run", + reason_code: contract.expected.lifecycle.reasonCode, + }); + } + } + }); }); for (const contract of POLICY_PASS_CONTRACTS) { describe(contract.passId, () => { - it("matches the literal numeric, severity, blocking, and inactive contract", () => { + it("matches the literal numeric, severity, and blocking contract", () => { const actual = contract.run(); const { expected } = contract; @@ -46,13 +79,6 @@ for (const contract of POLICY_PASS_CONTRACTS) { expect(actual.protected?.severities).toEqual(expected.protectedSeverities); } - expect(actual.inactive).toEqual({ - pass_id: contract.passId, - status: "not-run", - reason_code: expected.inactiveReason, - }); - expect(Object.keys(actual.inactive).sort()).toEqual(["pass_id", "reason_code", "status"]); - if (expected.variant === undefined) { expect(actual.variant).toBeUndefined(); } else { From fa68dfad8e752954656165b45f67e4c460bb4eba Mon Sep 17 00:00:00 2001 From: Codevena Date: Tue, 11 Aug 2026 20:21:16 +0200 Subject: [PATCH 52/55] fix(cli): describe authoritative replay options --- src/cli/index.ts | 10 +++-- tests/unit/cli-required-args.test.ts | 58 +++++++++++++++++++++++++++- 2 files changed, 63 insertions(+), 5 deletions(-) diff --git a/src/cli/index.ts b/src/cli/index.ts index 4ea1394..afde6b8 100644 --- a/src/cli/index.ts +++ b/src/cli/index.ts @@ -904,7 +904,7 @@ const bench = defineCommand({ meta: { name: "matrix", description: - "Ablation: run the corpus with suppression layers toggled and print the per-layer Δ (spec §8 money-shot)", + "Run exact internal policy ablations over captured baseline responses and print per-pass deltas", }, args: { corpus: { type: "string", required: true, description: "Corpus directory" }, @@ -912,7 +912,8 @@ const bench = defineCommand({ ablate: { type: "string", required: true, - description: "Comma-separated layers: critic,confidence-floor,reputation,scope-to-diff", + description: + "Comma-separated closed catalog IDs (e.g. evidence.fact-location); legacy aliases accepted for compatibility: critic,confidence-floor,reputation,scope-to-diff", }, providers: { type: "string", @@ -940,7 +941,7 @@ const bench = defineCommand({ "max-output-tokens": { type: "string", description: "OpenRouter output ceiling" }, authoritative: { type: "boolean", - description: "Require the paired critic-only authoritative protocol", + description: "Require complete paired trace and evidence validation", }, preregistration: { type: "string", description: "Committed preregistration JSON" }, "min-clean": { type: "string", description: "Required distinct clean cases" }, @@ -1175,7 +1176,8 @@ const rig = defineCommand({ script: { type: "string", required: true, description: "The turn script the run used" }, cassette: { type: "string", - description: "Also check the recording's integrity (entry count, FIFO keys, bodies)", + description: + "Also verify the recording by stable logical call identity, ordered response hashes, and bodies", }, }, async run({ args }) { diff --git a/tests/unit/cli-required-args.test.ts b/tests/unit/cli-required-args.test.ts index 25ced03..72a2f90 100644 --- a/tests/unit/cli-required-args.test.ts +++ b/tests/unit/cli-required-args.test.ts @@ -8,7 +8,8 @@ // marker). The pre-fix manual checks instead printed "... is required" and // exited 2, so the citty message + exit-1 distinguishes fixed from unfixed. import { describe, expect, it } from "bun:test"; -import { mkdtempSync, writeFileSync } from "node:fs"; +import { spawnSync } from "node:child_process"; +import { closeSync, mkdtempSync, openSync, readFileSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; @@ -28,6 +29,27 @@ async function run(args: string[]): Promise<{ code: number; stdout: string; stde return { code, stdout, stderr }; } +function runSync(args: string[]): { code: number; stdout: string; stderr: string } { + const root = mkdtempSync(join(tmpdir(), "rg-cli-help-")); + const outputPath = join(root, "output.txt"); + const output = openSync(outputPath, "w"); + try { + const proc = spawnSync("bun", [CLI, ...args], { + stdio: ["ignore", output, output], + env: { ...process.env, NODE_ENV: "production" }, + }); + closeSync(output); + return { code: proc.status ?? -1, stdout: readFileSync(outputPath, "utf8"), stderr: "" }; + } finally { + try { + closeSync(output); + } catch { + // The successful path closes before reading; only the exceptional path reaches this close. + } + rmSync(root, { recursive: true, force: true }); + } +} + describe("CLI required-arg declarations (F-079)", () => { const cases: Array<{ name: string; argv: string[]; flag: string }> = [ { name: "audit verify --file", argv: ["audit", "verify"], flag: "file" }, @@ -94,3 +116,37 @@ describe("Rig authority exit code", () => { expect(stderr).toContain("catalog-mismatch"); }); }); + +describe("policy replay CLI help contracts", () => { + it("describes Bench Matrix as exact internal closed-catalog ablation", () => { + const { code, stdout, stderr } = runSync(["bench", "matrix", "--help"]); + const help = `${stdout}${stderr}`; + + expect(code).toBe(0); + expect(help).toContain("exact internal policy ablations"); + expect(help).toContain("evidence.fact-location"); + expect(help).toContain("legacy aliases accepted for compatibility"); + expect(help).toContain("critic,confidence-floor,reputation,scope-to-diff"); + expect(help).not.toContain("suppression layers toggled"); + expect(help).not.toContain("critic-only authoritative protocol"); + }); + + it("describes Rig Cassette verification by stable logical identity and hashes", () => { + const { code, stdout, stderr } = runSync(["rig", "replay", "--help"]); + const help = `${stdout}${stderr}`; + + expect(code).toBe(0); + expect(help).toContain("stable logical call identity"); + expect(help).toContain("ordered response hashes"); + expect(help).not.toContain("FIFO"); + }); + + it("keeps Audit help as the unchanged control", () => { + const { code, stdout, stderr } = runSync(["audit", "--help"]); + const help = `${stdout}${stderr}`; + + expect(code).toBe(0); + expect(help).toContain("Audit utilities"); + expect(help).toContain("verify"); + }); +}); From 2829f238d36257276359223937bae8e034910322 Mon Sep 17 00:00:00 2001 From: Codevena Date: Tue, 11 Aug 2026 20:23:21 +0200 Subject: [PATCH 53/55] docs: hand off policy trace slice one --- NEXT_SESSION.md | 314 +++++------------- TEST_PLAN.md | 39 ++- docs/architecture.md | 48 ++- ...8-09-policy-accountability-trace-design.md | 30 +- 4 files changed, 198 insertions(+), 233 deletions(-) diff --git a/NEXT_SESSION.md b/NEXT_SESSION.md index 7dda53f..e65e19a 100644 --- a/NEXT_SESSION.md +++ b/NEXT_SESSION.md @@ -1,239 +1,93 @@ # Reviewgate — Next-Session Handoff -_Last updated: 2026-08-07, after the rig stale-report defect was diagnosed, specced and planned. -Supersedes all earlier content._ +_Last updated: 2026-08-11. Supersedes all earlier content._ ## One-line state -**The rig stale-report defect is fully diagnosed and measured, the design and the implementation -plan are written and committed — but NOT ONE LINE OF THE FIX IS IMPLEMENTED. The next session -implements `docs/superpowers/plans/2026-08-07-rig-stale-report-fix.md`, Task 1 first.** +**Policy Accountability & Pruning Slice 1 is implemented on +`feat/policy-accountability-trace` through core commit `2355ac0` and CLI-help contract commit +`fa68dfa`; this documentation commit closes its handoff and verification. The next milestone is +Slice 2 measurement and pruning design—not pass deletion.** -## Verified state (checked with commands at handoff time) +## Checkout and publication state | | | |---|---| -| my commits this session | **`384df2a`** (spec), **`f36abf1`** (plan), **`734f5eb`** (plan corrections) — docs only, zero code | -| pushed? | **NO.** ~~`master` is 8 ahead~~ → **12 ahead** of `origin/master` as of 2026-08-08 01:4x — 3 are mine, **9 are the Qwen session's** | -| ~~the other 5~~ the other 9 | `9fac6f8`, `ef54ed0`, `979bfea`, `e7c25e1`, `0f8b6cf`, `cef7022`, `56564ea`, `63a779f` (+ `b3032b3`, `9473973` already pushed) — **the SECOND SESSION's bench/Qwen work, now CLOSED OUT.** See "Second work stream" below | -| working tree | only `.reviewgate/lore/approvals.jsonl` (foreign gate state) — leave it alone | -| suite | **3209 pass / 12 skip / 0 fail**, exit 0 — re-run at `56564ea` by the Qwen session (139 s). ⚠ One earlier full run showed `1 fail` that did **not** reproduce in four subsequent runs; the name was never captured. Only hint: `cassette: prompt drift for codex-security` appeared solely in the failing run. Treat as flaky-unknown, not as green-by-proof | -| build | **deliberately NOT run.** Installed binary still `sha256:fc9b8c18…` | -| ~~Trailhead stamp left at `5543549` ON PURPOSE~~ | → **now `cef7022`, stamped 2026-08-08 (`56564ea`).** This is the completion of that decision, not an override: the 4 GEÄNDERT rows were the Qwen session's own work, and that session verified them (`bench.ts`/`runner.ts` entry points still correct, 0 FEHLT, 80/80 lines). The session that could honestly stamp them did | - -⚠ **A SECOND SESSION IS COMMITTING TO THIS CHECKOUT.** Never `git add -A`; stage explicit paths and -check `git log` before assuming a commit is yours. A `git worktree` remains the standing fix. - -## What got done — and how it was verified - -**Nothing was implemented. What exists is a diagnosis backed by measurement, plus a gated plan.** - -The handoff that suggested this task described it as "a dead turn inherits the previous turn's -`pending.json`". That was an order of magnitude too small. Measured against the recorded pilots: - -| | pilot-01 | pilot-02 | pilot-03 | total | -|---|---|---|---|---| -| turns opening with the previous turn's final report | 11/12 | 11/12 | 9/12 | **31/36** | -| reports owned by the turn | 19 | 14 | 14 | **47** | -| reports inherited from an earlier turn | 11 | 11 | 9 | **31** | -| reports owned by **no** turn (orphans) | 0 | 0 | 0 | **0** | - -**13 of 36 turns count findings they did not earn; 9 of those produced none of their own.** -Sharpest case: pilot-03 turn 5 has an EMPTY audit delta and still reports 3 findings, all turn 4's. - -Root cause: `driver.ts:201` promises to archive every version that **appears** while a turn runs; -it was implemented as every version that **exists**. The first poll fires 250 ms in, while the -predecessor's `pending.json` is still on disk. - -**Evidence, not adjectives — every number above came from a command run against -`rig/results/pilot-0{1,2,3}/`, not from reading code.** Also executed and confirmed: - -- `run_id` maps **1:1 to a turn** across all **34** recorded gate runs — none spans two turns. - This is what makes the ownership rule sound and retroactive. -- Every one of the 31 inherited reports is **byte-identical** to its predecessor's final report, - so dropping it loses nothing. -- The **pre-fix baseline** for all three pilots is captured (table below) so the correction delta - cannot be back-fitted. -- `createHash`/`existsSync`/`readFileSync`/`join`/`reviewgateDir` are already imported in - `driver.ts:6-20`; `window.runs`/`runDelta` are in scope at the harvest insertion point. - -**Pre-fix baseline — capture this again only if you distrust it; do not overwrite it:** - -| | pilot-01 | pilot-02 | pilot-03 | -|---|---|---|---| -| recall | 0.60 (3/5) | 0.33 (1/3) | 1.00 (2/2) | -| escape rate | 0.20 (1/5) | 0.67 (2/3) | 0.00 (0/2) | -| M2 slope | 0.0239/turn (n=10) | 0.0000/turn (n=9) | 0.0014/turn (n=9) | -| iterations median | 1 over 12 reviewed | 1 over 12 reviewed | 1 over 10 reviewed | -| cost | $0.0236 | $0.0125 | $0.0136 | - -### Plan gate: ONE round, and only half a gate - -- **agy (Slot B): PASS**, 0 CRITICAL / 0 WARN / 1 INFO. Findings file verified fresh (mtime - 11:14:35Z against a round start of 11:13:26Z), log 2767 bytes. Its INFO was **correct** and is - fixed in `734f5eb`. -- **Slot A (executing): STILL OPEN.** agy's log shows a single `readFile` — it reviewed by reading, - not by executing, despite being told to run the code. -- **The proof that this matters:** I found a plan-breaking defect agy missed while it asserted "the - rule produces deterministic, safe outcomes in all cases" — four fixture turns declare `reports` - but no `iterations`, so under the new rule their reports become orphans and **three existing - `criticRuns` tests collapse to `[]`**. That is now Task 1 Step 4. - -## THE NEXT TASK - -**Implement the plan, Task 1 → Task 4, in order.** -`docs/superpowers/plans/2026-08-07-rig-stale-report-fix.md` - -Why it is next: every future rig measurement rests on the harvester being right, and the corpus is -currently wrong in a way that is invisible from the reports themselves. The harvest half works -**retroactively and needs no rebuild**, so the three recorded pilots become usable again rather than -being written off. - -Entry points: `src/rig/harvest.ts:141` (`collectTurnFindings`) and `:413` (its call site); -`src/rig/driver.ts:214` (`startReportArchiver`). - -**Task 1 must land first** — it is a test-only refactor, and without its Step 4 Task 2 reddens three -existing tests for the wrong reason. - -## Traps — NEW this session - -- **`run_id` alone is the ownership key, never `(run_id, iter)`.** A gate that writes - `pending.json` for iteration 3 and dies before appending `run.complete` would have its REAL report - dropped as an orphan under a pair key. Verified 1:1 across 34 gate runs. -- **Four fixture turns model an impossible state** (`reports` with no `iterations`): the three - `criticRuns` tests at `rig-harvest.test.ts:360`, `:384`, `:407`. They need a gate iteration added. - Do NOT add one to `:609` ("a turn where the gate never ran") — that one is deliberately dead. -- **The trailhead stamp was deliberately NOT moved.** All 4 GEÄNDERT rows (`tests/unit/`, - `src/cli/commands/bench.ts`, `src/bench/runner.ts`, `src/cli/commands/`) are the PARALLEL - session's bench work, which this session never looked at. Stamping HEAD would claim a verification - that did not happen. 0 FEHLT, 66/70 still valid, `CLAUDE.md` at exactly 80/80 lines. -- **The gate escalated on findings that are not mine and cannot be honestly dispositioned.** - `F-002`/`F-003` on `src/providers/opencode.ts` are the parallel session's code, but the ownership - snapshot marked them `session_attributable: true` (their edits landed inside my baseline window), - so `out-of-scope` and `out-of-session` both fail closed. ~~They remain **open and escalated**~~ → - **RESOLVED 2026-08-08 by the session that owns that code:** `F-002` **fixed** (the measurement - scaffolding was removed from the reviewer path, `63a779f`), `F-003` **rejected with a reason** and - carried as a named next task. Both decisions are in `.reviewgate/decisions/1.jsonl`. The - quota-degraded-panel caveat still applies to any *re-run* (codex capped until 2026-08-08 11:07Z). -- **`harvest.ts` never reads `manifest.turns[].gateReviewed`** — the flag exists, is written by - the driver, and is consulted by nothing. The plan subsumes it rather than adding a second signal. -- **An `iterations === 0` warning that says "EXCLUDED from the M1/cost-per-turn samples" is true and - misleading** — findings, recall, escape and suppression were never excluded. - -## Traps — still standing - -- **Never run `bun run build` casually** — re-pins the binary AND deploys machine-wide via the - `~/.local/bin/reviewgate` symlink. Build → record sha → preregister → run. **Task 3's driver fix - reaches no real `rig run` until someone rebuilds; that is deliberately out of scope.** -- **Never pipe `bun test` through `tail`** — a red test's identity is lost. Redirect to a file. -- **`bun run lint`/`tsc` do NOT cover `rig/scripts/`.** Check it explicitly (needs `typeRoots` - pointing at `node_modules` — `bun-types` is not under `@types/`). -- **`agy` fails 0-byte intermittently, and reviews shallowly even when it does not.** A missing - findings file is an OPEN slot; so, arguably, is a PASS whose log shows no execution. -- **Codex quota resets 2026-08-08 11:07Z.** Until then the executing slot is agy or a Claude - subagent; that is the normal configuration, not a degraded one. -- **A rate over `reports/*-pending.json` is a rate over SURVIVORS.** Use `cassette.jsonl`. -- **`rig/results/` is gitignored** — every number here is reproducible only on this machine. -- **Never reimplement a shipped helper in a rig script** — import it. -- **`applySymbolSignatures` runs BEFORE `validateFindingFacts`** (`orchestrator.ts:2219`, `:2226`). -- Reviewgate's decision protocol assumes fix-and-decide within ONE turn; an agent that delegates a - fix to a background worker structurally cannot. Still unaddressed. - -## Open Trailhead note (carried forward) - -`CLAUDE.md`'s Mess-Rig row points at `src/rig/driver.ts` rather than the offline replays under -`rig/scripts/`. After this session's work the row is arguably *more* correct than before — the next -task's entry points are `src/rig/driver.ts` and `src/rig/harvest.ts`. Left as-is; `CLAUDE.md` is at -exactly 80/80 lines, so any change is a swap, not an addition. - -## Read-first order - -1. This file. -2. `docs/superpowers/plans/2026-08-07-rig-stale-report-fix.md` — the plan to execute. -3. `docs/superpowers/specs/2026-08-07-rig-stale-report-design.md` — why the rule is what it is, - especially §"Why `run_id` alone" and §"Failure handling". -4. `.reviewgate/ESCALATION.md` — the open, not-mine findings, before ending your first turn. - ---- - -# Second work stream — Qwen3.8-Max as a measured reviewer (session of 2026-08-07/08) - -_Independent of the rig stale-report task above. Both are live in this checkout._ - -## One-line state - -**The cost question is answered and the tooling is built and committed; the *quality* question is -untouched. Phase 2 (the 30-case exploratory bench) is the next step — but run it on a Standard tier, -not on Lite.** - -## What got done — and how it was verified - -| | | -|---|---| -| `scripts/measure-opencode-tokens.ts` (`9473973`, **pushed**) | Token oracle over opencode's SQLite session DB. 6/6 green; mutation seen red (coefficient 1.21 → 2.42 ⇒ 4 pass / 2 fail, measured value `118.68406`) | -| `bench --provider-model` (`ef54ed0`) | Pins a reviewer's upstream model into provenance. 12/12 green; mutation reddened **exactly the 4 predicted cases**, the sentinel test stayed green. Verified end-to-end: a real `bench run` wrote `"model": "alibaba-token-plan/qwen3.8-max"`, not `"default"` | -| Overhead + caching measurements (`9fac6f8`, `979bfea`, `e7c25e1`, `0f8b6cf`) | Artifacts under `bench/results/qwen-overhead/`. Every credit number is **console-read**, not modelled | -| Risk-control scope correction (`cef7022`) | Markus: the block suspends **purchases only**; renewal and tier changes are unaffected | -| Trailhead stamp (`56564ea`) | `verify-map.js`: 0 FEHLT, 4 GEÄNDERT (all this stream's), entry points re-checked | -| F-002 fix (`63a779f`) | Measurement scaffolding removed from the live reviewer path | - -**The numbers, all console-verified:** - -| | credits/call | | | -|---|---|---|---| -| baseline, default agent | 31.01 | 30 × 1 | 30 × 3 | -| + reduced tool set (`--agent`) | 22.86 | | | -| + warm cache | 9.17 | | | -| **real case** (2 calls/case, 1st cold) | **28.2 /case** | **846 cr** | **2,538 cr** | -| …as % of a **Lite** window (2,500) | | 34 % | **102 % — does not fit** | -| …as % of a **Standard** window (10,000) | | 8.5 % | **25.4 % — fits** | - -Smoke run: `2/2 cases scored → precision 1, recall 1, clean-FP 0`. **N=2 — that is a pipeline test, -not evidence about review quality. The acceptance bar in the spec is untouched.** - -## THE NEXT TASK — and why - -**Decide the tier before spending anything.** Phase 2 costs 34 % of a Lite window but 8.5 % of a -Standard one, and Phase 3 is impossible on Lite and routine on Standard. Running Phase 2 on Lite is -the expensive ordering: it burns a third of the week to answer a question whose follow-up you then -cannot afford. $12/month decides this, and the risk-control block does **not** stand in the way. - -Once the tier is settled, Phase 2 is `reviewgate bench run --corpus bench/cases --providers -opencode,ollama,claude-code --provider-model opencode=alibaba-token-plan/qwen3.8-max`. The bar is -preregistered in the spec §6: Qwen earns a slot if it finds **≥1 seeded bug that GLM-5.2 and -claude-code both miss**, at a clean-FP rate no worse than GLM-5.2's. - -**Second task, small and independent:** finding **F-003** (rejected, carried forward) — -`src/providers/opencode.ts` `complete()` still passes `--dangerously-skip-permissions`, which does -not exist in opencode 1.18.10. Own commit, own gate: it changes curator runtime behaviour. - -## Traps — NEW from this stream - -- **`--dangerously-skip-permissions` does not exist in opencode 1.18.10.** The flag is `--auto`, and - opencode **exits 0 on unknown flags** instead of rejecting them, so the dead flag was silently - ignored on every call for an unknown span of time. Fixed at `:97`, still live at `:242`. -- **Credits are read, never computed.** Fitting the uncached/cached coefficients across three - calibration points does **not converge** (1.25–1.71 and 0.05–0.43 per 1K). The console shows two - decimals (±0.125 credits). The token model ran **9 % low** on its one real test. Any credit figure - in a future doc must cite a console delta. -- **A bench case costs ~2 LLM calls, not 1**, and the first call of a run pays a cold cache. Any - per-*call* figure understates the per-*case* cost by ~3×. -- **`bench.ts` rejects a corpus with zero clean cases** (exit 4). A "just run one case" smoke test is - invalid; take one clean + one seeded. -- **The reduced-tool `--agent rg-reviewer` win (23.5K → 17.8K input tokens) is real but not shipped.** - It depended on `~/.config/opencode/agent/rg-reviewer.md`, which exists only on Markus's machine. - Make it a config option before reintroducing it — do not hard-code it in the adapter again. -- **`phases.brain.curator` points at `opencode` with `model: "minimax-m2"`, whose plan has expired.** - The call **hangs** instead of erroring — killed after 150 s. Independent of everything above. -- **Pay-per-token DashScope is blocked**, account-wide (`AccessDenied.Unpurchased` on all 5 models - tested), root cause `RISK.RISK_CONTROL_REJECTION`. KYC would unblock it but costs a passport scan - and a month of bank transactions — and buys only pay-as-you-go and Extra Bundles, **neither of - which this work needs**. Do not treat it as a prerequisite. - -## Read-first order for this stream - -1. `bench/results/qwen-overhead/DECISION.md` — the go/no-go and every caveat. -2. `docs/superpowers/specs/2026-08-07-qwen-reviewer-measurement-design.md` §5a (the live options) and - §6 (the preregistered acceptance bar). -3. `docs/superpowers/plans/2026-08-07-qwen-overhead-and-provider-model.md` — the three findings - mappings at the end are the record of what three gate rounds actually caught. +| branch | `feat/policy-accountability-trace` | +| isolated worktree | `/Users/markus/.config/superpowers/worktrees/reviewgate/policy-accountability-trace` | +| implementation boundary | `9bc72c1..fa68dfa` (Slice-1 code/tests and authoritative replay help; this handoff is the following documentation commit) | +| pushed? | **NO**—do not push without Markus's explicit permission | +| main checkout | out of scope; preserve its foreign `.reviewgate/lore/approvals.jsonl` | + +## What Slice 1 delivered + +- A closed `reviewgate.policy-catalog.v1` with 18 ablatable outcome-changing passes in fixed order + and two non-ablatable explanatory stages: `aggregation.cluster`, `verdict.compute`. +- Full ordered evaluations and compact material effects, opportunity/activation/protection counters, + exact lineage and final verdict identity. +- Canonical mode-`0600`, content-addressed Audit traces bound into `run.complete`; production trace + errors/overflow remain fail-open with respect to the already-computed policy verdict. +- Exact authoritative Bench pairing: only the baseline calls live providers; variants consume the + same captured logical responses and change only the internal ablation set. +- Exact Rig capture/replay: Cassette call identities, response order, source commit/diff and state + digests are bound; persistent baseline/counterfactual scratch branches are isolated from the + measured checkout and from each other. +- An 18-row production contract harness, four-class offline replay and ten mutation-proven + accountability boundaries. + +## Hard limits—carry these into every Slice-2 claim + +- Zero opportunities mean **no evidence**, not evidence that a pass is useless. +- Lore is additive and excluded from the 18 demoters; measure its added review/decision load + separately. +- Stateful history passes require seeded multi-turn sequences. A fresh per-case Bench makes them + inert by construction. +- Slice 1 ranked and deleted **no** pass and changed no intended production finding/verdict. +- `ImplicitOutcomeStore` is branch-locally preserved during replay, but current production only + writes it; no later policy input reads it back. Its divergent row count is persistence evidence, + not a demonstrated downstream review effect. +- One-pass leave-one-out is insufficient for interacting policy. At minimum measure critic × + confidence × reputation; diff × delta × session scope; cycle × region × FP history; and fact + location × token/LLM grounding × redaction × self-refutation. +- Authoritative Bench/Rig evidence is all-or-nothing. Missing/corrupt/overflowed/cross-catalog or + identity-mismatched artifacts invalidate the measurement with exit `4`; never coerce them to zero. + +## Next concrete task: specify and preregister Slice 2 + +Start from `src/core/policy/catalog.ts` and +`docs/superpowers/specs/2026-08-09-policy-accountability-trace-design.md` § Slice-2 handoff. Before +running or deleting anything: + +1. Write a Slice-2 measurement/pruning spec and implementation plan. +2. Freeze the 30-case × 3-repeat stateless replay corpus, seeded multi-turn Rig/Cassette sequences, + dogfood disposition source, opportunity minima and precision/recall/no-unique-contribution + deletion criteria. +3. Preregister the interaction groups above and define how multiple-testing/rare-pass uncertainty is + reported. +4. Pass the normal executable plan gate. Only then run measurements. + +Slice 2 owns rankings, interaction measurements and delete/consolidate decisions. Slice 3 extracts +only surviving policy and removes obsolete config/schema/marker/test/documentation surfaces. + +## Reverification commands + +```bash +bun test tests/unit/policy-catalog.test.ts tests/unit/policy-trace-schema.test.ts tests/unit/policy-trace-recorder.test.ts tests/unit/policy-pass-contract-matrix.test.ts tests/integration/policy-trace-equivalence.test.ts tests/integration/policy-trace-offline-replay.test.ts tests/unit/bench-matrix.test.ts tests/unit/rig-replay.test.ts tests/unit/audit-verify-corruption.test.ts +bunx tsc --noEmit +bun run lint +bun test +bun run build +./dist/reviewgate bench matrix --help +./dist/reviewgate rig replay --help +./dist/reviewgate audit --help +``` + +The last four commands are build/help smokes only. Do not start a live Bench, Rig replay or provider +operation as part of handoff verification. + +## Read first + +1. `docs/superpowers/specs/2026-08-09-policy-accountability-trace-design.md` +2. `docs/superpowers/plans/2026-08-09-policy-accountability-trace.md` +3. `docs/dev/2026-08-10-policy-trace-mutation-evidence.md` +4. `docs/architecture.md` and `TEST_PLAN.md` diff --git a/TEST_PLAN.md b/TEST_PLAN.md index d4e6fc2..d1133db 100644 --- a/TEST_PLAN.md +++ b/TEST_PLAN.md @@ -19,7 +19,7 @@ Snapshot/inspect files live under `flashbuddy/.reviewgate/`: `pending.json`/`pen ## Layer 1 — Automated (deterministic, no network) ``` export PATH="$HOME/.bun/bin:$PATH" -bun test # expect ~300 pass / 9 skip / 0 fail +bun test # require the real terminal summary: 0 fail bun run typecheck # clean bun run lint # clean ``` @@ -27,6 +27,43 @@ Covers every phase's logic with fakes: loop FSM, triage, aggregator (+dedup/crit signatures, cache, brain (store/select/engine/curator/lifecycle/fetcher/embeddings), config, audit, all adapters, full P0→P4 integration. +### Policy Accountability & Replay — Slice 1 + +This deterministic block is the focused acceptance suite for the closed 18-pass catalog, both +explanatory stages, fail-open production tracing and fail-closed authoritative measurement: + +```bash +bun test tests/unit/policy-catalog.test.ts \ + tests/unit/policy-trace-schema.test.ts \ + tests/unit/policy-trace-recorder.test.ts \ + tests/unit/policy-pass-contract-matrix.test.ts \ + tests/integration/policy-trace-equivalence.test.ts \ + tests/integration/policy-trace-offline-replay.test.ts \ + tests/unit/bench-matrix.test.ts \ + tests/unit/rig-replay.test.ts \ + tests/unit/audit-verify-corruption.test.ts +``` + +The suite must prove: + +- all 18 catalog rows and the `aggregation.cluster`/`verdict.compute` stages are present in fixed + order, with explicit no-opportunity, no-match, active, ablated and protected contracts; +- trace-on/off leaves findings, legacy markers, Markdown, counts and verdict byte-equivalent after + optional telemetry is removed; +- Audit/Bench policy artifacts are canonical and content-addressed; Rig state and Cassette evidence + are mode-`0600`, contained and bound by exact content hashes/identities; +- Bench uses one live baseline and exact captured-response replay for internally ablated variants; +- Rig replays exact calls in persistent, isolated baseline/counterfactual branches without live + provider or network calls and without production-state writes; +- missing, corrupt, overflowed, reordered, cross-catalog or mismatched evidence is non-authoritative + and authoritative commands exit `4` rather than substituting zero counters. + +The ten required mutation proofs, their named red regressions and restored source hashes are kept in +`docs/dev/2026-08-10-policy-trace-mutation-evidence.md`. Do not replace them with a green-only full +suite. Slice 1 is measurement plumbing, not evidence that a pass is useful or useless: zero +opportunities do not support deletion, Lore is additive/outside the 18, and history passes need +seeded multi-turn sequences. + ## Layer 2 — Real CLI/API e2e (gated; needs real providers + OPENROUTER_API_KEY) ``` REVIEWGATE_E2E=1 bun test tests/e2e/ diff --git a/docs/architecture.md b/docs/architecture.md index 5a3ff42..9bc3a4a 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -62,6 +62,50 @@ Decides **allow-stop vs. block**: triage → cache check → research → reviewer panel → critic → aggregate → write report ``` +### Policy accountability (`src/core/policy/`) + +The review path has a closed, versioned catalog of **18 outcome-changing passes**. The catalog is +not a second rule engine: production predicates and precedence remain in the existing +Orchestrator/Aggregator path, while `PolicyTraceRecorder` records each pass's opportunity, match, +protection and material transition at that path's actual execution point. + +| Class | Ordered catalog IDs | +|---|---| +| evidence | `evidence.fact-location`, `evidence.self-refutation`, `evidence.grounding-token`, `evidence.redaction-placeholder` | +| value judgment | `judgment.hypothetical`, `judgment.grounding-llm`, `judgment.critic`, `judgment.confidence`, `judgment.reputation`, `judgment.test-security`, `judgment.docs-cap` | +| scope | `scope.diff`, `scope.delta`, `scope.session` | +| history | `history.fp-signature`, `history.cycle-rejected`, `history.fp-cluster`, `history.region-rejected` | + +`aggregation.cluster` and `verdict.compute` are the two non-ablatable explanatory stages. Lore is +additive—it may append findings, but it is not one of the 18 demoters and is measured separately. +The catalog and its fixed order live in `src/core/policy/catalog.ts`; the persisted schema lives in +`src/schemas/policy-trace.ts`. + +Normal audited Gate runs never receive an ablation set. They persist a complete canonical trace to +`.reviewgate/audit/YYYY/MM/DD/policy/-i-.json` and bind its relative reference +and SHA-256 into `run.complete` plus the compact `pending.json.policy_summary`. The artifact is +mode `0600`, limited to 1 MiB and verified through the audit chain. Trace recording or persistence +failure is telemetry-only: the already-computed policy result and Gate verdict survive, while the +trace status becomes `error` or `overflow`. This does not relax ordinary reviewer failures, which +still fail closed. + +Exact ablation is internal to measurement code: + +- `bench matrix` makes the baseline the only live-provider path, captures the globally ordered + reviewer/preflight/completion results, then replays each variant through the same policy path + with only `policyAblations` changed. The matrix directory contains content-addressed `artifacts/` + for results, response manifests, policy traces and their trace-set binding. +- A traced Rig run binds `manifest.json`, `cassette.jsonl`, `policy-replay/` envelopes, exact diffs + and content-addressed policy-state snapshots. Replay joins responses by stable logical call ID + and ordered hashes, then runs baseline/counterfactual in separate persistent branch-local scratch + checkouts. Production state is never a replay target. + +Bench/Rig treat a missing, corrupt, incomplete, cross-catalog or identity-mismatched evidence set +as invalid measurement and exit `4`; no absent counter is interpreted as zero. Stateful history +passes require seeded multi-turn sequences. Branch-local `ImplicitOutcomeStore` writes are retained +as causal evidence, but current production writes that store without feeding it back into later +policy inputs. + ## Module map | Area | Responsibility | @@ -71,6 +115,8 @@ triage → cache check → research → reviewer panel → critic → aggregate | **`src/providers/`** | One adapter per reviewer CLI (`codex.ts`, `gemini.ts`, `claude.ts`, `openrouter.ts`, `opencode.ts`, `ollama.ts`), all implementing `adapter-base.ts`. Most spawn the real CLIs via `src/utils/spawn.ts` (`spawnSafely`, which closes stdin — codex hangs otherwise); `openrouter.ts` and `ollama.ts` are subprocess-free HTTP adapters instead (`SUBPROCESSLESS_PROVIDERS` in `registry.ts`). `review-output.ts` holds the shared `REVIEW_OUTPUT_SCHEMA` and parses reviewer JSON into `Finding`s. | | **`src/hosts/`** | Generates and merges native Claude Code and Codex lifecycle hooks. Codex commands resolve the Git root, preserve hook stdin, identify `REVIEWGATE_AGENT_HOST=codex`, and fail closed when the Stop shim is unavailable. Hook installation and Codex hash trust are intentionally separate states. | | **`src/core/`** | `aggregator.ts` (severity-weighted verdict + dedup + consensus), `critic.ts` (demote-only adversarial pass), `report-writer.ts` (renders `pending.md`/`pending.json`), `state-store.ts` (locked, atomic `state.json`). | +| **`src/core/policy/`** | Closed 18-pass catalog, ordered in-memory trace recorder and internal-only replay/ablation contract. | +| **`src/audit/`** | Hash-chained audit events plus canonical, content-addressed policy-trace persistence and verification. | | **`src/research/`** | `symbol-graph.ts` (tree-sitter, TS/Python `.wasm` grammars), `conventions.ts`, `research-writer.ts` produce `research.md`, injected as trusted context before the diff fence. | | **`src/core/brain/`** | Per-repo memory ("Brain") + Curator. Default OFF. `fetcher.ts` is an SSRF-hardened `safeFetch`; the curator phase is non-blocking, timeout-bounded, and never changes the verdict. | | **`src/config/`** | `reviewgate.config.ts` is parsed as data, never executed. `control-plane.ts` fingerprints source/effective policy separately, retains the last-known-good config, forces candidates through a special path, and requires a prior-policy pass plus TTY approval for weakening/non-monotonic changes. Invalid present configs block. The approved full config also participates in the review cache key. | @@ -118,7 +164,7 @@ Everything lives under `.reviewgate/` as plain files: | `decisions/.jsonl` | no | the agent's accept/reject ledger | | `state.json` | no | loop FSM state | | `cache/reviews/.json` | no | per-diff cached verdicts | -| `audit/…` | no | sha256 hash-chained event log | +| `audit/…` | no | sha256 hash-chained event log plus day-partitioned `policy/*.json` traces | | `brain.{json,md}` | yes | committed per-repo memory (when Brain enabled) | | `ESCALATION.md` | no | written when a run escalates to the human | diff --git a/docs/superpowers/specs/2026-08-09-policy-accountability-trace-design.md b/docs/superpowers/specs/2026-08-09-policy-accountability-trace-design.md index f4b9d15..a9915a9 100644 --- a/docs/superpowers/specs/2026-08-09-policy-accountability-trace-design.md +++ b/docs/superpowers/specs/2026-08-09-policy-accountability-trace-design.md @@ -1,6 +1,34 @@ # Policy Accountability & Pruning — Slice 1: Policy Trace & Replay -_Written 2026-08-09. Status: approved by Markus; implementation planning in progress._ +_Written 2026-08-09. Status: Slice 1 core implementation is complete through `2355ac0`; the +authoritative replay help contract is corrected in `fa68dfa`, and final documentation is this +handoff commit._ + +## Implementation status — 2026-08-11 + +Slice 1 now implements the closed `reviewgate.policy-catalog.v1` inventory of 18 ablatable policy +passes and the two non-ablatable explanatory stages `aggregation.cluster` and `verdict.compute`. +Production uses the same predicates and precedence as before; the trace recorder, Audit binding, +Bench Matrix and Rig replay observe or internally ablate that path rather than maintaining a shadow +policy model. + +The implementation includes canonical content-addressed Audit/Bench artifacts, SHA-bound Rig state +and Cassette evidence, strict hash/order/catalog/state identities, branch-local multi-turn replay +and the 18-row production contract harness. Production trace instrumentation remains fail-open with +respect to the already-computed policy outcome; authoritative measurement rejects incomplete or +corrupt evidence with exit `4`. + +No pass was ranked, deleted or default-disabled in Slice 1. Lore remains additive and outside the 18 +demoters. Zero opportunities are not negative evidence, and stateful history passes require seeded +multi-turn sequences. Replay preserves `ImplicitOutcomeStore` writes in each branch, but current +production does not read that store into later policy inputs; its divergence therefore proves +branch-local persistence, not a current downstream review effect. + +Final verification at this boundary covered the 173-test focused policy suite, TypeScript, Biome, +the full 454-file suite (`3505 pass`, `12 skip`, `0 fail`), a fresh compiled build and the Bench, +Rig and Audit help surfaces. The required commands and durable acceptance semantics live in +`TEST_PLAN.md`; exact command output and the earlier pre-help baseline are retained in the ignored +Task-11 execution report. ## Context From 86cb319b90d85446784750cc9ffc5922c071ad39 Mon Sep 17 00:00:00 2001 From: Codevena Date: Tue, 11 Aug 2026 21:18:42 +0200 Subject: [PATCH 54/55] fix(rig): bind authoritative replay artifacts --- src/cassette/store.ts | 206 +++++++++++++++++- src/cli/commands/rig.ts | 35 +++- src/core/policy/replay-capture.ts | 33 ++- src/rig/driver.ts | 37 ++-- src/rig/harvest.ts | 64 ++++-- src/rig/policy-replay-state.ts | 1 + src/rig/replay.ts | 84 +++++++- src/schemas/policy-replay.ts | 40 +++- src/schemas/rig-result.ts | 34 +++ tests/unit/cassette-store.test.ts | 37 +++- tests/unit/policy-replay-capture.test.ts | 91 ++++++++ tests/unit/rig-ablate.test.ts | 252 ++++++++++++++++++++++- tests/unit/rig-driver.test.ts | 120 ++++++++++- 13 files changed, 970 insertions(+), 64 deletions(-) diff --git a/src/cassette/store.ts b/src/cassette/store.ts index ace8546..19a3d1e 100644 --- a/src/cassette/store.ts +++ b/src/cassette/store.ts @@ -1,15 +1,207 @@ -// src/cassette/store.ts -import { appendFileSync, existsSync, mkdirSync, readFileSync } from "node:fs"; -import { dirname } from "node:path"; +import { + constants, + type Stats, + closeSync, + existsSync, + fstatSync, + lstatSync, + mkdirSync, + openSync, + readFileSync, + readSync, + realpathSync, + writeSync, +} from "node:fs"; +import { dirname, isAbsolute, relative, resolve, sep } from "node:path"; import { type CassetteEntry, CassetteEntrySchema } from "../schemas/cassette.ts"; -// Append-only JSONL: a single appendFileSync of one line is atomic on POSIX, so the -// concurrent panel (Promise.allSettled) can record without a lock or lost entries. -// Single-process only — cross-process recording to one cassette is unsupported. +export const PRIVATE_CASSETTE_MAX_BYTES = 64 * 1024 * 1024; + +function isContained(root: string, candidate: string): boolean { + const rel = relative(root, candidate); + return rel === "" || (!rel.startsWith(`..${sep}`) && rel !== ".." && !isAbsolute(rel)); +} + +function assertPrivateRegular(stat: Stats, path: string): void { + if (!stat.isFile()) throw new Error(`cassette: ${path} is not a regular file`); + if (stat.nlink !== 1) throw new Error(`cassette: ${path} is a hardlink (nlink=${stat.nlink})`); + if ((stat.mode & 0o7777) !== 0o600) { + throw new Error(`cassette: ${path} mode must be exactly 0600`); + } +} + +function assertSamePathFile(path: string, opened: Stats): Stats { + const current = lstatSync(path); + if ( + current.isSymbolicLink() || + !current.isFile() || + current.nlink !== 1 || + (current.mode & 0o7777) !== 0o600 || + current.dev !== opened.dev || + current.ino !== opened.ino + ) { + throw new Error(`cassette: ${path} changed identity or is not a private 0600 file`); + } + return current; +} + +/** Exclusively create the Rig's empty private cassette before any agent is spawned. */ +export function createPrivateCassette(path: string): void { + const fd = openSync( + path, + constants.O_WRONLY | constants.O_CREAT | constants.O_EXCL | constants.O_NOFOLLOW, + 0o600, + ); + try { + const opened = fstatSync(fd); + assertPrivateRegular(opened, path); + assertSamePathFile(path, opened); + } finally { + closeSync(fd); + } +} + +function openPrivateCassette( + path: string, + flags: number, +): { + fd: number; + stat: Stats; + real: string; +} { + const before = lstatSync(path); + if (before.isSymbolicLink()) throw new Error(`cassette: refusing symlink ${path}`); + if (!before.isFile()) throw new Error(`cassette: ${path} is not a regular file`); + if (before.nlink !== 1) + throw new Error(`cassette: ${path} is a hardlink (nlink=${before.nlink})`); + if ((before.mode & 0o7777) !== 0o600) { + throw new Error(`cassette: ${path} mode must be exactly 0600`); + } + const real = realpathSync(path); + const fd = openSync(path, flags | constants.O_NOFOLLOW); + try { + const opened = fstatSync(fd); + assertPrivateRegular(opened, path); + if ( + opened.dev !== before.dev || + opened.ino !== before.ino || + opened.size !== before.size || + opened.mtimeMs !== before.mtimeMs || + opened.ctimeMs !== before.ctimeMs + ) { + throw new Error(`cassette: ${path} changed while opening`); + } + assertSamePathFile(path, opened); + if (realpathSync(path) !== real) throw new Error(`cassette: ${path} changed real path`); + return { fd, stat: opened, real }; + } catch (error) { + closeSync(fd); + throw error; + } +} + +function assertContainedPrivatePath(path: string, root: string, real: string): void { + const rootReal = realpathSync(root); + if (!isContained(resolve(root), resolve(path)) || !isContained(rootReal, real)) { + throw new Error(`cassette: ${path} is outside the measured repository`); + } +} + +/** Strict fstat-only size sampling used for per-turn byte ranges. */ +export function privateCassetteSize(path: string, root: string): number { + const opened = openPrivateCassette(path, constants.O_RDONLY); + try { + assertContainedPrivatePath(path, root, opened.real); + const after = fstatSync(opened.fd); + assertPrivateRegular(after, path); + assertSamePathFile(path, after); + if ( + after.dev !== opened.stat.dev || + after.ino !== opened.stat.ino || + after.size !== opened.stat.size || + after.mtimeMs !== opened.stat.mtimeMs || + after.ctimeMs !== opened.stat.ctimeMs || + realpathSync(path) !== opened.real + ) { + throw new Error(`cassette: ${path} changed while sampling size`); + } + return after.size; + } finally { + closeSync(opened.fd); + } +} + +/** Bounded, stable, single-buffer read used for both the authoritative hash and copy. */ +export function readPrivateCassette( + path: string, + root: string, + maxBytes = PRIVATE_CASSETTE_MAX_BYTES, +): Buffer { + const opened = openPrivateCassette(path, constants.O_RDONLY); + try { + assertContainedPrivatePath(path, root, opened.real); + if (opened.stat.size > maxBytes) throw new Error(`cassette: ${path} exceeds ${maxBytes} bytes`); + const bytes = Buffer.alloc(opened.stat.size); + let offset = 0; + while (offset < bytes.length) { + const count = readSync(opened.fd, bytes, offset, bytes.length - offset, offset); + if (count === 0) throw new Error(`cassette: ${path} changed while reading`); + offset += count; + } + const extra = Buffer.alloc(1); + if (readSync(opened.fd, extra, 0, 1, bytes.length) !== 0) { + throw new Error(`cassette: ${path} grew while reading`); + } + const after = fstatSync(opened.fd); + assertPrivateRegular(after, path); + assertSamePathFile(path, after); + if ( + after.dev !== opened.stat.dev || + after.ino !== opened.stat.ino || + after.size !== opened.stat.size || + after.mtimeMs !== opened.stat.mtimeMs || + after.ctimeMs !== opened.stat.ctimeMs || + realpathSync(path) !== opened.real + ) { + throw new Error(`cassette: ${path} changed while reading`); + } + return bytes; + } finally { + closeSync(opened.fd); + } +} + +// Append-only JSONL. The synchronous single write keeps concurrent in-process recorder +// completions from interleaving, while all identity checks happen before victim bytes move. export async function appendEntry(path: string, entry: CassetteEntry): Promise { const dir = dirname(path); if (!existsSync(dir)) mkdirSync(dir, { recursive: true }); - appendFileSync(path, `${JSON.stringify(entry)}\n`, { mode: 0o600 }); + if (!existsSync(path)) { + try { + createPrivateCassette(path); + } catch (error) { + if (!existsSync(path)) throw error; + } + } + const opened = openPrivateCassette(path, constants.O_WRONLY | constants.O_APPEND); + try { + const line = Buffer.from(`${JSON.stringify(entry)}\n`, "utf8"); + const written = writeSync(opened.fd, line); + if (written !== line.length) throw new Error(`cassette: short append to ${path}`); + const after = fstatSync(opened.fd); + assertPrivateRegular(after, path); + assertSamePathFile(path, after); + if ( + after.dev !== opened.stat.dev || + after.ino !== opened.stat.ino || + after.size !== opened.stat.size + line.length || + realpathSync(path) !== opened.real + ) { + throw new Error(`cassette: ${path} changed while appending`); + } + } finally { + closeSync(opened.fd); + } } export function loadCassette(path: string): CassetteEntry[] { diff --git a/src/cli/commands/rig.ts b/src/cli/commands/rig.ts index 4192075..b8dc5a1 100644 --- a/src/cli/commands/rig.ts +++ b/src/cli/commands/rig.ts @@ -14,6 +14,7 @@ import { realpathSync, } from "node:fs"; import { dirname, isAbsolute, relative, resolve, sep } from "node:path"; +import { createPrivateCassette } from "../../cassette/store.ts"; import { POLICY_CATALOG_VERSION, POLICY_PASS_IDS, @@ -76,10 +77,9 @@ export async function runRigRun(input: RigRunInput): Promise // nowhere, which is the exact failure the guard exists to prevent — discovered only after // a multi-turn run has already spent its quota. Check the destination is writable NOW. // - // BEST-EFFORT BY CONSTRUCTION, and deliberately so: the parent checks, but the CHILD - // writes the cassette (through the inherited env var), so the directory can still vanish - // in between. This is a pre-flight check that catches the common misconfiguration, not a - // guarantee — do not let a later reader mistake it for one (gate finding F-002). + // The parent validates the destination now and exclusively creates the private file after + // all other pre-flight checks. The child may only append through the no-follow Store path; + // Driver revalidates the same inode before every size/read/hash/copy operation. const cassettePath = input.cassetteEnv.slice("record:".length); // An absolute path is REQUIRED. `record:cassette.jsonl` has a dirname of ".", which // existsSync always accepts, and the file would then land relative to the SPAWNED @@ -111,12 +111,26 @@ export async function runRigRun(input: RigRunInput): Promise // rig's own results directory passed pre-flight and then produced twelve turns of nothing. // Cost three pilot attempts to find (field, 2026-08-05); mirror the recorder's rule here, // where it is still free to be wrong. - const repoPrefix = resolve(input.repoRoot) + sep; - if (!resolve(cassettePath).startsWith(repoPrefix)) { + const repoLexical = resolve(input.repoRoot); + const cassetteRelative = relative(repoLexical, resolve(cassettePath)); + const repoReal = realpathSync(input.repoRoot); + const cassetteDirReal = realpathSync(cassetteDir); + const cassetteRealRelative = relative(repoReal, cassetteDirReal); + const isRelativeInside = (value: string): boolean => + value === "" || (!value.startsWith(`..${sep}`) && value !== ".." && !isAbsolute(value)); + if (!isRelativeInside(cassetteRelative) || !isRelativeInside(cassetteRealRelative)) { throw new Error( `rig run: REVIEWGATE_CASSETTE must point INSIDE the repo under review (${input.repoRoot}), but got "${cassettePath}". The recorder refuses to write outside it, and it refuses during the gate's SETUP phase — so every turn would complete with the agent's edits made and no review at all. Put the cassette in the sandbox and copy it out after the run.`, ); } + try { + lstatSync(cassettePath); + throw new Error( + `rig run: private cassette ${cassettePath} already exists; refusing to follow or overwrite it`, + ); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; + } // Validate the script BEFORE spawning anything: a malformed script must stop the run // before it burns quota, not halfway through turn 7. const script = loadTurnScript(input.scriptPath); @@ -133,7 +147,6 @@ export async function runRigRun(input: RigRunInput): Promise `rig run: ${input.repoRoot} has ${dirty.length} uncommitted change(s). This run would let an agent edit that directory with acceptEdits, driven by prompts from ${input.scriptPath}. Point it at a throwaway repo, or pass allowDirtyRepo once you have read the script and accept what it will do.`, ); } - const repoReal = realpathSync(input.repoRoot); const output = resolve(input.outDir); mkdirSync(output, { recursive: true, mode: 0o700 }); const outputStat = lstatSync(output); @@ -165,6 +178,7 @@ export async function runRigRun(input: RigRunInput): Promise }); const sourceCommit = await gitHeadSha(repoReal); if (sourceCommit === null) throw new Error("rig run: could not resolve the source commit"); + createPrivateCassette(cassettePath); const emptyCassetteSha256 = new Bun.CryptoHasher("sha256").update("").digest("hex"); process.stderr.write( `rig run: an agent will EDIT ${input.repoRoot} with acceptEdits, for ${script.turns.length} scripted turn(s).\n`, @@ -237,7 +251,11 @@ export async function runRigAblate(input: RigAblateInput): Promise { `rig ablate: exact --layer must be one closed-catalog id: ${POLICY_PASS_IDS.join(", ")}`, ); } - const siblingManifest = resolve(dirname(input.resultPath), "manifest.json"); + const script = loadTurnScript(input.scriptPath); + const siblingManifest = resolve( + dirname(input.resultPath), + base.policyReplay.artifactBinding?.manifestRef ?? "manifest.json", + ); const manifestPath = existsSync(siblingManifest) ? siblingManifest : base.provenance.manifest_path; @@ -245,6 +263,7 @@ export async function runRigAblate(input: RigAblateInput): Promise { await replayPolicyAblations({ manifestPath, sourceRepoRoot: input.sourceRepoRoot ?? process.cwd(), + authority: { result: base, scriptId: script.id }, ...(passId === undefined ? {} : { passId: passId as PolicyPassId }), }), ); diff --git a/src/core/policy/replay-capture.ts b/src/core/policy/replay-capture.ts index 219ee96..6fb4b75 100644 --- a/src/core/policy/replay-capture.ts +++ b/src/core/policy/replay-capture.ts @@ -18,6 +18,8 @@ import { type PolicyReplayEnvelopeInput, PolicyReplayEnvelopeInputSchema, PolicyReplayEnvelopeSchema, + isFormalPolicyReplayUlid, + isTrustedPolicyReplayRunIdString, } from "../../schemas/policy-replay.ts"; import { writeFileIfAbsent } from "../../utils/atomic-write.ts"; import type { AggregateInput } from "../aggregator.ts"; @@ -131,12 +133,28 @@ function sanitizeString(value: string): { value: string; changed: boolean } { return { value, changed: false }; } -function sanitizeStrings(value: unknown): { value: unknown; changed: boolean } { +function sanitizeStrings( + value: unknown, + path: readonly (string | number)[], + trustedRunId: string | null, +): { value: unknown; changed: boolean } { + if ( + typeof value === "string" && + trustedRunId !== null && + isTrustedPolicyReplayRunIdString({ + value, + path, + runId: trustedRunId, + policyTraceRunId: trustedRunId, + }) + ) { + return { value, changed: false }; + } if (typeof value === "string") return sanitizeString(value); if (Array.isArray(value)) { let changed = false; - const out = value.map((entry) => { - const sanitized = sanitizeStrings(entry); + const out = value.map((entry, index) => { + const sanitized = sanitizeStrings(entry, [...path, index], trustedRunId); changed ||= sanitized.changed; return sanitized.value; }); @@ -146,7 +164,7 @@ function sanitizeStrings(value: unknown): { value: unknown; changed: boolean } { let changed = false; const out: Record = {}; for (const [key, entry] of Object.entries(value as Record)) { - const sanitized = sanitizeStrings(entry); + const sanitized = sanitizeStrings(entry, [...path, key], trustedRunId); changed ||= sanitized.changed; out[key] = sanitized.value; } @@ -227,7 +245,12 @@ export function sanitizePolicyReplayEnvelope( input: PolicyReplayEnvelopeInput, ): PolicyReplayEnvelope { const structural = PolicyReplayEnvelopeInputSchema.parse(input); - const sanitized = sanitizeStrings(structural); + const trustedRunId = + structural.run_id === structural.policy_trace.run_id && + isFormalPolicyReplayUlid(structural.run_id) + ? structural.run_id + : null; + const sanitized = sanitizeStrings(structural, [], trustedRunId); const value = sanitized.value as PolicyReplayEnvelopeInput; return PolicyReplayEnvelopeSchema.parse({ ...value, diff --git a/src/rig/driver.ts b/src/rig/driver.ts index 28d1e89..a9c033f 100644 --- a/src/rig/driver.ts +++ b/src/rig/driver.ts @@ -15,6 +15,7 @@ import { statSync, } from "node:fs"; import { join } from "node:path"; +import { privateCassetteSize, readPrivateCassette } from "../cassette/store.ts"; import type { RigManifest, RigManifestTurn } from "../schemas/rig-manifest.ts"; import { writeFileAtomic } from "../utils/atomic-write.ts"; import { collectDiff } from "../utils/git.ts"; @@ -375,6 +376,10 @@ export async function runDriver(opts: DriverOpts): Promise { const maxTurns = Math.max(1, Math.min(opts.maxTurns ?? script.turns.length, script.turns.length)); const quiesceTimeoutMs = opts.quiesceTimeoutMs ?? QUIESCE_TIMEOUT_MS; const manifestPath = join(opts.outDir, "manifest.json"); + if (opts.policyReplay !== undefined) { + // Authority starts before the agent: never let a hostile cassette path reach the child. + privateCassetteSize(opts.policyReplay.cassettePath, opts.repoRoot); + } const manifest: DriverRunManifest = { schema: "reviewgate.rig.manifest.v1", // Not a random id: a run is identified by the script it ran and when it started, so a @@ -382,7 +387,7 @@ export async function runDriver(opts: DriverOpts): Promise { runId: `${script.id}-${new Date().toISOString().replace(/[:.]/g, "-")}`, scriptId: script.id, outDir: opts.outDir, - cassettePath: recordingCassettePath(), + cassettePath: opts.policyReplay?.cassettePath ?? recordingCassettePath(), ...(opts.policyReplay === undefined ? {} : { policyReplay: opts.policyReplay.metadata }), turns: [], }; @@ -393,7 +398,10 @@ export async function runDriver(opts: DriverOpts): Promise { const startedAt = Date.now(); // Sampled BEFORE the agent runs: everything the cassette grows by during this turn is // this turn's reviewer traffic, which is what makes the entries addressable per turn. - const cassetteBefore = cassetteSize(manifest.cassettePath ?? null); + const cassetteBefore = + opts.policyReplay === undefined + ? cassetteSize(manifest.cassettePath ?? null) + : privateCassetteSize(opts.policyReplay.cassettePath, opts.repoRoot); const auditBytesBefore = auditBytes(opts.repoRoot); const replayBefore = opts.policyReplay === undefined ? null : policyReplayInventory(opts.policyReplay.sinkDir); @@ -426,7 +434,10 @@ export async function runDriver(opts: DriverOpts): Promise { const src = reviewgateDir(opts.repoRoot); if (existsSync(src)) await copyWithRetry(src, join(snapshotDir, ".reviewgate"), turn.index); - const cassetteAfter = cassetteSize(manifest.cassettePath ?? null); + const cassetteAfter = + opts.policyReplay === undefined + ? cassetteSize(manifest.cassettePath ?? null) + : privateCassetteSize(opts.policyReplay.cassettePath, opts.repoRoot); const diffBytes = await captureTurnDiff(opts.repoRoot, snapshotDir); // Checked BEFORE the snapshot is declared good: an unreviewed turn is not a slow turn, it // is a turn that produced no measurement, and the run must not quietly accumulate them. @@ -443,17 +454,17 @@ export async function runDriver(opts: DriverOpts): Promise { .filter(([ref]) => ref.endsWith(".json")) .map(([ref, sha256]) => ({ ref, sha256 })); if (manifest.policyReplay !== undefined && opts.policyReplay !== undefined) { - manifest.policyReplay.cassetteSha256 = sha256FileOrEmpty(opts.policyReplay.cassettePath); - try { - writeFileAtomic( - join(opts.outDir, manifest.policyReplay.cassetteRef), - readFileSync(opts.policyReplay.cassettePath, "utf8"), - { mode: 0o600 }, - ); - } catch { - // Missing/unreadable copy leaves the immutable hash bound but no artifact; - // authoritative replay reports the exact missing-cassette reason. + const cassetteBytes = readPrivateCassette(opts.policyReplay.cassettePath, opts.repoRoot); + const cassetteText = new TextDecoder("utf-8", { fatal: true }).decode(cassetteBytes); + if (!Buffer.from(cassetteText, "utf8").equals(cassetteBytes)) { + throw new Error("rig driver: cassette is not canonical UTF-8"); } + manifest.policyReplay.cassetteSha256 = createHash("sha256") + .update(cassetteBytes) + .digest("hex"); + writeFileAtomic(join(opts.outDir, manifest.policyReplay.cassetteRef), cassetteText, { + mode: 0o600, + }); } manifest.turns.push({ index: turn.index, diff --git a/src/rig/harvest.ts b/src/rig/harvest.ts index 8913c0c..2a45a06 100644 --- a/src/rig/harvest.ts +++ b/src/rig/harvest.ts @@ -28,9 +28,10 @@ // (`/reports/*-pending.json`), which is why that archiver is load-bearing rather // than redundant with the final `pending.json` (a turn that ends green overwrites the // report that caught the defect). +import { createHash } from "node:crypto"; import { existsSync, readFileSync, readdirSync } from "node:fs"; import { platform, release } from "node:os"; -import { dirname, join } from "node:path"; +import { basename, dirname, join } from "node:path"; import { matchesAnyTag } from "../bench/matcher.ts"; import { makeMetric, summarizeSpread } from "../bench/metrics.ts"; import { POLICY_CATALOG_VERSION, POLICY_PASS_IDS } from "../core/policy/catalog.ts"; @@ -522,9 +523,8 @@ function harvestTurn( } export function harvest(manifestPath: string, scriptPath: string): RigResult { - const manifest = RigManifestSchema.parse( - JSON.parse(readFileSync(manifestPath, "utf8")) as unknown, - ); + const manifestBytes = readFileSync(manifestPath); + const manifest = RigManifestSchema.parse(JSON.parse(manifestBytes.toString("utf8")) as unknown); const policyReplay = validateRigPolicyReplayArtifacts({ manifest, manifestPath }); const script = loadTurnScript(scriptPath); if (manifest.scriptId !== script.id) { @@ -636,6 +636,44 @@ export function harvest(manifestPath: string, scriptPath: string): RigResult { ); } + const resultPolicyReplay: RigResult["policyReplay"] = (() => { + if (policyReplay === null) { + return { + authoritative: false, + catalogVersion: null, + sourceCommit: null, + passIds: [], + reason: + "legacy run: no exact policy replay metadata; four-layer counts are non-authoritative", + }; + } + const metadata = manifest.policyReplay; + if (metadata === undefined) { + throw new Error("rig harvest: validated policy replay is missing manifest metadata"); + } + return { + authoritative: true, + catalogVersion: POLICY_CATALOG_VERSION, + sourceCommit: policyReplay.sourceCommit, + passIds: [...POLICY_PASS_IDS], + reason: null, + artifactBinding: { + manifestRef: basename(manifestPath), + manifestSha256: createHash("sha256").update(manifestBytes).digest("hex"), + scriptId: manifest.scriptId, + initialStateRef: metadata.initialStateRef, + initialStateSha256: metadata.initialStateSha256, + initialStateDigest: metadata.initialStateDigest, + cassetteRef: metadata.cassetteRef, + cassetteSha256: metadata.cassetteSha256, + turns: manifest.turns.map((turn) => ({ + index: turn.index, + traces: turn.policyReplay?.traces ?? [], + })), + }, + }; + })(); + const result: RigResult = { schema: "reviewgate.rig.result.v1", runId: manifest.runId, @@ -677,23 +715,7 @@ export function harvest(manifestPath: string, scriptPath: string): RigResult { }, suppression, }, - policyReplay: - policyReplay === null - ? { - authoritative: false, - catalogVersion: null, - sourceCommit: null, - passIds: [], - reason: - "legacy run: no exact policy replay metadata; four-layer counts are non-authoritative", - } - : { - authoritative: true, - catalogVersion: POLICY_CATALOG_VERSION, - sourceCommit: policyReplay.sourceCommit, - passIds: [...POLICY_PASS_IDS], - reason: null, - }, + policyReplay: resultPolicyReplay, warnings, }; // Validate what we are about to hand out: the null contracts in RigTurnRecordSchema are the diff --git a/src/rig/policy-replay-state.ts b/src/rig/policy-replay-state.ts index a268d06..cc58652 100644 --- a/src/rig/policy-replay-state.ts +++ b/src/rig/policy-replay-state.ts @@ -121,6 +121,7 @@ export type RigAuthorityInvalidity = | "cassette-hash-mismatch" | "invalid-cassette" | "source-state-alias" + | "result-manifest-mismatch" | "live-provider-call"; export class RigAuthorityError extends Error { diff --git a/src/rig/replay.ts b/src/rig/replay.ts index e2faa61..cf814ab 100644 --- a/src/rig/replay.ts +++ b/src/rig/replay.ts @@ -2,8 +2,9 @@ // Exact new runs replay captured policy inputs through production pass functions in isolated // checkouts. Legacy runs retain the older deterministic harvest/heuristic self-check, explicitly // non-authoritative for policy ablation rather than pretending missing opportunities were zero. +import { createHash } from "node:crypto"; import { existsSync, readFileSync } from "node:fs"; -import { join } from "node:path"; +import { basename, join } from "node:path"; import { canonicalJson } from "../audit/canonical.ts"; import { type AggregateInput, aggregate } from "../core/aggregator.ts"; import { validateFindingFacts } from "../core/fact-check.ts"; @@ -28,7 +29,7 @@ import { parseDeletedPaths } from "../diff/hunks.ts"; import { CassetteEntrySchema } from "../schemas/cassette.ts"; import type { PolicyReplayEnvelope } from "../schemas/policy-replay.ts"; import type { PolicyTrace } from "../schemas/policy-trace.ts"; -import { RigManifestSchema } from "../schemas/rig-manifest.ts"; +import { type RigManifest, RigManifestSchema } from "../schemas/rig-manifest.ts"; import type { RigResult } from "../schemas/rig-result.ts"; import { compareCodeUnits } from "../utils/compare.ts"; import { implicitOutcomesPath, knownFpPath, reputationJsonPath } from "../utils/paths.ts"; @@ -653,10 +654,19 @@ export async function replayPolicyAblations(input: { manifestPath: string; sourceRepoRoot: string; passId?: PolicyPassId; + authority?: { result: RigResult; scriptId: string }; }): Promise { - const manifest = RigManifestSchema.parse( - JSON.parse(readFileSync(input.manifestPath, "utf8")) as unknown, - ); + const manifestBytes = readFileSync(input.manifestPath); + const manifest = RigManifestSchema.parse(JSON.parse(manifestBytes.toString("utf8")) as unknown); + if (input.authority !== undefined) { + assertRigResultManifestBinding({ + result: input.authority.result, + manifest, + manifestPath: input.manifestPath, + manifestBytes, + scriptId: input.authority.scriptId, + }); + } const validated = validateRigPolicyReplayArtifacts({ manifest, manifestPath: input.manifestPath, @@ -726,6 +736,70 @@ export async function replayPolicyAblations(input: { return rows; } +function authorityMismatch(message: string): never { + throw new RigAuthorityError("result-manifest-mismatch", message); +} + +/** Bind the harvested result and selected script to the exact manifest before any replay. */ +export function assertRigResultManifestBinding(input: { + result: RigResult; + manifest: RigManifest; + manifestPath: string; + manifestBytes: Buffer; + scriptId: string; +}): void { + const statement = input.result.policyReplay; + const binding = statement?.artifactBinding; + const metadata = input.manifest.policyReplay; + if (statement?.authoritative !== true || binding === undefined || metadata === undefined) { + authorityMismatch("authoritative result is missing its content-addressed manifest binding"); + } + if ( + binding.manifestRef !== basename(input.manifestPath) || + binding.manifestSha256 !== createHash("sha256").update(input.manifestBytes).digest("hex") + ) { + authorityMismatch("selected manifest does not match the harvested manifest content address"); + } + if ( + input.result.runId !== input.manifest.runId || + input.result.provenance.run_id !== input.manifest.runId + ) { + authorityMismatch("result and manifest run ids differ"); + } + if ( + input.result.provenance.script_id !== input.manifest.scriptId || + binding.scriptId !== input.manifest.scriptId || + input.scriptId !== input.manifest.scriptId + ) { + authorityMismatch("selected script does not match the harvested manifest script id"); + } + if ( + statement.catalogVersion !== metadata.catalogVersion || + statement.sourceCommit !== metadata.sourceCommit || + binding.initialStateRef !== metadata.initialStateRef || + binding.initialStateSha256 !== metadata.initialStateSha256 || + binding.initialStateDigest !== metadata.initialStateDigest || + binding.cassetteRef !== metadata.cassetteRef || + binding.cassetteSha256 !== metadata.cassetteSha256 + ) { + authorityMismatch("result and manifest source, state, catalog, or cassette identities differ"); + } + const manifestInventory = input.manifest.turns.map((turn) => ({ + index: turn.index, + traces: turn.policyReplay?.traces ?? [], + })); + const resultInventory = input.result.turns.map((turn) => ({ + index: turn.index, + traces: turn.policyReplay?.traces.map(({ ref, sha256 }) => ({ ref, sha256 })) ?? [], + })); + if ( + canonicalJson(binding.turns) !== canonicalJson(manifestInventory) || + canonicalJson(binding.turns) !== canonicalJson(resultInventory) + ) { + authorityMismatch("result and manifest turn/trace inventories differ"); + } +} + export function renderPolicyAblationRows(rows: RigPolicyAblationRow[]): string { const lines = ["Reviewgate rig — exact policy ablation (closed catalog)", ""]; for (const row of rows) { diff --git a/src/schemas/policy-replay.ts b/src/schemas/policy-replay.ts index 092c5a5..9145275 100644 --- a/src/schemas/policy-replay.ts +++ b/src/schemas/policy-replay.ts @@ -1,4 +1,5 @@ import { createHash } from "node:crypto"; +import { decodeTime } from "ulid"; import { z } from "zod"; import { POLICY_CATALOG_VERSION, @@ -308,11 +309,48 @@ function visitStrings( walk(value, []); } +export function isFormalPolicyReplayUlid(value: string): boolean { + if (!/^[0-9A-HJKMNP-TV-Z]{26}$/.test(value)) return false; + try { + decodeTime(value); + return true; + } catch { + return false; + } +} + +export function isTrustedPolicyReplayRunIdString(input: { + value: string; + path: readonly (string | number)[]; + runId: string; + policyTraceRunId: string; +}): boolean { + if ( + input.runId !== input.policyTraceRunId || + input.value !== input.runId || + !isFormalPolicyReplayUlid(input.runId) + ) { + return false; + } + return ( + (input.path.length === 1 && input.path[0] === "run_id") || + (input.path.length === 2 && input.path[0] === "policy_trace" && input.path[1] === "run_id") + ); +} + export const PolicyReplayEnvelopeSchema = PolicyReplayEnvelopeBaseSchema.superRefine( (value, ctx) => { if (value.lossless) { visitStrings(value, (stringValue, path) => { - if (!isAuthoritativeThrowableString(stringValue)) { + if ( + !isAuthoritativeThrowableString(stringValue) && + !isTrustedPolicyReplayRunIdString({ + value: stringValue, + path, + runId: value.run_id, + policyTraceRunId: value.policy_trace.run_id, + }) + ) { ctx.addIssue({ code: z.ZodIssueCode.custom, path, diff --git a/src/schemas/rig-result.ts b/src/schemas/rig-result.ts index 46a272d..0b5c20b 100644 --- a/src/schemas/rig-result.ts +++ b/src/schemas/rig-result.ts @@ -16,6 +16,38 @@ import { MetricSchema, SpreadStatSchema } from "./bench-result.ts"; import { FindingSchema } from "./finding.ts"; import { CriticInfoSchema } from "./pending-report.ts"; +const Sha256Schema = z.string().regex(/^[0-9a-f]{64}$/); +const PolicyReplayTraceRefSchema = z + .string() + .regex(/^[0-9a-f]{12}-i(?:0|[1-9]\d*)-[0-9a-f]{12}\.json$/); + +/** + * Immutable identity copied from the exact manifest used by the harvester. Optional keeps + * pre-binding results parseable, but authoritative replay refuses to use one without it. + */ +export const RigPolicyReplayArtifactBindingSchema = z + .object({ + manifestRef: z.string().regex(/^[^/\\]+$/), + manifestSha256: Sha256Schema, + scriptId: z.string().min(1), + initialStateRef: z.string().regex(/^policy-state\/[0-9a-f]{64}\.json$/), + initialStateSha256: Sha256Schema, + initialStateDigest: Sha256Schema, + cassetteRef: z.literal("cassette.jsonl"), + cassetteSha256: Sha256Schema, + turns: z.array( + z + .object({ + index: z.number().int().positive(), + traces: z.array( + z.object({ ref: PolicyReplayTraceRefSchema, sha256: Sha256Schema }).strict(), + ), + }) + .strict(), + ), + }) + .strict(); + /** * M6 — which suppression layer removed or demoted how many findings. * @@ -281,6 +313,7 @@ export const RigResultSchema = z .nullable(), passIds: z.array(z.enum(POLICY_PASS_IDS)), reason: z.string().nullable(), + artifactBinding: RigPolicyReplayArtifactBindingSchema.optional(), }) .strict() .superRefine((value, ctx) => { @@ -320,3 +353,4 @@ export type RigSlope = z.infer; export type RigMetrics = z.infer; export type RigProvenance = z.infer; export type RigResult = z.infer; +export type RigPolicyReplayArtifactBinding = z.infer; diff --git a/tests/unit/cassette-store.test.ts b/tests/unit/cassette-store.test.ts index 9d1ef27..fe28a43 100644 --- a/tests/unit/cassette-store.test.ts +++ b/tests/unit/cassette-store.test.ts @@ -1,6 +1,14 @@ // tests/unit/cassette-store.test.ts import { describe, expect, it } from "bun:test"; -import { mkdtempSync, readFileSync, writeFileSync } from "node:fs"; +import { + chmodSync, + linkSync, + mkdirSync, + mkdtempSync, + readFileSync, + symlinkSync, + writeFileSync, +} from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { appendEntry, cassetteFromEnv, loadCassette } from "../../src/cassette/store.ts"; @@ -37,6 +45,33 @@ describe("cassette store (JSONL)", () => { expect(loaded.map((e) => e.key)).toEqual(["a", "b"]); }); + it("never follows a cassette symlink or hardlink when appending", async () => { + const dir = mkdtempSync(join(tmpdir(), "rg-cas-links-")); + const victim = join(dir, "victim.txt"); + writeFileSync(victim, "host secret", { mode: 0o600 }); + const symlink = join(dir, "symlink.jsonl"); + const hardlink = join(dir, "hardlink.jsonl"); + symlinkSync(victim, symlink); + linkSync(victim, hardlink); + + await expect(appendEntry(symlink, entry("symlink"))).rejects.toThrow(); + await expect(appendEntry(hardlink, entry("hardlink"))).rejects.toThrow(/hardlink|link/i); + expect(readFileSync(victim, "utf8")).toBe("host secret"); + }); + + it("requires a private 0600 regular cassette before appending", async () => { + const dir = mkdtempSync(join(tmpdir(), "rg-cas-mode-")); + const publicFile = join(dir, "public.jsonl"); + writeFileSync(publicFile, "", { mode: 0o600 }); + chmodSync(publicFile, 0o644); + await expect(appendEntry(publicFile, entry("public"))).rejects.toThrow(/0600|mode/i); + + const special = join(dir, "directory.jsonl"); + mkdirSync(special); + await expect(appendEntry(special, entry("special"))).rejects.toThrow(); + expect(readFileSync(publicFile, "utf8")).toBe(""); + }); + it("skips a malformed line without aborting", () => { const dir = mkdtempSync(join(tmpdir(), "rg-cas2-")); const p = join(dir, "c.jsonl"); diff --git a/tests/unit/policy-replay-capture.test.ts b/tests/unit/policy-replay-capture.test.ts index fc6631e..0bdb849 100644 --- a/tests/unit/policy-replay-capture.test.ts +++ b/tests/unit/policy-replay-capture.test.ts @@ -285,6 +285,97 @@ describe("policy replay envelope schema", () => { }); describe("policy replay capture", () => { + test("preserves only the trusted matching Gate ULID identity despite its entropy", () => { + const measuredRepoRoot = gitRepo(); + const runId = "01KZS1PT1A6VXW9VDGBNCTJ8KV"; + const withRunId = ( + message = "A real issue", + envelopeRunId = runId, + traceRunId = envelopeRunId, + ): PolicyReplayEnvelope => { + const candidate = envelope(); + candidate.run_id = envelopeRunId; + candidate.policy_trace.run_id = traceRunId; + const prePolicyFinding = candidate.pre_policy_findings[0]; + const aggregateFinding = candidate.aggregate.findings[0]; + const finalFinding = candidate.policy_final_findings[0]; + if ( + prePolicyFinding === undefined || + aggregateFinding === undefined || + finalFinding === undefined + ) { + throw new Error("ULID fixture is missing its finding"); + } + prePolicyFinding.message = message; + aggregateFinding.message = message; + finalFinding.message = message; + candidate.response_calls = candidate.response_calls.map((call) => ({ + ...call, + call_id: policyReplayCallId({ + runId: envelopeRunId, + iter: candidate.iter, + kind: call.kind, + provider: call.provider, + method: call.method, + key: call.key, + promptSha256: call.prompt_sha256, + ordinal: call.ordinal, + slot: call.slot, + attempt: call.attempt, + occurrence: call.occurrence, + }), + })); + return candidate; + }; + + const trustedSink = mkdtempSync(join(tmpdir(), "rg-policy-ulid-trusted-")); + const trusted = capturePolicyReplayEnvelope({ + sinkDir: trustedSink, + measuredRepoRoot, + envelope: withRunId(), + }); + expect(trusted).toMatchObject({ status: "complete" }); + if (trusted.status !== "complete") throw new Error("capture failed"); + expect(trusted.envelope.lossless).toBe(true); + expect(trusted.envelope.run_id).toBe(runId); + expect(trusted.envelope.policy_trace.run_id).toBe(runId); + expect( + verifyPolicyReplayEnvelope({ + sinkDir: trustedSink, + ref: trusted.ref, + sha256: trusted.sha256, + authoritative: true, + }).ok, + ).toBe(true); + + const untrustedSink = mkdtempSync(join(tmpdir(), "rg-policy-ulid-untrusted-")); + const untrusted = capturePolicyReplayEnvelope({ + sinkDir: untrustedSink, + measuredRepoRoot, + envelope: withRunId(runId), + }); + expect(untrusted.status).toBe("complete"); + if (untrusted.status !== "complete") throw new Error("capture failed"); + expect(untrusted.envelope.lossless).toBe(false); + expect(untrusted.envelope.pre_policy_findings[0]?.message).not.toBe(runId); + + const invalidUlid = `8${runId.slice(1)}`; + expect( + capturePolicyReplayEnvelope({ + sinkDir: mkdtempSync(join(tmpdir(), "rg-policy-ulid-invalid-")), + measuredRepoRoot, + envelope: withRunId("A real issue", invalidUlid), + }), + ).toEqual({ status: "error", reason: "invalid-envelope" }); + expect( + capturePolicyReplayEnvelope({ + sinkDir: mkdtempSync(join(tmpdir(), "rg-policy-ulid-mismatch-")), + measuredRepoRoot, + envelope: withRunId("A real issue", runId, "01KZS1PT1A6VXW9VDGBNCTJ8KW"), + }), + ).toEqual({ status: "error", reason: "invalid-envelope" }); + }); + test("writes canonical mode-0600 data outside the measured repo and verifies its identity", () => { const measuredRepoRoot = gitRepo(); const outputRoot = mkdtempSync(join(tmpdir(), "rg-policy-replay-output-")); diff --git a/tests/unit/rig-ablate.test.ts b/tests/unit/rig-ablate.test.ts index 8b7a4a4..f026260 100644 --- a/tests/unit/rig-ablate.test.ts +++ b/tests/unit/rig-ablate.test.ts @@ -6,9 +6,15 @@ import { makeMetric, summarizeSpread } from "../../src/bench/metrics.ts"; import { RigLayerSelectorError, runRigAblate } from "../../src/cli/commands/rig.ts"; import { POLICY_CATALOG_VERSION, POLICY_PASS_IDS } from "../../src/core/policy/catalog.ts"; import { ablate, renderAblationMatrix } from "../../src/rig/ablate.ts"; -import { renderPolicyAblationRows } from "../../src/rig/replay.ts"; +import { RigAuthorityError } from "../../src/rig/policy-replay-state.ts"; +import { assertRigResultManifestBinding, renderPolicyAblationRows } from "../../src/rig/replay.ts"; import type { Finding } from "../../src/schemas/finding.ts"; -import type { RigResult, RigTurnRecord } from "../../src/schemas/rig-result.ts"; +import { RigManifestSchema } from "../../src/schemas/rig-manifest.ts"; +import { + type RigResult, + RigResultSchema, + type RigTurnRecord, +} from "../../src/schemas/rig-result.ts"; const ZERO = { critic: 0, reputation: 0, fp_ledger: 0, lore: 0 }; @@ -91,7 +97,249 @@ function result(turns: RigTurnRecord[], over: Partial = {}): RigResul const NO_TAGS = new Map(); +function exactManifest(runId: string, scriptId: string, sourceCommit: string) { + return { + schema: "reviewgate.rig.manifest.v1", + runId, + scriptId, + outDir: "/unused", + cassettePath: null, + policyReplay: { + catalogVersion: POLICY_CATALOG_VERSION, + sourceCommit, + initialStateRef: `policy-state/${"1".repeat(64)}.json`, + initialStateSha256: "2".repeat(64), + initialStateDigest: "3".repeat(64), + cassetteSha256: "4".repeat(64), + cassetteRef: "cassette.jsonl", + captureDir: "policy-replay", + }, + turns: [ + { + index: 1, + snapshotDir: "/unused/turn-01", + agentExitCode: 0, + wallMs: 1, + policyReplay: { + status: "complete", + traces: [{ ref: `${"a".repeat(12)}-i1-${"b".repeat(12)}.json`, sha256: "5".repeat(64) }], + }, + }, + ], + }; +} + +function exactResultForManifest( + manifest: ReturnType, + manifestSha256: string, +) { + const firstTrace = manifest.turns[0]?.policyReplay.traces[0]; + if (firstTrace === undefined) throw new Error("exact result fixture is missing its trace"); + const base = result([ + turn({ + index: 1, + policyReplay: { + status: "complete", + traces: [ + { + ...firstTrace, + runId: manifest.runId, + iter: 1, + stateSha256: manifest.policyReplay.initialStateDigest, + lossless: true, + }, + ], + reason: null, + }, + }), + ]); + return { + ...base, + runId: manifest.runId, + provenance: { + ...base.provenance, + run_id: manifest.runId, + script_id: manifest.scriptId, + manifest_path: "/unused/manifest-a.json", + }, + policyReplay: { + authoritative: true, + catalogVersion: POLICY_CATALOG_VERSION, + sourceCommit: manifest.policyReplay.sourceCommit, + passIds: [...POLICY_PASS_IDS], + reason: null, + artifactBinding: { + manifestRef: "manifest.json", + manifestSha256, + scriptId: manifest.scriptId, + initialStateRef: manifest.policyReplay.initialStateRef, + initialStateSha256: manifest.policyReplay.initialStateSha256, + initialStateDigest: manifest.policyReplay.initialStateDigest, + cassetteRef: manifest.policyReplay.cassetteRef, + cassetteSha256: manifest.policyReplay.cassetteSha256, + turns: manifest.turns.map((entry) => ({ + index: entry.index, + traces: entry.policyReplay.traces, + })), + }, + }, + }; +} + describe("rig ablate", () => { + test("binds every authoritative source/state/cassette/turn inventory field", () => { + const manifestValue = exactManifest("run-a", "script-a", "c".repeat(40)); + const manifestBytes = Buffer.from(`${JSON.stringify(manifestValue)}\n`); + const manifest = RigManifestSchema.parse(manifestValue); + const validResult = RigResultSchema.parse( + exactResultForManifest( + manifestValue, + new Bun.CryptoHasher("sha256").update(manifestBytes).digest("hex"), + ), + ); + const validate = (candidate: RigResult): void => + assertRigResultManifestBinding({ + result: candidate, + manifest, + manifestPath: "/artifact/manifest.json", + manifestBytes, + scriptId: "script-a", + }); + expect(() => validate(validResult)).not.toThrow(); + + const authority = (candidate: RigResult) => { + const statement = candidate.policyReplay; + const binding = statement?.artifactBinding; + const firstTrace = binding?.turns[0]?.traces[0]; + if (statement === undefined || binding === undefined || firstTrace === undefined) { + throw new Error("exact result fixture is missing authority fields"); + } + return { statement, binding, firstTrace }; + }; + + const mutations: Array<(candidate: RigResult) => void> = [ + (candidate) => { + candidate.runId = "run-b"; + }, + (candidate) => { + authority(candidate).statement.sourceCommit = "d".repeat(40); + }, + (candidate) => { + authority(candidate).binding.initialStateSha256 = "6".repeat(64); + }, + (candidate) => { + authority(candidate).binding.cassetteSha256 = "7".repeat(64); + }, + (candidate) => { + authority(candidate).firstTrace.sha256 = "8".repeat(64); + }, + ]; + for (const mutate of mutations) { + const candidate = structuredClone(validResult); + mutate(candidate); + try { + validate(candidate); + throw new Error("expected authority rejection"); + } catch (error) { + expect(error).toBeInstanceOf(RigAuthorityError); + expect((error as RigAuthorityError).code).toBe("result-manifest-mismatch"); + } + } + }); + + test("rejects an authoritative result combined with a different valid run manifest", async () => { + const root = mkdtempSync(join(tmpdir(), "rg-exact-bind-")); + const sourceCommit = "c".repeat(40); + const manifestA = exactManifest("run-a", "script-a", sourceCommit); + const manifestB = exactManifest("run-b", "script-b", sourceCommit); + const manifestABytes = `${JSON.stringify(manifestA)}\n`; + const manifestBBytes = `${JSON.stringify(manifestB)}\n`; + const manifestAPath = join(root, "manifest-a.json"); + writeFileSync(manifestAPath, manifestABytes); + writeFileSync(join(root, "manifest.json"), manifestBBytes); + const exact = exactResultForManifest( + manifestA, + new Bun.CryptoHasher("sha256").update(manifestABytes).digest("hex"), + ); + exact.provenance.manifest_path = manifestAPath; + const resultPath = join(root, "result.json"); + writeFileSync(resultPath, JSON.stringify(exact)); + const scriptPath = join(root, "script-a.json"); + writeFileSync( + scriptPath, + JSON.stringify({ + schema: "reviewgate.rig.turn-script.v1", + id: "script-a", + turns: [{ index: 1, prompt: "safe", seeded: null }], + }), + ); + + try { + await runRigAblate({ resultPath, scriptPath, sourceRepoRoot: root }); + throw new Error("expected authority rejection"); + } catch (error) { + expect(error).toBeInstanceOf(RigAuthorityError); + expect((error as RigAuthorityError).code).toBe("result-manifest-mismatch"); + } + const child = Bun.spawn( + [ + "bun", + "run", + "src/cli/index.ts", + "rig", + "ablate", + "--result", + resultPath, + "--script", + scriptPath, + ], + { + cwd: join(import.meta.dir, "..", ".."), + stdout: "pipe", + stderr: "pipe", + }, + ); + const [exitCode, stdout, stderr] = await Promise.all([ + child.exited, + new Response(child.stdout).text(), + new Response(child.stderr).text(), + ]); + expect(exitCode).toBe(4); + expect(stderr).toContain("result-manifest-mismatch"); + expect(stdout).not.toContain("exact policy ablation"); + }); + + test("binds authoritative --script to the harvested script identity", async () => { + const root = mkdtempSync(join(tmpdir(), "rg-exact-script-bind-")); + const manifest = exactManifest("run-a", "script-a", "c".repeat(40)); + const manifestBytes = `${JSON.stringify(manifest)}\n`; + writeFileSync(join(root, "manifest.json"), manifestBytes); + const exact = exactResultForManifest( + manifest, + new Bun.CryptoHasher("sha256").update(manifestBytes).digest("hex"), + ); + exact.provenance.manifest_path = join(root, "manifest.json"); + const resultPath = join(root, "result.json"); + writeFileSync(resultPath, JSON.stringify(exact)); + const scriptPath = join(root, "script-b.json"); + writeFileSync( + scriptPath, + JSON.stringify({ + schema: "reviewgate.rig.turn-script.v1", + id: "script-b", + turns: [{ index: 1, prompt: "safe", seeded: null }], + }), + ); + + try { + await runRigAblate({ resultPath, scriptPath, sourceRepoRoot: root }); + throw new Error("expected authority rejection"); + } catch (error) { + expect(error).toBeInstanceOf(RigAuthorityError); + expect((error as RigAuthorityError).code).toBe("result-manifest-mismatch"); + } + }); + test("keeps exact catalog selectors and legacy aliases in their own result modes", async () => { const root = mkdtempSync(join(tmpdir(), "rg-layer-selector-")); const scriptPath = join(root, "script.json"); diff --git a/tests/unit/rig-driver.test.ts b/tests/unit/rig-driver.test.ts index e34206a..9d738d4 100644 --- a/tests/unit/rig-driver.test.ts +++ b/tests/unit/rig-driver.test.ts @@ -6,6 +6,7 @@ import { mkdtempSync, readFileSync, readdirSync, + symlinkSync, writeFileSync, } from "node:fs"; import { tmpdir } from "node:os"; @@ -57,6 +58,82 @@ const appendingAgent = (root: string) => (prompt: string) => [ ]; describe("rig driver", () => { + test("rejects a symlinked authoritative cassette before the agent and copies no host bytes", async () => { + const { root, scriptPath } = sandbox(1); + const outDir = mkdtempSync(join(tmpdir(), "rg-rig-cassette-out-")); + const sinkDir = join(outDir, "policy-replay"); + mkdirSync(sinkDir, { mode: 0o700 }); + const victim = join(mkdtempSync(join(tmpdir(), "rg-rig-victim-")), "secret.txt"); + writeFileSync(victim, "HOST-SECRET", { mode: 0o600 }); + const cassettePath = join(root, "cassette.jsonl"); + symlinkSync(victim, cassettePath); + const agentMarker = join(root, "agent-ran"); + + await expect( + runDriver({ + scriptPath, + outDir, + repoRoot: root, + agentCmd: () => ["bash", "-c", 'printf ran > "$1"', "agent", agentMarker], + maxTurns: 1, + policyReplay: { + sinkDir, + cassettePath, + metadata: { + catalogVersion: POLICY_CATALOG_VERSION, + sourceCommit: "a".repeat(40), + initialStateRef: `policy-state/${"b".repeat(64)}.json`, + initialStateSha256: "c".repeat(64), + initialStateDigest: "d".repeat(64), + cassetteSha256: "e".repeat(64), + cassetteRef: "cassette.jsonl", + captureDir: "policy-replay", + }, + }, + }), + ).rejects.toThrow(/cassette|symlink/i); + expect(existsSync(agentMarker)).toBe(false); + expect(readFileSync(victim, "utf8")).toBe("HOST-SECRET"); + expect(existsSync(join(outDir, "cassette.jsonl"))).toBe(false); + }); + + test("rejects a cassette swapped to a host symlink before the stable read", async () => { + const { root, scriptPath } = sandbox(1); + const outDir = mkdtempSync(join(tmpdir(), "rg-rig-cassette-swap-out-")); + const sinkDir = join(outDir, "policy-replay"); + mkdirSync(sinkDir, { mode: 0o700 }); + const victim = join(mkdtempSync(join(tmpdir(), "rg-rig-swap-victim-")), "secret.txt"); + writeFileSync(victim, "HOST-SECRET", { mode: 0o600 }); + const cassettePath = join(root, "cassette.jsonl"); + writeFileSync(cassettePath, "", { mode: 0o600 }); + + await expect( + runDriver({ + scriptPath, + outDir, + repoRoot: root, + agentCmd: () => ["bash", "-c", 'rm "$1"; ln -s "$2" "$1"', "agent", cassettePath, victim], + maxTurns: 1, + policyReplay: { + sinkDir, + cassettePath, + metadata: { + catalogVersion: POLICY_CATALOG_VERSION, + sourceCommit: "a".repeat(40), + initialStateRef: `policy-state/${"b".repeat(64)}.json`, + initialStateSha256: "c".repeat(64), + initialStateDigest: "d".repeat(64), + cassetteSha256: "e".repeat(64), + cassetteRef: "cassette.jsonl", + captureDir: "policy-replay", + }, + }, + }), + ).rejects.toThrow(/cassette|symlink/i); + expect(readFileSync(victim, "utf8")).toBe("HOST-SECRET"); + expect(existsSync(join(outDir, "cassette.jsonl"))).toBe(false); + }); + test("exports only the replay sink and records immutable trace/state identity outside the repo", async () => { const { root, scriptPath } = sandbox(1); execFileSync("git", ["init", "-q", "."], { cwd: root }); @@ -118,10 +195,25 @@ describe("rig driver", () => { refs[1], refs[0], ]); - expect(existsSync(join(outDir, "cassette.jsonl"))).toBe(true); + const copiedCassette = readFileSync(join(outDir, "cassette.jsonl")); + const copiedHash = new Bun.CryptoHasher("sha256").update(copiedCassette).digest("hex"); + expect(copiedHash).toBe(manifest.policyReplay?.cassetteSha256 ?? ""); expect(existsSync(join(root, ".reviewgate", "policy-replay"))).toBe(false); }); + test("hashes and copies the same single stable cassette buffer", () => { + const source = readFileSync( + join(import.meta.dir, "..", "..", "src", "rig", "driver.ts"), + "utf8", + ); + expect(source.match(/readPrivateCassette\(/g)).toHaveLength(1); + expect(source).toContain(".update(cassetteBytes)"); + expect(source).toContain( + "writeFileAtomic(join(opts.outDir, manifest.policyReplay.cassetteRef), cassetteText", + ); + expect(source).not.toContain("readFileSync(opts.policyReplay.cassettePath"); + }); + test("carries a capture overflow marker into the authoritative turn status", async () => { const { root, scriptPath } = sandbox(1); execFileSync("git", ["init", "-q", "."], { cwd: root }); @@ -518,6 +610,32 @@ describe("rig run cassette destination", () => { /does not exist/, ); }); + + test("rejects an existing cassette symlink before any run can follow it", async () => { + const repo = mkdtempSync(join(tmpdir(), "rg-rig-link-repo-")); + execFileSync("git", ["init", "-q", "."], { cwd: repo }); + writeFileSync(join(repo, "tracked.txt"), "safe\n"); + execFileSync("git", ["add", "tracked.txt"], { cwd: repo }); + execFileSync( + "git", + ["-c", "user.email=rig@example.invalid", "-c", "user.name=rig", "commit", "-qm", "init"], + { cwd: repo }, + ); + const victim = join(mkdtempSync(join(tmpdir(), "rg-rig-link-victim-")), "secret"); + writeFileSync(victim, "HOST-SECRET", { mode: 0o600 }); + const cassettePath = join(repo, "cassette.jsonl"); + symlinkSync(victim, cassettePath); + + await expect( + runRigRun({ + scriptPath: join(import.meta.dir, "..", "..", "rig", "scripts", "pilot-01.json"), + outDir: mkdtempSync(join(tmpdir(), "rg-rig-link-out-")), + repoRoot: repo, + cassetteEnv: `record:${cassettePath}`, + }), + ).rejects.toThrow(/cassette.*already exists|private cassette|symlink/i); + expect(readFileSync(victim, "utf8")).toBe("HOST-SECRET"); + }); }); describe("rig run repo guards", () => { From aaa97fc4ff73b21b64e687779da6e33470c4a706 Mon Sep 17 00:00:00 2001 From: Codevena Date: Tue, 11 Aug 2026 21:32:09 +0200 Subject: [PATCH 55/55] docs: finalize policy trace verification --- NEXT_SESSION.md | 15 ++++++++------- TEST_PLAN.md | 3 ++- docs/architecture.md | 9 +++++---- ...26-08-09-policy-accountability-trace-design.md | 14 ++++++-------- 4 files changed, 21 insertions(+), 20 deletions(-) diff --git a/NEXT_SESSION.md b/NEXT_SESSION.md index e65e19a..c48b6ce 100644 --- a/NEXT_SESSION.md +++ b/NEXT_SESSION.md @@ -5,9 +5,9 @@ _Last updated: 2026-08-11. Supersedes all earlier content._ ## One-line state **Policy Accountability & Pruning Slice 1 is implemented on -`feat/policy-accountability-trace` through core commit `2355ac0` and CLI-help contract commit -`fa68dfa`; this documentation commit closes its handoff and verification. The next milestone is -Slice 2 measurement and pruning design—not pass deletion.** +`feat/policy-accountability-trace` through final implementation/security commit `86cb319`; this +documentation commit closes its handoff and verification. The next milestone is Slice 2 +measurement and pruning design—not pass deletion.** ## Checkout and publication state @@ -15,7 +15,7 @@ Slice 2 measurement and pruning design—not pass deletion.** |---|---| | branch | `feat/policy-accountability-trace` | | isolated worktree | `/Users/markus/.config/superpowers/worktrees/reviewgate/policy-accountability-trace` | -| implementation boundary | `9bc72c1..fa68dfa` (Slice-1 code/tests and authoritative replay help; this handoff is the following documentation commit) | +| implementation boundary | `9bc72c1..86cb319` (Slice-1 code/tests, authoritative replay help and final artifact-binding hardening; this handoff is the following documentation commit) | | pushed? | **NO**—do not push without Markus's explicit permission | | main checkout | out of scope; preserve its foreign `.reviewgate/lore/approvals.jsonl` | @@ -29,9 +29,10 @@ Slice 2 measurement and pruning design—not pass deletion.** errors/overflow remain fail-open with respect to the already-computed policy verdict. - Exact authoritative Bench pairing: only the baseline calls live providers; variants consume the same captured logical responses and change only the internal ablation set. -- Exact Rig capture/replay: Cassette call identities, response order, source commit/diff and state - digests are bound; persistent baseline/counterfactual scratch branches are isolated from the - measured checkout and from each other. +- Exact Rig capture/replay: the result is SHA-bound to its Manifest, script, source, initial state, + Cassette and complete turn/trace inventory; private mode-`0600` Cassette call identities, + response order, source commit/diff and state digests are bound. Persistent + baseline/counterfactual scratch branches are isolated from the measured checkout and each other. - An 18-row production contract harness, four-class offline replay and ten mutation-proven accountability boundaries. diff --git a/TEST_PLAN.md b/TEST_PLAN.md index d1133db..48dd4a4 100644 --- a/TEST_PLAN.md +++ b/TEST_PLAN.md @@ -51,7 +51,8 @@ The suite must prove: - trace-on/off leaves findings, legacy markers, Markdown, counts and verdict byte-equivalent after optional telemetry is removed; - Audit/Bench policy artifacts are canonical and content-addressed; Rig state and Cassette evidence - are mode-`0600`, contained and bound by exact content hashes/identities; + are mode-`0600`, contained and bound by exact content hashes/identities; `result.json` is bound to + the exact Manifest, script, source, initial state, Cassette and complete turn/trace inventory; - Bench uses one live baseline and exact captured-response replay for internally ablated variants; - Rig replays exact calls in persistent, isolated baseline/counterfactual branches without live provider or network calls and without production-state writes; diff --git a/docs/architecture.md b/docs/architecture.md index 9bc3a4a..e3a3f94 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -95,10 +95,11 @@ Exact ablation is internal to measurement code: reviewer/preflight/completion results, then replays each variant through the same policy path with only `policyAblations` changed. The matrix directory contains content-addressed `artifacts/` for results, response manifests, policy traces and their trace-set binding. -- A traced Rig run binds `manifest.json`, `cassette.jsonl`, `policy-replay/` envelopes, exact diffs - and content-addressed policy-state snapshots. Replay joins responses by stable logical call ID - and ordered hashes, then runs baseline/counterfactual in separate persistent branch-local scratch - checkouts. Production state is never a replay target. +- A traced Rig run binds `result.json` to the exact SHA-addressed Manifest, script, source commit, + initial state, private mode-`0600` Cassette and complete turn/trace inventory. The Cassette is + read once through a contained, stable no-follow file descriptor; replay joins responses by stable + logical call ID and ordered hashes, then runs baseline/counterfactual in separate persistent + branch-local scratch checkouts. Production state is never a replay target. Bench/Rig treat a missing, corrupt, incomplete, cross-catalog or identity-mismatched evidence set as invalid measurement and exit `4`; no absent counter is interpreted as zero. Stateful history diff --git a/docs/superpowers/specs/2026-08-09-policy-accountability-trace-design.md b/docs/superpowers/specs/2026-08-09-policy-accountability-trace-design.md index a9915a9..206b430 100644 --- a/docs/superpowers/specs/2026-08-09-policy-accountability-trace-design.md +++ b/docs/superpowers/specs/2026-08-09-policy-accountability-trace-design.md @@ -1,8 +1,7 @@ # Policy Accountability & Pruning — Slice 1: Policy Trace & Replay -_Written 2026-08-09. Status: Slice 1 core implementation is complete through `2355ac0`; the -authoritative replay help contract is corrected in `fa68dfa`, and final documentation is this -handoff commit._ +_Written 2026-08-09. Status: Slice 1 implementation and final artifact-binding hardening are +complete through `86cb319`; final documentation is this handoff commit._ ## Implementation status — 2026-08-11 @@ -24,11 +23,10 @@ multi-turn sequences. Replay preserves `ImplicitOutcomeStore` writes in each bra production does not read that store into later policy inputs; its divergence therefore proves branch-local persistence, not a current downstream review effect. -Final verification at this boundary covered the 173-test focused policy suite, TypeScript, Biome, -the full 454-file suite (`3505 pass`, `12 skip`, `0 fail`), a fresh compiled build and the Bench, -Rig and Audit help surfaces. The required commands and durable acceptance semantics live in -`TEST_PLAN.md`; exact command output and the earlier pre-help baseline are retained in the ignored -Task-11 execution report. +Final verification at this boundary covered the focused policy/replay suites, TypeScript, Biome, +the full 454-file suite (`3515 pass`, `12 skip`, `0 fail`; 12,123 assertions), a fresh compiled +build and the Bench, Rig and Audit help surfaces. The required commands and durable acceptance +semantics live in `TEST_PLAN.md`; the Task reports retain the preceding gate evidence. ## Context