From 50f666c19248c54c9c8127f7ea5d291a4a44e3be Mon Sep 17 00:00:00 2001 From: coreytshaffer <78175888+coreytshaffer@users.noreply.github.com> Date: Wed, 8 Jul 2026 18:32:19 -0700 Subject: [PATCH] Add tc eval review CLI wrapper (CR-RH-003) Co-Authored-By: Claude Opus 4.8 --- docs/evals/review_harness_result_contract.md | 8 +- .../evals/review_harness_submission_schema.md | 7 +- docs/operations/review-harness-cli.md | 73 +++++ tests/test_eval_review_cli.py | 277 ++++++++++++++++++ triage_core/review_result.py | 21 +- triage_core/tc_cli.py | 118 +++++++- 6 files changed, 498 insertions(+), 6 deletions(-) create mode 100644 docs/operations/review-harness-cli.md create mode 100644 tests/test_eval_review_cli.py diff --git a/docs/evals/review_harness_result_contract.md b/docs/evals/review_harness_result_contract.md index fc68615..2162c32 100644 --- a/docs/evals/review_harness_result_contract.md +++ b/docs/evals/review_harness_result_contract.md @@ -108,9 +108,15 @@ gate when a `context-supported` citation does not mechanically resolve. Because the gate fails, `next_safe_action` is `null` even though an `authorized-next-action` claim is present. +## CLI + +The checker is exposed read-only via `tc eval review` — see the +[`tc eval review` CLI](../operations/review-harness-cli.md). The CLI adds no +checking logic; it validates a submission, runs `build_review_result`, and +renders or writes the result. + ## What This Does Not Do -- No CLI (the `tc eval review` name remains reserved for a later slice). - No model calls, no command execution, no raw-prose classification. - No citation *quality* judgment beyond section-scoped anchor resolution. - No action approval and no correctness, safety, certification, or diff --git a/docs/evals/review_harness_submission_schema.md b/docs/evals/review_harness_submission_schema.md index 8d295e8..df0b656 100644 --- a/docs/evals/review_harness_submission_schema.md +++ b/docs/evals/review_harness_submission_schema.md @@ -129,10 +129,11 @@ values carry no claim text or file paths. Stable codes include: human-review routing, next-safe-action selection) — see the [Review Harness Result Contract](review_harness_result_contract.md). -## Reserved (Not Yet Implemented) +## CLI -- The `tc eval review` CLI subcommand (name reserved under the existing `eval` - family). +- `tc eval review` validates a submission and runs the checker against a + supplied context packet — see the + [`tc eval review` CLI](../operations/review-harness-cli.md). ## Related diff --git a/docs/operations/review-harness-cli.md b/docs/operations/review-harness-cli.md new file mode 100644 index 0000000..9917343 --- /dev/null +++ b/docs/operations/review-harness-cli.md @@ -0,0 +1,73 @@ +# `tc eval review` CLI + +`tc eval review` is a thin, read-only wrapper around the evidence-bound review +harness. It validates a pre-tagged review submission, runs the deterministic +checker against an explicitly supplied context packet, and renders or writes the +result. It adds no checking logic of its own. + +## Boundary + +> This result verifies structural grounding and routing only. It is not a +> correctness, safety, certification, or production-readiness verdict. + +The command executes nothing, calls no model, classifies no raw prose, and +approves no action. It reads the submission and the context packet, and the only +file it may write is the caller-supplied `--output` path — it touches no ledger, +identity, or other runtime state. + +## Usage + +```bash +python -m triage_core.tc_cli eval review \ + --submission \ + --context-packet \ + [--changed-path ...] \ + [--output ] \ + [--print-json] \ + [--fail-on-gate] +``` + +## Arguments + +| Argument | Required | Purpose | +|---|---|---| +| `--submission` | yes | Path to a `review_submission_v0` JSON file. Validated before use. | +| `--context-packet` | yes | Path to the context bundle text (with `FILE:` markers) the review was performed against. | +| `--changed-path` | no | A repo-relative changed path for the scope check. Repeatable. Omitted → `scope_check: not_checked`. | +| `--output` | no | Write the `review_result_v0` JSON here. Omitted → render to stdout. | +| `--print-json` | no | Also print the JSON result to stdout. | +| `--fail-on-gate` | no | Exit non-zero (`3`) when `grounding_gate` is `fail`. | + +## Output and exit codes + +- **No `--output`:** prints the rendered review result (ASCII, leak-safe — ids, + codes, counts, and the boundary line only) to stdout. +- **`--output`:** writes the JSON result to that path and prints a short + `Success:` line instead of the rendered result. +- **`--print-json`:** prints the JSON result to stdout in addition to the normal + behavior. +- **Exit codes:** + - `0` — validation and checking succeeded (the grounding-gate outcome is data, + reported in the result). + - `1` — input or validation error (missing or unparseable submission, + structural validation failures, missing context packet). No result is + produced. + - `3` — only with `--fail-on-gate`, after successful validation and checking, + when `grounding_gate == fail`. + +## Example + +Running the shipped example fixtures produces an intentional grounding-gate +**FAIL** (one `context-supported` citation anchor does not resolve): + +```bash +python -m triage_core.tc_cli eval review \ + --submission tests/fixtures/evals/model_review/review_submission_v0.example.json \ + --context-packet tests/fixtures/evals/model_review/review_context_packet.example.md +``` + +## Related + +- [Review Harness Submission Schema](../evals/review_harness_submission_schema.md) +- [Review Harness Result Contract](../evals/review_harness_result_contract.md) +- [Evidence-Bound Model Review Protocol](../evals/evidence_bound_review_protocol.md) diff --git a/tests/test_eval_review_cli.py b/tests/test_eval_review_cli.py new file mode 100644 index 0000000..12005e6 --- /dev/null +++ b/tests/test_eval_review_cli.py @@ -0,0 +1,277 @@ +import json +import sys +from pathlib import Path + +import pytest + +from triage_core.review_result import BOUNDARY, build_review_result + +FIXTURE_DIR = Path(__file__).parent / "fixtures" / "evals" / "model_review" +EXAMPLE_SUBMISSION = str(FIXTURE_DIR / "review_submission_v0.example.json") +EXAMPLE_PACKET = str(FIXTURE_DIR / "review_context_packet.example.md") + + +def run_cli(monkeypatch, args): + from triage_core.tc_cli import main + + monkeypatch.setattr(sys, "argv", ["tc"] + args) + try: + main() + return 0 + except SystemExit as exc: + return exc.code if exc.code is not None else 0 + + +def write_json(path, obj): + Path(path).write_text(json.dumps(obj), encoding="utf-8") + return str(path) + + +def passing_submission(): + return { + "schema_version": "review_submission_v0", + "context_packet_ref": "packet.md", + "claims": [ + { + "id": "s1", + "text": "Supported.", + "category": "context-supported", + "citation": "FILE: a.txt", + }, + {"id": "n1", "text": "Next action.", "category": "authorized-next-action"}, + ], + "declared_scope": ["docs/"], + } + + +def passing_packet(path): + Path(path).write_text( + "---\nFILE: a.txt\n---\nsome content\n", encoding="utf-8" + ) + return str(path) + + +# --- Example fixtures: intentional FAIL --------------------------------------- + + +def test_example_renders_fail_and_exits_zero(monkeypatch, capsys): + code = run_cli( + monkeypatch, + ["eval", "review", "--submission", EXAMPLE_SUBMISSION, "--context-packet", EXAMPLE_PACKET], + ) + out = capsys.readouterr().out + assert code == 0 + assert "grounding_gate: fail" in out + assert BOUNDARY in out + + +def test_fail_on_gate_exits_three_on_fail(monkeypatch, capsys): + code = run_cli( + monkeypatch, + [ + "eval", + "review", + "--submission", + EXAMPLE_SUBMISSION, + "--context-packet", + EXAMPLE_PACKET, + "--fail-on-gate", + ], + ) + assert code == 3 + assert "grounding_gate: fail" in capsys.readouterr().out + + +# --- Passing case ------------------------------------------------------------- + + +def test_passing_case_renders_pass_and_exits_zero(monkeypatch, capsys, tmp_path): + sub = write_json(tmp_path / "sub.json", passing_submission()) + packet = passing_packet(tmp_path / "packet.md") + code = run_cli( + monkeypatch, + ["eval", "review", "--submission", sub, "--context-packet", packet], + ) + out = capsys.readouterr().out + assert code == 0 + assert "grounding_gate: pass" in out + assert "next_safe_action: claim n1" in out + + +def test_fail_on_gate_exits_zero_when_passing(monkeypatch, tmp_path): + sub = write_json(tmp_path / "sub.json", passing_submission()) + packet = passing_packet(tmp_path / "packet.md") + code = run_cli( + monkeypatch, + ["eval", "review", "--submission", sub, "--context-packet", packet, "--fail-on-gate"], + ) + assert code == 0 + + +# --- Output modes ------------------------------------------------------------- + + +def test_output_writes_json_and_prints_success(monkeypatch, capsys, tmp_path): + sub = write_json(tmp_path / "sub.json", passing_submission()) + packet = passing_packet(tmp_path / "packet.md") + out_path = tmp_path / "result.json" + code = run_cli( + monkeypatch, + ["eval", "review", "--submission", sub, "--context-packet", packet, "--output", str(out_path)], + ) + out = capsys.readouterr().out + assert code == 0 + assert "Success: Wrote review_result_v0 to" in out + assert "grounding_gate:" not in out # rendered result is not printed with --output + written = json.loads(out_path.read_text(encoding="utf-8")) + expected = build_review_result( + passing_submission(), Path(packet).read_text(encoding="utf-8") + ) + assert written == expected + + +def test_print_json_emits_json(monkeypatch, capsys, tmp_path): + sub = write_json(tmp_path / "sub.json", passing_submission()) + packet = passing_packet(tmp_path / "packet.md") + code = run_cli( + monkeypatch, + ["eval", "review", "--submission", sub, "--context-packet", packet, "--print-json"], + ) + out = capsys.readouterr().out + assert code == 0 + assert '"schema_version": "review_result_v0"' in out + + +# --- Scope check via --changed-path ------------------------------------------- + + +def test_changed_path_out_of_scope_fails_gate(monkeypatch, capsys, tmp_path): + sub = write_json(tmp_path / "sub.json", passing_submission()) # declared_scope docs/ + packet = passing_packet(tmp_path / "packet.md") + code = run_cli( + monkeypatch, + [ + "eval", + "review", + "--submission", + sub, + "--context-packet", + packet, + "--changed-path", + "triage_core/tc_cli.py", + "--fail-on-gate", + ], + ) + out = capsys.readouterr().out + assert code == 3 + assert "scope_check: fail" in out + assert "out_of_scope: triage_core/tc_cli.py" in out + + +def test_changed_path_within_scope_passes(monkeypatch, capsys, tmp_path): + sub = write_json(tmp_path / "sub.json", passing_submission()) + packet = passing_packet(tmp_path / "packet.md") + code = run_cli( + monkeypatch, + [ + "eval", + "review", + "--submission", + sub, + "--context-packet", + packet, + "--changed-path", + "docs/evals/x.md", + ], + ) + out = capsys.readouterr().out + assert code == 0 + assert "scope_check: pass" in out + + +# --- Input and validation errors ---------------------------------------------- + + +def test_invalid_submission_exits_one_and_prints_no_result(monkeypatch, capsys, tmp_path): + bad = passing_submission() + bad["claims"][0]["category"] = "made-up-category" + sub = write_json(tmp_path / "sub.json", bad) + packet = passing_packet(tmp_path / "packet.md") + code = run_cli( + monkeypatch, + ["eval", "review", "--submission", sub, "--context-packet", packet], + ) + out = capsys.readouterr().out + assert code == 1 + assert "Submission failed validation" in out + assert "$.claims[0].category: invalid_category" in out + assert "grounding_gate" not in out + + +def test_malformed_json_submission_exits_one(monkeypatch, capsys, tmp_path): + sub = tmp_path / "sub.json" + sub.write_text("{not valid json", encoding="utf-8") + packet = passing_packet(tmp_path / "packet.md") + code = run_cli( + monkeypatch, + ["eval", "review", "--submission", str(sub), "--context-packet", packet], + ) + out = capsys.readouterr().out + assert code == 1 + assert "Could not parse submission JSON" in out + + +def test_missing_submission_file_exits_one(monkeypatch, capsys, tmp_path): + packet = passing_packet(tmp_path / "packet.md") + code = run_cli( + monkeypatch, + ["eval", "review", "--submission", str(tmp_path / "nope.json"), "--context-packet", packet], + ) + out = capsys.readouterr().out + assert code == 1 + assert "Submission file not found" in out + + +def test_missing_context_packet_exits_one(monkeypatch, capsys, tmp_path): + sub = write_json(tmp_path / "sub.json", passing_submission()) + code = run_cli( + monkeypatch, + ["eval", "review", "--submission", sub, "--context-packet", str(tmp_path / "nope.md")], + ) + out = capsys.readouterr().out + assert code == 1 + assert "Context packet file not found" in out + + +# --- Leak-safety and no command execution ------------------------------------- + + +def test_stdout_does_not_echo_claim_text(monkeypatch, capsys, tmp_path): + secret = "SECRET-CLAIM-TEXT-SHOULD-NOT-LEAK" + submission = passing_submission() + submission["claims"][0]["text"] = secret + sub = write_json(tmp_path / "sub.json", submission) + packet = passing_packet(tmp_path / "packet.md") + run_cli( + monkeypatch, + ["eval", "review", "--submission", sub, "--context-packet", packet, "--print-json"], + ) + assert secret not in capsys.readouterr().out + + +def test_validation_command_in_submission_is_never_executed(monkeypatch, capsys, tmp_path): + submission = passing_submission() + sentinel = tmp_path / "sentinel_should_not_be_deleted.txt" + sentinel.write_text("still here", encoding="utf-8") + submission["validation"] = [ + {"command": f"rm -rf {sentinel}", "recorded_result": "not_recorded"} + ] + sub = write_json(tmp_path / "sub.json", submission) + packet = passing_packet(tmp_path / "packet.md") + code = run_cli( + monkeypatch, + ["eval", "review", "--submission", sub, "--context-packet", packet], + ) + assert code == 0 + assert "grounding_gate: pass" in capsys.readouterr().out + assert sentinel.exists() # command was recorded, never executed diff --git a/triage_core/review_result.py b/triage_core/review_result.py index 7c7077b..6673c61 100644 --- a/triage_core/review_result.py +++ b/triage_core/review_result.py @@ -26,8 +26,10 @@ from __future__ import annotations +import json import re -from typing import Any, Dict, List, Optional, Sequence, Tuple +from pathlib import Path +from typing import Any, Dict, List, Optional, Sequence, Tuple, Union SCHEMA_VERSION = "review_result_v0" @@ -262,3 +264,20 @@ def _path_in_scope(path: str, scope: Sequence[str]) -> bool: if normalized == candidate or normalized.startswith(candidate + "/"): return True return False + + +def write_review_result( + result: Dict[str, Any], output_path: Union[str, Path] +) -> Path: + """Write a ``review_result_v0`` to a JSON file (deterministic, sorted keys). + + Pure I/O helper mirroring ``eval_outcome_contract.write_actual_outcome``. It + only writes the caller-supplied path; it touches no ledger, identity, or + other runtime state. + """ + path = Path(output_path) + path.parent.mkdir(parents=True, exist_ok=True) + with open(path, "w", encoding="utf-8") as handle: + json.dump(result, handle, indent=2, sort_keys=True) + handle.write("\n") + return path diff --git a/triage_core/tc_cli.py b/triage_core/tc_cli.py index ce66911..8f5d366 100644 --- a/triage_core/tc_cli.py +++ b/triage_core/tc_cli.py @@ -1503,6 +1503,75 @@ def tc_eval_export_forbidden_tool_smoke(output_dir: str, case_id: str) -> None: print(f"Success: Wrote eval export-forbidden-tool-smoke contract file to {file_path}") +def tc_eval_review( + submission_path: str, + context_packet_path: str, + changed_paths, + output_path, + print_json: bool, + fail_on_gate: bool, +) -> None: + """Thin, read-only wrapper: validate a submission, run the checker, report. + + This adds no checking logic of its own. It validates the submission with the + CR-RH-001 validator, runs the CR-RH-002 deterministic checker against the + supplied context packet, and renders or writes the review_result_v0. It + executes nothing, calls no model, classifies no prose, and approves no + action. The only file it may write is the caller-supplied --output path. + """ + import json + import os + import sys + from pathlib import Path + + from triage_core.review_submission import ( + load_review_submission, + validate_review_submission, + ) + from triage_core.review_result import ( + build_review_result, + render_review_result, + write_review_result, + ) + + if not os.path.exists(submission_path): + print(f"Error: Submission file not found: {submission_path}") + sys.exit(1) + + try: + submission = load_review_submission(submission_path) + except (ValueError, json.JSONDecodeError): + print(f"Error: Could not parse submission JSON: {submission_path}") + sys.exit(1) + + errors = validate_review_submission(submission) + if errors: + print("Error: Submission failed validation:") + for err in errors: + print(f" - {err['path']}: {err['code']}") + sys.exit(1) + + if not os.path.exists(context_packet_path): + print(f"Error: Context packet file not found: {context_packet_path}") + sys.exit(1) + + context_packet = Path(context_packet_path).read_text(encoding="utf-8") + + result = build_review_result(submission, context_packet, changed_paths or None) + + if output_path: + written = write_review_result(result, output_path) + print(f"Success: Wrote review_result_v0 to {written}") + else: + print(render_review_result(result), end="") + + if print_json: + print(json.dumps(result, indent=2, sort_keys=True)) + + if fail_on_gate and result["grounding_gate"] == "fail": + sys.exit(3) + + def tc_context_plan(input_path: str, model_profile: str) -> None: import os import sys @@ -2123,6 +2192,44 @@ def main(): help="The fixture case_id to match (e.g., forbidden_tool_call_001)" ) + eval_review_parser = eval_subparsers.add_parser( + "review", + help="Validate a review submission and run the deterministic checker against a context packet", + ) + eval_review_parser.add_argument( + "--submission", + required=True, + type=str, + help="Path to a review_submission_v0 JSON file", + ) + eval_review_parser.add_argument( + "--context-packet", + required=True, + type=str, + help="Path to the context packet text file the review was performed against", + ) + eval_review_parser.add_argument( + "--changed-path", + action="append", + type=str, + help="A repo-relative changed path for the scope check (repeatable)", + ) + eval_review_parser.add_argument( + "--output", + type=str, + help="Optional path to write the review_result_v0 JSON file", + ) + eval_review_parser.add_argument( + "--print-json", + action="store_true", + help="Also print the JSON result to stdout", + ) + eval_review_parser.add_argument( + "--fail-on-gate", + action="store_true", + help="Exit non-zero (3) when grounding_gate is fail", + ) + # context context_parser = subparsers.add_parser("context", help="Manage and plan token context") context_subparsers = context_parser.add_subparsers(dest="context_command") @@ -2440,8 +2547,17 @@ def main(): tc_eval_export_privacy_smoke(args.output_dir, args.case_id) elif args.eval_command == "export-forbidden-tool-smoke": tc_eval_export_forbidden_tool_smoke(args.output_dir, args.case_id) + elif args.eval_command == "review": + tc_eval_review( + args.submission, + args.context_packet, + args.changed_path, + args.output, + args.print_json, + args.fail_on_gate, + ) else: - eval_parser.error("eval requires a subcommand: export-smoke, export-privacy-smoke, or export-forbidden-tool-smoke") + eval_parser.error("eval requires a subcommand: export-smoke, export-privacy-smoke, export-forbidden-tool-smoke, or review") elif args.command == "context": if args.context_command == "plan": tc_context_plan(args.input, args.model)