From 0ee1b7d557c484a558018c6110ee58087aaa0220 Mon Sep 17 00:00:00 2001 From: Akshaya Shanbhogue Date: Thu, 3 Sep 2026 13:06:27 -0700 Subject: [PATCH 1/6] =?UTF-8?q?feat(cli):=20add=20`coder-eval=20execute`?= =?UTF-8?q?=20=E2=80=94=20run=20tasks=20without=20grading=20them?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `coder-eval execute` is `coder-eval run` with the grading half removed: the agent runs and the full trajectory lands in task.json as usual, but no criterion is checked, `weighted_score` stays None, and the row finalizes as the new `FinalStatus.NOT_GRADED`. It exists so an external harness can own the verdict — the motivating case is Harbor (Terminal-Bench 2.0), which builds its own container, calls coder-eval as the agent, and grades with its own tests/test.sh. Grading twice there would be worse than not grading: coder-eval's verdict would be reported alongside Harbor's without being the one that counts. ## NOT_GRADED is a fourth reporting category `FinalStatus.NOT_GRADED.category == "ungraded"`, not a fold into one of the existing three — folding into "failed" would depress every pass rate, into "succeeded" would invent verdicts, into "error" would report a healthy run as broken. Ungraded rows therefore leave BOTH sides of every rate: `RunSummary` / `VariantAggregate` `pass_rate` and `error_share` now divide by `tasks_graded` (`tasks_run - tasks_not_graded`), which is identical to `tasks_run` for every graded run. `tasks_not_graded` is part of the sum-to-`tasks_run` invariant, not a `tasks_failed` sub-counter, and is defaulted so pre-existing run.json/experiment.json still parse. `weighted_score` is set to None explicitly rather than left to `calculate_weighted_score`, which writes 0.0 for an empty results list — a value indistinguishable from "graded and scored zero" that every downstream `score or 0.0` would launder into a real-looking failure. Only SUCCESS/FAILURE collapse into NOT_GRADED. ERROR, TIMEOUT, BUILD_FAILED, MAX_TURNS_EXHAUSTED and the budget stops are facts about the *run*, not about grading, so they still apply and `execute` still exits non-zero on a crash. ## The switch `BatchRunConfig.grade` -> `Orchestrator(grade=...)`, gating all four grading call sites (single-shot, evaluate-only, the simulation dialog check, post-failure diagnostics). It crosses the docker boundary in context.json, defaulting to True in-container so a host predating `execute` keeps grading. It is deliberately NOT a task-config field: no 5-layer merge, no -D path. A task YAML must never be able to declare itself ungraded; only the invoking command decides. `run` and `execute` share one body (`run_command.run_pipeline`) and differ solely in that flag — no third code path. Only the Typer signature is restated, and a test asserts the two option sets stay in step. ## Refused rather than degraded - `--junit-xml`: a report of verdicts, and there are none. (reports_junit still emits for an ungraded row it encounters elsewhere.) - `--resume`: partition_for_resume treats "has any final status" as finalized, so a NOT_GRADED row would be skipped by a later `run --resume` rather than graded. - Simulation tasks: the dialog loop reads criteria results to decide whether to keep talking, so an ungraded dialog would silently change its own stopping behavior. Rejected by name at startup. - `stop_early:` blocks go inert: early stop cuts a run once the criteria decide the outcome, and here the full trajectory is the deliverable. ## Ripple The explicit-mapping guards did their job — every surface below failed loudly rather than silently mis-bucketing the new member: pyright on `reports_junit._category_of`, the `_status_badge` category tests, the published-action gate's "every FinalStatus must be classified" test, and CE018's enum-parity check. - reports_junit: ungraded -> (already counted by _set_counts). - reports_html: neutral badge; the "no member falls through to neutral" guard now allows it for ungraded only. - reports / reports_experiment: a "Not Graded" line, and the pass rate reads "n/a" for a fully ungraded run instead of 0.0% (an ordinary EMPTY run keeps its original 0/0 rendering — different facts). - experiment aggregation: average_score means over graded rows only, and _pick_worst_status ranks ungraded least-urgent so any real verdict wins. - verify-published-action.yml: NOT_GRADED hard-fails. That job runs the published action, which always grades, so reaching it means the action is dispatching the wrong command and every score gate is measuring nothing. - evalboard statusCategory: NOT_GRADED -> "unknown", the category every consumer already treats as "no verdict here". Not a pass, not a failure. ## Verification `make verify` and `make evalboard-verify` both green. The new suite covers the status semantics, an end-to-end execute against the agentless task (asserting pre_run's file IS written, so a skipped task can't pass), a negative control proving `run` still scores that same task 1.0, the docker context.json round-trip, and the run/execute signature parity. Scoped out of this PR: relaxing the non-empty `success_criteria` validator. `execute` on an existing task YAML needs no such change; it is only needed for a foreign task format that has no criteria to declare, and belongs with that work. Co-Authored-By: Claude Opus 5 (1M context) --- .github/workflows/verify-published-action.yml | 12 + CLAUDE.md | 4 +- docs/REPORT_SCHEMA.md | 21 +- docs/USER_GUIDE.md | 34 ++ evalboard/lib/status.ts | 7 + src/coder_eval/cli/__init__.py | 5 +- src/coder_eval/cli/execute_command.py | 224 +++++++++++++ src/coder_eval/cli/run_command.py | 90 +++++- src/coder_eval/cli/run_helpers.py | 9 +- .../cli/run_task_internal_command.py | 5 + src/coder_eval/isolation/docker_runner.py | 8 + src/coder_eval/models/enums.py | 18 +- src/coder_eval/models/experiment.py | 24 +- src/coder_eval/models/results.py | 37 ++- src/coder_eval/orchestration/batch.py | 3 + src/coder_eval/orchestration/config.py | 14 + src/coder_eval/orchestration/experiment.py | 20 +- src/coder_eval/orchestrator.py | 75 ++++- src/coder_eval/reports.py | 13 +- src/coder_eval/reports_experiment.py | 5 +- src/coder_eval/reports_html.py | 5 +- src/coder_eval/reports_junit.py | 10 +- .../ce018_no_final_status_name_denylist.py | 1 + tests/test_execute_command.py | 293 ++++++++++++++++++ tests/test_reports_html.py | 19 +- 25 files changed, 920 insertions(+), 36 deletions(-) create mode 100644 src/coder_eval/cli/execute_command.py create mode 100644 tests/test_execute_command.py diff --git a/.github/workflows/verify-published-action.yml b/.github/workflows/verify-published-action.yml index 134dbeb1..914cbdc0 100644 --- a/.github/workflows/verify-published-action.yml +++ b/.github/workflows/verify-published-action.yml @@ -541,6 +541,18 @@ jobs: "uploaded run dir before re-running; this is an unattended paid job.") sys.exit(1) + # NOT_GRADED means a task ran but was never scored. This job invokes the + # published action, which runs `coder-eval run` (graded), so reaching it is + # impossible unless the action started dispatching `coder-eval execute` -- + # in which case every score gate below silently measures nothing and the job + # goes green having verified no verdict at all. Hard-fail, don't tolerate. + ungraded = [s for s in statuses if s == "NOT_GRADED"] + if ungraded: + print("::error::task(s) reported NOT_GRADED -- the published action ran without " + "grading. `coder-eval run` always grades, so the action is dispatching the " + "wrong command and every score gate in this job is measuring nothing.") + sys.exit(1) + # Exit-contract check, conditional on the model having actually performed. # Ignoring the step's exit code entirely (see the continue-on-error rationale # above) would also hide a REGRESSION in the action's own exit logic -- e.g. a diff --git a/CLAUDE.md b/CLAUDE.md index ac00d5ef..896c2e4b 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -89,7 +89,8 @@ coder_eval/ │ ├── cli/ # CLI commands (Typer + Rich) │ ├── __init__.py # Typer app setup (core commands) -│ ├── run_command.py # `coder-eval run` +│ ├── run_command.py # `coder-eval run` + `run_pipeline` (the body BOTH run and execute share) +│ ├── execute_command.py # `coder-eval execute` — Typer signature only; delegates to run_pipeline(grade=False) │ ├── plan_command.py # `coder-eval plan` │ ├── report_command.py # `coder-eval report` │ ├── run_helpers.py # CLI helper functions @@ -146,6 +147,7 @@ action.yml # Published composite GitHub Action (coder-ev - **Reference solutions are directory-only, and shielded (partially) from the agent**: `task.reference` is a single required `directory:` (relative to the task YAML) — the inline `code:` / single-file `file:` forms are gone, because a directory is the only shape that can be permission-gated as a unit; a `model_validator(mode="before")` gives the removed forms a migration error. The orchestrator stages a **per-run private copy** (`orchestration/evaluation.py::stage_reference_dir`, symlinks stripped) into a tempdir, removed in `_cleanup` via `path_utils.rmtree_restrictive` (keyed on `_reference_staging_root`, recorded BEFORE the copy so a failed copy still cleans up; `rmtree(ignore_errors=True)` silently declines on a tree left at 000) and deliberately never preserved into `run_dir/artifacts`. That copy is held at mode `000` for the whole of every `agent.communicate` call via **`Sandbox.set_permissions`**, the driver-aware wrapper over `fs_permissions.py::set_permissions`. Windows **stack**: exiting restores the *enclosing* window's mode, only the outermost exit restores the pre-window mode — that is what makes a mid-turn re-grant (`mode=READ_ONLY_MODE`) expressible, and it covers two windows at the same mode so no refcount is needed. The window is enforced **only inside a docker container** (`Sandbox.enforces_permission_windows`) and is a no-op on the host, where the agent shares our uid. **That gate keys on the `CODER_EVAL_IN_CONTAINER` env var, NOT `sandbox.driver`** — `run_task_internal_command` rewrites `driver: docker` → `tempdir` before building the in-container Orchestrator, so a driver-based gate would silently disable the anti-cheat on exactly the path that needs it (regression-guarded by `TestSandboxDriverGate`); `resolve_reference_dir` gates its `/work/references` branch on the same var for the same reason. The task directory is **not** shielded (`:ro` mount → EROFS, and the same YAML is readable at `/work/input`). Criteria address reference files with the `$REFERENCE_DIR` token (same resolver as `$TASK_DIR`) and the `REFERENCE_DIR` env var for `run_command`; `reference_comparison` names one file via `reference_file`. Docker mounts a throwaway **read-write** copy at `/work/references` (a `:ro` mount cannot be chmod'd — EROFS), masks the in-task-dir original with an empty tmpfs, and drops `DAC_OVERRIDE`/`DAC_READ_SEARCH`. `FOWNER`/`CHOWN` are deliberately **NOT** dropped: the in-container orchestrator that applies the window is the same root process with the same caps, so dropping `FOWNER` breaks *the harness's own* chmod wherever the bind mount preserves a non-root owner (native Linux — verified: `chmod: Operation not permitted`), i.e. exactly where the drop would otherwise bite. A window that cannot be applied is now a hard error, not a warning: `Sandbox.set_permissions` passes `strict=True` whenever it enforces, so an unprotected run fails instead of producing a normal-looking score. **KNOWN GAP — this is defense-in-depth, not a boundary**: (a) `chmod(2)` is gated on owner-or-`CAP_FOWNER` and the container runs as root owning the copy, so a deliberate `chmod 755 /work/references` restores access; (b) the window spans `agent.communicate` only, and nothing reaps agent child processes at turn end, so a backgrounded read loop succeeds once the window closes. The **write** half of (b) is closed — `path_utils.digest_tree` hashes the tree at staging and `Orchestrator._verify_reference_integrity` re-checks before grading, raising `ReferenceTamperedError` (→ `FinalStatus.ERROR`) on a mismatch so an agent cannot overwrite the reference to drive `reference_comparison` to 1.0. Passive reads are blocked; an adversarial agent is not. Full containment requires running the agent as a non-root uid AND holding the window for the agent's whole lifetime — follow-up. `tasks/anti_cheat_reference` probes the passive-read half. - **Harness run-limit parity**: a shared `BaseAgentConfig` field must mean the same thing on every backend, so a divergence is either fixed or documented — never silent. **`run_limits.max_turns` on Codex/Antigravity counts VISIBLE turns** (resolved tool calls, read live off the shared `EventCollector.visible_turn_count`, the same list `TurnRecord.commands` holds) because one `communicate()` is a single SDK turn on both, so a native counter would clamp at 1; claude-code keeps its native SDK cap, whose unit (an agent-loop turn) absorbs arbitrarily many parallel calls — the same number is NOT the same budget across harnesses. OpenCode likewise keeps a native unit — the CLI streams a real multi-step loop per `communicate()` (`step_start`/`step_finish`), so `max_turns: N` allows N complete steps and cuts cleanly when step N+1 begins. The cap is enforced on the same loop boundary as the cooperative early stop and finalizes cleanly as `max_turns_exhausted` (no crash, no retry); on Antigravity that boundary lives in `_drain()`, so the background-work poll loop honors it too. Known unfixed divergences: `permission_mode` on Codex and Antigravity (both run unconfined — the sandbox driver is the isolation boundary), `disallowed_tools` on Codex (forwarded, not SDK-enforced), `allowed_tools`/`disallowed_tools` on Antigravity (not read at all), `turn_timeout` on Antigravity (bounded by an earlier internal poll deadline at 80% of it), `allowed_tools`/`disallowed_tools`/`system_prompt` on OpenCode (no CLI knob; warned at `start()`, not enforced), and **`agent.plugins[].path` depth** — claude-code REQUIRES a plugin root holding `skills/` and silently loads NOTHING from a bare skills directory, while Codex and Antigravity scan both depths and accept either. That is the costly direction: the wrong depth produces no error, every positive row of an activation suite scores 0, and the suite reports recall 0.0, which reads exactly like a skill that never triggers. Held to the plugin-root shape (for `SKILL_SOURCE_PATH` only) by lint rule CE045. `plugins` on OpenCode is **honored for skills**: each local plugin root is mapped to the `skills` dir its `.claude-plugin/plugin.json` declares (default `/skills`, never the root itself — `skills.paths` is scanned recursively and a plugin root can hold a self-referential symlink) and injected via `OPENCODE_CONFIG_CONTENT`, which `--pure` does not suppress; a plugin's agents/hooks/commands/MCP servers are still dropped. Full table + rationale: docs/agents/HARNESS_PARITY.md. - **sandbox isolation**: Tasks that don't need MCP servers should set `setting_sources: []` in their `agent:` block to isolate the sandbox from the host project's CLAUDE.md and settings. Without this, the host project's CLAUDE.md (often 20 KB+) is injected into every API call, inflating cache-creation tokens and cost significantly. +- **Execute vs. run (the grading switch)**: `coder-eval execute` is `coder-eval run` with grading removed — the agent runs and the full trajectory is captured, but no criterion is checked, `weighted_score` is `None` (never `0.0`, which would be indistinguishable from "graded and scored zero"), and the row finalizes as **`FinalStatus.NOT_GRADED`**, whose `category` is a **fourth** bucket, `"ungraded"`. Ungraded rows leave BOTH sides of every rate: `RunSummary.pass_rate` / `error_share` and `VariantAggregate.pass_rate` divide by `tasks_graded` (`tasks_run - tasks_not_graded`), and `tasks_not_graded` is part of the sum-to-`tasks_run` invariant, not a `tasks_failed` sub-counter. **Only SUCCESS/FAILURE collapse into it** — `ERROR`, `TIMEOUT`, `BUILD_FAILED`, `MAX_TURNS_EXHAUSTED` and the budget stops are facts about the *run*, not about grading, and still apply (so `execute` still exits non-zero on a crash). The switch is `BatchRunConfig.grade` → `Orchestrator(grade=...)` → the **four** grading call sites (single-shot, evaluate-only, the simulation dialog check, and post-failure diagnostics); it crosses the docker boundary in `context.json` (defaulting to `True` in-container, so a host predating `execute` keeps grading). It is **deliberately not a task-config field** — no 5-layer merge, no `-D` path — because a task YAML must never declare itself ungraded; only the invoking command decides. `run` and `execute` share one body (`run_command.run_pipeline`) and differ solely in that flag, so there is no third code path. Three things are refused rather than degraded: `--junit-xml` (a report of verdicts, and there are none — though `reports_junit` still emits `` for an ungraded row it encounters), `--resume` (`partition_for_resume` treats "has any final status" as finalized, so a `NOT_GRADED` row would be *skipped* by a later `run --resume` instead of graded), and simulation tasks (their turn-continuation logic reads criteria results, so an ungraded dialog would silently change its own stopping behavior). `stop_early:` blocks are inert under `execute` for the same reason the kill switch exists: the full trajectory is the deliverable. Motivating consumer: an external harness (Harbor / Terminal-Bench 2.0) that builds its own container, calls coder-eval as the agent, and grades with its own tests. - **Run-time caps (non-criterion enforcement)**: `TaskDefinition.run_limits` (`RunLimits` model) is the single namespace for all *task-level* run-time caps — `max_turns` / `task_timeout` / `turn_timeout` (structural) and `max_input_tokens` / `max_output_tokens` / `max_total_tokens` / `max_usd` (cumulative budget). Token/USD breaches abort with `FinalStatus.TOKEN_BUDGET_EXCEEDED` or `COST_BUDGET_EXCEEDED` (both `category == "failed"`). Structural caps are set from the CLI via `-D run_limits.max_turns=…` / `-D run_limits.task_timeout=…` / `-D run_limits.turn_timeout=…` (field-merged into `run_limits`); budget caps via `-D run_limits.max_usd=…` etc. or YAML. Layered config uses field-merge — a variant block overrides individual keys without replacing the task's block. The one *per-criterion* cap, `stop_early.decide_within`, deliberately lives on `LiveSuccessCriterion` instead (see below) — the watcher must attribute a decision-step timeout to a specific criterion, which `RunLimits` (task-scoped, criterion-agnostic) cannot express. - **Early stop on criterion (opt-in, per-criterion arming)**: a `stop_early:` block (`StopEarlyPolicy`) on a criterion ends a single-shot run early once the run's **armed** criteria decide the outcome, so a raised `max_turns` isn't wasted on the smoke flavor. The block's PRESENCE is the arming and alone activates the watcher — there is **no run-level master switch**: `run_limits.stop_early: false` is the run-level KILL SWITCH that force-disarms every block (the one-line experiment-variant/`-D` override for an authoritative full run), and `run_limits.stop_early: true` (the removed master arm) is a hard `EarlyStopConfigError` at resolution. The block exists on `LiveSuccessCriterion` only (currently `skill_triggered`, `command_executed` — so arming an unobservable criterion is unrepresentable, a pydantic extra-forbid error). Arming carries one implicit trigger (a native live-fail may fail-stop the run); its keys refine it: `on_pass: stop` (pass-stop the moment the criterion live-passes; default `continue` just latches) and `decide_within: N` (still undecided after N tool-call steps latches an **effective fail**, fed through the same fail-stop rule, reported as `decision_budget_exceeded` — an ordinary weighted fail, NOT a gate-bypassing force-fail; cumulative across retry attempts of the same turn). A trigger whose polarity the instance can't decide (per the abstract, checker-independent `live_decidable_polarities()`, a pure function of the criterion's own fields, paired with the checker's `live_verdict` override by lint rule CE025, a registry-based whole-tree check) is **inert by design** — one dataset-fanned YAML line serves both positive rows (pass/timeout live) and distractor rows (fail live). Verdicts **latch**: once a criterion decides, its `live_verdict` is never polled again. Stop rule is weighted, not strict-boolean: `run_limits.stop_early_gate_threshold` (default `1.0`, reproducing strict-AND behavior exactly) is the minimum weighted score (`Σ weight·score / Σ weight` over the armed subset) required to pass; a fail-stop fires once the armed set's **ceiling** (best case for everything still undecided) can no longer reach the threshold — so a low-weight fail or timeout that can't doom the gate is absorbed and the run continues — and is **deferred while any pass-capable armed criterion is undecided** (a distractor misfire never truncates a positive row's recall signal); a pass-stop fires once the `on_pass: stop` subset's **floor** (worst case) already meets the threshold, and is symmetrically **deferred while any pass-capable armed criterion outside the `on_pass: stop` subset is undecided** (so an early pass never freezes a sibling `on_pass: continue` criterion's signal out of the trajectory). A fail-stop is therefore verdict-preserving; a pass-stop can miss a *later* distractor misfire, so authoritative P/R/F1 comes from a kill-switched (`stop_early: false`) run. Driven by `orchestration/early_stop.py::EarlyStopWatcher` (built when `early_stop_active(task)`: ≥1 armed criterion, kill switch not thrown) through the agent's cooperative `should_stop` seam (tool-call granularity, no SIGKILL); live verdicts only *trigger* the stop — the standard `check_all_async` on the frozen trajectory is authoritative. Gating is **FIRED-ONLY**: a run the watcher actually cut gates on the **armed subset** via the weighted `EvaluationResult.armed_criteria_passed`; a run that completes naturally — armed or not — gates strict-AND via `all_criteria_passed`, so adding a block never changes the verdict of a run it didn't cut. Note the gate keys on the watcher having FIRED (`result.early_stop is not None`), not on confirmed truncation — an agent that ignores `should_stop`, or a stop firing on the final message, still gates armed-only. Every resolution-time guardrail violation is a hard error at resolution (plan *and* run); the one load-time case — a `stop_early:` block on a non-live criterion — is a pydantic schema error at task load, which the run surface reports as a skipped task like any other malformed task. A runtime verdict bug **fails open** to a full run. Surfaces: `EarlyStopInfo` (incl. `gate_threshold` at stop time), report notes/badges, `stopped_early` run.json rows, `EarlyStopped`/`EarlyStopReason` telemetry dims. Worked rationale: docs/TASK_DEFINITION_GUIDE.md § `stop_early`. No blocks anywhere ⇒ behavior byte-for-byte unchanged. diff --git a/docs/REPORT_SCHEMA.md b/docs/REPORT_SCHEMA.md index f9d5554a..b4c5dce0 100644 --- a/docs/REPORT_SCHEMA.md +++ b/docs/REPORT_SCHEMA.md @@ -44,7 +44,8 @@ run-level summary; full per-replicate detail lives in each `task.json`. | `start_time` / `end_time` | `datetime` | Run window. | | `total_duration_seconds` | `float` | Wall-clock. | | `tasks_run` | `int` | Total replicates executed. | -| `tasks_succeeded` / `tasks_failed` / `tasks_error` | `int` | Category counts. **Invariant:** the three sum to `tasks_run`. | +| `tasks_succeeded` / `tasks_failed` / `tasks_error` / `tasks_not_graded` | `int` | Category counts. **Invariant:** the four sum to `tasks_run`. | +| `tasks_not_graded` | `int` | Tasks run by `coder-eval execute` — executed, deliberately unscored. Excluded from **both** sides of `pass_rate`. Defaults to `0`, so pre-`execute` `run.json` still parses. | | `tasks_token_budget_exceeded` / `tasks_cost_budget_exceeded` | `int` | Sub-counters of `tasks_failed` (not part of the invariant). | | `skipped_tasks` | `list[{path, reason}]` | Load failures / `skip: true` opt-outs. | | `max_parallel` | `int` | Concurrency used. | @@ -59,8 +60,9 @@ publishing different numbers for the same run. | Key | Type | Meaning | | --- | --- | --- | -| `pass_rate` | `float \| None` | `tasks_succeeded / tasks_run` — errors are in the denominator, counted as misses. `None` on an empty run (0/0 is unknown, not 0%). | -| `error_share` | `float \| None` | `tasks_error / tasks_run`. Diagnostic only; never adjusts the rate. | +| `pass_rate` | `float \| None` | `tasks_succeeded / tasks_graded` — errors are in the denominator, counted as misses; ungraded tasks are in neither. `None` on an empty or fully ungraded run (0/0 is unknown, not 0%). | +| `error_share` | `float \| None` | `tasks_error / tasks_graded`. Diagnostic only; never adjusts the rate. | +| `tasks_graded` | `int` | `tasks_run - tasks_not_graded`. The denominator of both rates above. | | `total_cost_usd` | `float \| None` | **The bill**: agent + judge + simulator, summed over the rows. `None` when nothing could be priced. | | `agent_cost_usd` | `float \| None` | Subject-agent spend alone. The harness-vs-harness comparison figure — judge spend is a property of the suite's criteria and identical across harnesses, so leaving it in would make two harnesses look closer than they are. | | `eval_overhead_cost_usd` | `float \| None` | Judge + simulator spend. The other half of `total_cost_usd`. | @@ -232,7 +234,8 @@ the same weighted armed gate as a native fail), ## `variant.json` — `VariantAggregate` A single aggregate (not wrapped): `variant_id`, `tasks_run`, `tasks_succeeded`, -`tasks_failed`, `tasks_error` (same sum-to-`tasks_run` invariant), `average_score`, +`tasks_failed`, `tasks_error`, `tasks_not_graded` (same sum-to-`tasks_run` invariant), `average_score` +(the mean over **graded** rows only), `average_duration`, `total_tokens`, `replicate_count`, `tasks_token_budget_exceeded`, `tasks_cost_budget_exceeded`. @@ -308,10 +311,20 @@ String enum values and their reporting category: | `COST_BUDGET_EXCEEDED` | failed | `$` | | `ERROR` | error | `!` | | `BUILD_FAILED` | error | `B` | +| `NOT_GRADED` | ungraded | `?` | > **Gotcha:** `BUILD_FAILED` (a failed Docker image build) categorizes as **error**, > not failed — easy to miscount downstream. +`NOT_GRADED` is produced only by [`coder-eval execute`](USER_GUIDE.md#coder-eval-execute--run-without-grading): +the task ran and its full trajectory was captured, but no criterion was checked, so +`weighted_score` is `None` (**not** `0.0` — that would be indistinguishable from a task +that was graded and scored zero). `ungraded` is a fourth reporting category, not a fold +into one of the other three: counting it as failed would depress every pass rate, and +counting it as succeeded would invent a verdict. Execution facts still win over it — a +crash, timeout, or budget breach under `execute` reports `ERROR` / `TIMEOUT` / +`TOKEN_BUDGET_EXCEEDED` as usual. + `TOKEN_BUDGET_EXCEEDED` and `COST_BUDGET_EXCEEDED` are produced by the cumulative budget caps under `run_limits:` (`max_input_tokens` / `max_output_tokens` / `max_total_tokens`, and `max_usd` respectively), checked after each completed agent turn — see diff --git a/docs/USER_GUIDE.md b/docs/USER_GUIDE.md index 1676690d..878e72a9 100644 --- a/docs/USER_GUIDE.md +++ b/docs/USER_GUIDE.md @@ -58,6 +58,40 @@ flags of their own. They live under `run_limits:` in the task YAML, or on the co `-D run_limits.=` (e.g. `-D run_limits.max_usd=2.50`). The complete field reference is in the [Task Definition Guide](TASK_DEFINITION_GUIDE.md#run-limits). +### `coder-eval execute` — run without grading + +```bash +coder-eval execute tasks/hello_date.yaml # run, capture, score nothing +coder-eval execute tasks/*.yaml --run-dir ./my-run -j 3 # every `run` flag but two +``` + +Identical to `coder-eval run` except that no success criterion is checked. Each task +executes normally and its full trajectory lands in `task.json` as usual, but +`weighted_score` stays `null` and the row finalizes as `NOT_GRADED` — a reporting +category of its own, excluded from both sides of every pass rate. The two commands +share one implementation, so they cannot drift apart. + +Use it when something *else* owns the verdict — an external harness that builds its own +container and runs its own tests — or to separate one expensive agent run from grading +you want to iterate on afterwards. Grade the results later with +[`coder-eval evaluate`](#coder-eval-evaluate--test-criteria-without-an-agent). + +**Only the verdict is withheld, never the facts of the run.** A crash, timeout, or +budget breach still reports `ERROR` / `TIMEOUT` / `TOKEN_BUDGET_EXCEEDED` and still +exits non-zero, exactly as under `run`. + +Every `run` flag is available except three things, each refused rather than quietly +degraded: + +| Not supported | Why | +| --- | --- | +| `--junit-xml` | A JUnit report reports verdicts, and there are none. | +| `--resume` | Resume treats "has any final status" as finalized, so a `NOT_GRADED` row would be *skipped* by a later `run --resume` rather than graded. | +| Simulation tasks | The dialog loop reads criteria results to decide whether to keep talking, so an ungraded dialog would silently change its own stopping behavior. Rejected by name at startup. | + +`stop_early:` blocks are also inert here: early stop exists to cut a run once the +criteria decide the outcome, and under `execute` the full trajectory is the deliverable. + ### `coder-eval plan` — validate tasks ```bash diff --git a/evalboard/lib/status.ts b/evalboard/lib/status.ts index f2b02520..17796f4c 100644 --- a/evalboard/lib/status.ts +++ b/evalboard/lib/status.ts @@ -2,8 +2,14 @@ // Mirrors coder_eval `FinalStatus.category` (src/coder_eval/models/enums.py): // SUCCESS -> passed // ERROR / BUILD_FAILED -> error (BUILD_FAILED is an environment/setup failure) +// NOT_GRADED -> unknown (`coder-eval execute`: ran, deliberately unscored) // anything else (FAILURE, TIMEOUT, MAX_TURNS_EXHAUSTED, …) -> failed // +// NOT_GRADED maps to "unknown" rather than gaining a category of its own: every +// consumer already handles "unknown" (a null status) as "no verdict here", which +// is exactly what an ungraded row is. It is therefore not a pass, not a failure, +// and sorts in the middle — the same treatment a missing status gets. +// // Note: this only categorizes coder_eval task statuses. UI status display // (e.g. StatusPill) also handles flow execution statuses like "Completed" // and "Faulted" and uses its own logic. @@ -16,6 +22,7 @@ export function statusCategory(status: string | null): StatusCategory { if (!status) return "unknown"; if (status === "SUCCESS") return "passed"; if (status === "ERROR" || status === "BUILD_FAILED") return "error"; + if (status === "NOT_GRADED") return "unknown"; return "failed"; } diff --git a/src/coder_eval/cli/__init__.py b/src/coder_eval/cli/__init__.py index 40eab01d..0ded1cae 100644 --- a/src/coder_eval/cli/__init__.py +++ b/src/coder_eval/cli/__init__.py @@ -7,6 +7,7 @@ from .aggregate_command import aggregate_command from .console import console from .evaluate_command import evaluate_command +from .execute_command import execute_command from .plan_command import plan_command from .report_command import report_command from .run_command import run_command @@ -46,7 +47,8 @@ def main( Run 'coder-eval COMMAND --help' for help on a specific command. Available commands: - - run: Execute evaluation tasks + - run: Execute evaluation tasks and grade them + - execute: Execute evaluation tasks WITHOUT grading them - plan: Validate task files (dry-run) - evaluate: Run criteria against a directory without an agent - report: Display or export evaluation reports @@ -75,6 +77,7 @@ def main( # emits a CoderEval.Cli. event (Status/DurationMs/ErrorType) on completion; # functools.wraps preserves the signature so Typer still parses each command's flags. app.command(name="run")(track_command("run")(run_command)) +app.command(name="execute")(track_command("execute")(execute_command)) app.command(name="plan")(track_command("plan")(plan_command)) app.command(name="evaluate")(track_command("evaluate")(evaluate_command)) app.command(name="report")(track_command("report")(report_command)) diff --git a/src/coder_eval/cli/execute_command.py b/src/coder_eval/cli/execute_command.py new file mode 100644 index 00000000..b90fa0e3 --- /dev/null +++ b/src/coder_eval/cli/execute_command.py @@ -0,0 +1,224 @@ +"""Execute command - run evaluation tasks WITHOUT grading them. + +``coder-eval execute`` is ``coder-eval run`` with the grading half removed: the +sandbox is built, the agent runs, and the full trajectory is captured into the +usual ``task.json`` / ``run.json`` layout — but no success criterion is checked, +``weighted_score`` stays ``None``, and each row finalizes as +``FinalStatus.NOT_GRADED``. + +It exists so an *external* harness can own the verdict. The motivating case is +Harbor (Terminal-Bench 2.0), which builds its own container, calls coder-eval as +the agent, and grades with its own ``tests/test.sh``. Grading twice there would +be worse than not grading at all: coder-eval's verdict would be reported +alongside Harbor's without being the one that counts. + +Every flag on ``run`` is available here except two, and both omissions are +deliberate: + +* ``--junit-xml`` — a JUnit report is a report of verdicts, and there are none. +* ``--resume`` — ``partition_for_resume`` treats "has any final status" as + finalized, so a ``NOT_GRADED`` row would be skipped by a later ``run --resume`` + rather than graded. Supporting it needs resume to distinguish "done" from + "executed but unscored"; until then, refusing is the honest option. + +The command shares ``run``'s entire body (``run_command.run_pipeline``); only the +Typer signature is restated, because Typer builds its parser from the signature. +``tests/test_execute_command.py`` asserts the two signatures stay in step. +""" + +from pathlib import Path + +import click +import typer + +from ..models import PreservationMode +from .run_command import run_pipeline + + +def execute_command( + task_files: list[Path] | None = typer.Argument( # noqa: B008 + None, + help="Path(s) to task YAML file(s). Defaults to all tasks/ recursively.", + ), + preservation_mode: PreservationMode | None = typer.Option( # noqa: B008 + None, + "--preservation-mode", + help=( + "How to persist each task's sandbox: NONE (delete), MOVE_ON_WRITE " + "(run in a tempdir, move into run_dir/artifacts), or DIRECT_WRITE " + "(run directly in run_dir/artifacts). Default is driver-derived — " + "docker → DIRECT_WRITE, else MOVE_ON_WRITE. Explicit value always wins." + ), + ), + run_dir: Path | None = typer.Option( # noqa: B008 + None, + "--run-dir", + help="Custom run directory (default: auto-generated timestamped directory in runs/)", + ), + max_parallel: int = typer.Option( + 1, + "--max-parallel", + "-j", + help="Maximum number of tasks to run concurrently (default: 1 = sequential)", + min=1, + ), + verbose: bool = typer.Option( + False, + "--verbose", + "-v", + help="Enable verbose (DEBUG level) logging", + ), + log_file: Path | None = typer.Option( # noqa: B008 + None, + "--log-file", + help="Log to file in addition to console", + ), + tags: str | None = typer.Option( + None, + "--tags", + "-t", + help="Only run tasks matching any of these tags (comma-separated, e.g., 'smoke,golden')", + ), + exclude_tags: str | None = typer.Option( + None, + "--exclude-tags", + help="Skip tasks matching any of these tags (comma-separated, e.g., 'example,integration')", + ), + include_skipped: bool = typer.Option( + False, + "--include-skipped", + help=( + "Also run tasks marked `skip: true` in their YAML. Off by default so the " + "nightly/CI keep excluding them; use for on-demand / local runs of " + "quarantined or opt-in tasks." + ), + ), + agent_type: str | None = typer.Option( + None, + "--type", + "-T", + help="Override agent type for all tasks (e.g. 'claude-code', 'codex', or a plugin kind)", + ), + model: str | None = typer.Option( + None, + "--model", + "-m", + help="Override agent model for all tasks (e.g., claude-sonnet-4-20250514)", + ), + stream: str | None = typer.Option( + None, + "--stream", + "-s", + click_type=click.Choice(["full", "minimal"], case_sensitive=False), + help="Stream LLM events to terminal: 'full' or 'minimal' (turn-level only). Disables progress bar.", + ), + backend: str | None = typer.Option( + None, + "--backend", + "-b", + click_type=click.Choice(["direct", "bedrock", "litellm"], case_sensitive=False), + help="API backend (default: from API_BACKEND env var)", + ), + experiment: Path | None = typer.Option( # noqa: B008 + None, + "--experiment", + "-e", + help="Experiment definition YAML (default: experiments/default.yaml)", + ), + sample: int | None = typer.Option( + None, + "--sample", + help=( + "For dataset-backed tasks, use a random N-row sample " + "(fixed seed: reproducible, unbiased across paths). Cheap dataset smoke-test." + ), + min=1, + ), + sample_per_stratum: int | None = typer.Option( + None, + "--sample-per-stratum", + help=( + "For dataset-backed tasks, keep up to N rows per stratum (stratify_field, " + "default expected_skill) — a stratified sample that overrides the task's " + "dataset.sample_per_stratum without editing the YAML. Ignored when --sample is set. " + "Nondeterministic (re-draws each run) unless the task sets dataset.sample_seed." + ), + min=1, + ), + repeats: int | None = typer.Option( + None, + "--repeats", + help="Run each (task, variant) N times. Overrides experiment/variant `repeats:`. Must be >=1.", + min=1, + ), + driver: str | None = typer.Option( + None, + "--driver", + click_type=click.Choice(["tempdir", "docker"], case_sensitive=False), + help="Override sandbox driver for all tasks. 'docker' runs each task in a fresh container.", + ), + set_overrides: list[str] = typer.Option( # noqa: B008 + [], + "--set", + "-D", + metavar="PATH=VALUE", + help=( + "Override any resolved task-config field under agent/run_limits/sandbox, " + "e.g. -D run_limits.max_turns=30 -D agent.permission_mode=plan " + "-D agent.sdk_options.effort=high -D sandbox.docker.network=none. " + "Repeatable. Validated against the schema. A path set by both an alias " + "and -D is an error; values are YAML-parsed (on/off/yes/no stay strings). " + "(--model and --driver are shorthand aliases for -D agent.model / " + "-D sandbox.driver.)" + ), + ), +) -> None: + """Run evaluation tasks WITHOUT checking their success criteria. + + Identical to `coder-eval run` except that nothing is graded: each task + executes, its full trajectory is captured to task.json, and the row + finalizes as NOT_GRADED with no weighted_score. Use it when an external + harness owns the verdict, or to separate an expensive agent run from + grading you want to iterate on afterwards. + + Grade the results afterwards with `coder-eval evaluate`. + + Execution failures still fail: a crash, timeout, or budget breach reports + ERROR / TIMEOUT / TOKEN_BUDGET_EXCEEDED and exits non-zero exactly as under + `run`. Only the verdict is withheld, never the facts of the run. + + Not supported here: --junit-xml (no verdicts to report), --resume (a + NOT_GRADED row would be mistaken for a finalized one), and simulation tasks + (their turn-continuation logic reads criteria results). + + Examples: + + coder-eval execute tasks/hello_date.yaml + + coder-eval execute tasks/*.yaml --run-dir ./my-run --max-parallel 3 + """ + run_pipeline( + grade=False, + task_files=task_files, + preservation_mode=preservation_mode, + run_dir=run_dir, + # Not exposed as flags — see the module docstring for why each is refused. + resume=False, + junit_xml=None, + max_parallel=max_parallel, + verbose=verbose, + log_file=log_file, + tags=tags, + exclude_tags=exclude_tags, + include_skipped=include_skipped, + agent_type=agent_type, + model=model, + stream=stream, + backend=backend, + experiment=experiment, + sample=sample, + sample_per_stratum=sample_per_stratum, + repeats=repeats, + driver=driver, + set_overrides=set_overrides, + ) diff --git a/src/coder_eval/cli/run_command.py b/src/coder_eval/cli/run_command.py index 9b22929c..d820e301 100644 --- a/src/coder_eval/cli/run_command.py +++ b/src/coder_eval/cli/run_command.py @@ -354,6 +354,65 @@ def run_command( coder-eval run tasks/*.yaml --tags golden,basic --exclude-tags example """ + run_pipeline( + grade=True, + task_files=task_files, + preservation_mode=preservation_mode, + run_dir=run_dir, + resume=resume, + max_parallel=max_parallel, + verbose=verbose, + log_file=log_file, + junit_xml=junit_xml, + tags=tags, + exclude_tags=exclude_tags, + include_skipped=include_skipped, + agent_type=agent_type, + model=model, + stream=stream, + backend=backend, + experiment=experiment, + sample=sample, + sample_per_stratum=sample_per_stratum, + repeats=repeats, + driver=driver, + set_overrides=set_overrides, + ) + + +def run_pipeline( + *, + grade: bool, + task_files: list[Path] | None, + preservation_mode: PreservationMode | None, + run_dir: Path | None, + resume: bool, + max_parallel: int, + verbose: bool, + log_file: Path | None, + junit_xml: Path | None, + tags: str | None, + exclude_tags: str | None, + include_skipped: bool, + agent_type: str | None, + model: str | None, + stream: str | None, + backend: str | None, + experiment: Path | None, + sample: int | None, + sample_per_stratum: int | None, + repeats: int | None, + driver: str | None, + set_overrides: list[str], +) -> None: + """The shared body of ``coder-eval run`` and ``coder-eval execute``. + + Everything below the Typer signature is identical for both commands; the only + difference is ``grade``, which decides whether success criteria are checked + (``run``) or the trajectory is captured and left unscored (``execute``). Both + commands are pure flag-parsing wrappers over this function, so a behavior + change can never apply to one and miss the other. + """ # --resume needs an explicit run dir to resume into (auto-generated dirs are always fresh). if resume and run_dir is None: raise typer.BadParameter("--resume requires --run-dir pointing at the run to continue.") @@ -414,6 +473,7 @@ def run_command( resume=resume, include_skipped=include_skipped, junit_xml=junit_xml, + grade=grade, ) ) except KeyboardInterrupt: @@ -439,6 +499,7 @@ async def _run_all_tasks( resume: bool = False, include_skipped: bool = False, junit_xml: Path | None = None, + grade: bool = True, ) -> None: """Async entry point for running all tasks (optionally in parallel). @@ -459,6 +520,7 @@ async def _run_all_tasks( experiment_path: Optional path to experiment YAML (default: experiments/default.yaml) junit_xml: Optional path to write a JUnit XML report to, after the run summary is persisted and before the failure exit-code gate. + grade: False for `coder-eval execute` — run and capture, score nothing. """ # Prepare run directory run_dir = prepare_run_directory(run_dir) @@ -484,6 +546,7 @@ async def _run_all_tasks( repeats=repeats, verbose=verbose, include_skipped=include_skipped, + grade=grade, ) from ..telemetry import flush_telemetry, track_event @@ -500,6 +563,7 @@ async def _run_all_tasks( "StreamMode": stream_mode or "none", "Resume": resume, "ExperimentProvided": experiment_path is not None, + "Grade": grade, }, ) @@ -513,7 +577,7 @@ async def _run_all_tasks( try: # Always run through experiment layer (defaults to experiments/default.yaml) summary, failed_suite_gates = await _run_with_experiment( - all_task_files, config, experiment_path, stream_mode, max_parallel, resume=resume + all_task_files, config, experiment_path, stream_mode, max_parallel, resume=resume, grade=grade ) # Aggregate task logs into run.log @@ -600,6 +664,7 @@ async def _run_with_experiment( stream_mode: str | None, max_parallel: int, resume: bool = False, + grade: bool = True, ) -> tuple[RunSummary, int]: """Run tasks through the experiment resolution layer. @@ -670,6 +735,23 @@ async def _run_with_experiment( except ValueError as e: raise typer.BadParameter(str(e)) from e + # Simulation tasks are rejected under `execute`, not silently degraded. The + # dialog loop reads criteria results to decide whether to keep talking, so an + # ungraded dialog would quietly change its own stopping behavior and produce a + # trajectory that is not the one `run` would have produced. Rejecting is a + # config error (exit 2), and it names the offending tasks. + if not grade: + simulated = sorted( + rt.task.task_id for rt in resolved if rt.task.simulation is not None and rt.task.simulation.enabled + ) + if simulated: + raise typer.BadParameter( + "`coder-eval execute` does not support simulation tasks (their turn-continuation " + + "logic depends on criteria results): " + + ", ".join(simulated) + + ". Use `coder-eval run` for these." + ) + if skipped: console.print( f"[yellow]⚠[/] {len(skipped)} task file(s) skipped " @@ -745,6 +827,12 @@ async def _run_with_experiment( # Per-suite pass-rate rollups for dataset-backed tasks (no-op when none were used). # Pass `resolved` through so suite_thresholds on each criterion can be evaluated. + # Skipped entirely under `execute`: a rollup aggregates per-criterion results, + # and there are none — running it would gate a suite on an empty aggregate and + # report a threshold failure for a run that was never measured. + if not grade: + return summary, 0 + from ..reports import write_suite_rollups rollups = write_suite_rollups(config.run_dir, task_results, resolved_tasks=resolved) diff --git a/src/coder_eval/cli/run_helpers.py b/src/coder_eval/cli/run_helpers.py index 9ef304cf..90324ca2 100644 --- a/src/coder_eval/cli/run_helpers.py +++ b/src/coder_eval/cli/run_helpers.py @@ -132,7 +132,14 @@ def print_execution_summary(run_dir: Path, summary: RunSummary) -> None: summary: Run execution summary """ console.print(f"\n[bold green]Run complete:[/bold green] {run_dir}") - console.print(f"[bold]Results:[/bold] {summary.tasks_succeeded}/{summary.tasks_run} succeeded") + # An ungraded run has no pass rate to report — printing "0/N succeeded" for a + # clean `coder-eval execute` reads as a total failure. Report what actually + # happened instead, and keep the graded line for whatever WAS graded. + if summary.tasks_not_graded: + console.print(f"[bold]Results:[/bold] {summary.tasks_not_graded}/{summary.tasks_run} executed, not graded") + console.print("[dim]Grade later: uv run coder-eval evaluate [/dim]") + if summary.tasks_graded: + console.print(f"[bold]Results:[/bold] {summary.tasks_succeeded}/{summary.tasks_graded} succeeded") console.print(f"[dim]View report: open {run_dir / 'experiment.md'}[/dim]") console.print(f"[dim]View report: uv run coder-eval report {run_dir}[/dim]") diff --git a/src/coder_eval/cli/run_task_internal_command.py b/src/coder_eval/cli/run_task_internal_command.py index 3157e4a7..8bc885b5 100644 --- a/src/coder_eval/cli/run_task_internal_command.py +++ b/src/coder_eval/cli/run_task_internal_command.py @@ -153,6 +153,10 @@ def _watch_host_heartbeat() -> None: # a missing key falls back to the docker default (DIRECT_WRITE) — a deliberate # default, not version back-compat. preservation_mode = PreservationMode(context.get("preservation_mode", PreservationMode.DIRECT_WRITE.value)) + # `coder-eval run` vs `coder-eval execute`, decided host-side. Defaults to + # True (grade) so a host that predates `execute` — which never writes the + # key — keeps its exact behavior. + grade: bool = context.get("grade", True) # Docker WORKDIR alignment: the host resolves the concrete WORKDIR # (config value / "auto" -> `docker inspect` / fallback) and forwards it here. # Absent -> None -> standard run_dir/artifacts workspace. @@ -197,6 +201,7 @@ def _watch_host_heartbeat() -> None: config_lineage=config_lineage, replicate_index=replicate_index, workspace_dir=workspace_dir, + grade=grade, ) # Install the stdout-NDJSON stream callback so per-tool-call events diff --git a/src/coder_eval/isolation/docker_runner.py b/src/coder_eval/isolation/docker_runner.py index 31683d75..bb967222 100644 --- a/src/coder_eval/isolation/docker_runner.py +++ b/src/coder_eval/isolation/docker_runner.py @@ -540,11 +540,16 @@ def __init__( preservation_mode: PreservationMode = PreservationMode.DIRECT_WRITE, stream_callback: StreamCallback | None = None, verbose: bool = False, + grade: bool = True, ) -> None: self.rt = rt self.preservation_mode = preservation_mode self.stream_callback = stream_callback self.verbose = verbose + # Forwarded to the in-container orchestrator via context.json. It is a + # run-level decision made by the CLI, so it cannot be recovered from the + # staged task.yaml on the other side. + self.grade = grade # Set by _prepare_host_mounts: the tmp lean copy of ~/.claude that # _build_argv mounts read-write. None when there is no ~/.claude to # forward or the mount is opted out (CODER_EVAL_NO_CLAUDE_MOUNT). @@ -721,6 +726,9 @@ def _dump_task_yaml() -> str: "replicate_index": self.rt.replicate_index, "config_lineage": {k: v.model_dump(mode="json") for k, v in self.rt.config_lineage.items()}, "preservation_mode": self.preservation_mode.value, + # `coder-eval run` vs `coder-eval execute`. Not derivable from + # task.yaml on the container side (deliberately not a task field). + "grade": self.grade, "source_yaml": self.rt.source_yaml, # Docker WORKDIR alignment: concrete path the in-container # orchestrator runs at + captures out (None = standard workspace). diff --git a/src/coder_eval/models/enums.py b/src/coder_eval/models/enums.py index 0cba3650..7135de37 100644 --- a/src/coder_eval/models/enums.py +++ b/src/coder_eval/models/enums.py @@ -15,9 +15,16 @@ class FinalStatus(StrEnum): MAX_TURNS_EXHAUSTED = "MAX_TURNS_EXHAUSTED" TOKEN_BUDGET_EXCEEDED = "TOKEN_BUDGET_EXCEEDED" COST_BUDGET_EXCEEDED = "COST_BUDGET_EXCEEDED" + # `coder-eval execute` ran the agent but deliberately skipped grading, so + # there is no verdict to report. Distinct from FAILURE (which asserts the + # criteria were checked and did not pass) and from ERROR (which asserts + # something went wrong). Only SUCCESS/FAILURE collapse into it — every + # other member records an *execution* fact that still applies when the + # run is ungraded. + NOT_GRADED = "NOT_GRADED" @property - def category(self) -> Literal["succeeded", "failed", "error"]: + def category(self) -> Literal["succeeded", "failed", "error", "ungraded"]: """Classify this status into a reporting category (the SSOT for failed/succeeded/error).""" return _STATUS_CATEGORIES[self] @@ -31,7 +38,7 @@ def icon(self) -> str: # catch-all default) so a newly-added status fails the assert below until it is # classified — rather than silently collapsing into "failed" (which would skew # reports AND the telemetry Category dimension). Mirrors the _STATUS_ICONS guard. -_STATUS_CATEGORIES: dict[FinalStatus, Literal["succeeded", "failed", "error"]] = { +_STATUS_CATEGORIES: dict[FinalStatus, Literal["succeeded", "failed", "error", "ungraded"]] = { FinalStatus.SUCCESS: "succeeded", FinalStatus.FAILURE: "failed", FinalStatus.ERROR: "error", @@ -43,6 +50,12 @@ def icon(self) -> str: FinalStatus.MAX_TURNS_EXHAUSTED: "failed", FinalStatus.TOKEN_BUDGET_EXCEEDED: "failed", FinalStatus.COST_BUDGET_EXCEEDED: "failed", + # A fourth category, not a fold into one of the three. Folding into + # "failed" would depress every pass rate; folding into "succeeded" would + # invent verdicts; folding into "error" would report a healthy run as + # broken. Reporting surfaces exclude it from BOTH the numerator and the + # denominator of a pass rate — an ungraded task was never measured. + FinalStatus.NOT_GRADED: "ungraded", } assert set(_STATUS_CATEGORIES) == set(FinalStatus), "Missing category for FinalStatus member" @@ -57,6 +70,7 @@ def icon(self) -> str: FinalStatus.MAX_TURNS_EXHAUSTED: "M", FinalStatus.TOKEN_BUDGET_EXCEEDED: "#", FinalStatus.COST_BUDGET_EXCEEDED: "$", + FinalStatus.NOT_GRADED: "?", } assert set(_STATUS_ICONS) == set(FinalStatus), "Missing icon for FinalStatus member" diff --git a/src/coder_eval/models/experiment.py b/src/coder_eval/models/experiment.py index fe6ae6ff..4c28982b 100644 --- a/src/coder_eval/models/experiment.py +++ b/src/coder_eval/models/experiment.py @@ -222,6 +222,13 @@ class VariantAggregate(BaseModel): # noqa: CE009 -- persisted result model; rou tasks_succeeded: int tasks_failed: int tasks_error: int + # Fourth bucket of the task_count invariant (see RunSummary.tasks_not_graded). + # Defaulted so experiment.json written before `coder-eval execute` still loads. + tasks_not_graded: int = Field( + default=0, + ge=0, + description="Tasks executed without grading (`coder-eval execute`). Excluded from pass_rate entirely.", + ) average_score: float average_duration: float total_tokens: int | None = None @@ -244,16 +251,25 @@ class VariantAggregate(BaseModel): # noqa: CE009 -- persisted result model; rou @model_validator(mode="after") def _check_task_count_invariant(self) -> VariantAggregate: - if self.tasks_succeeded + self.tasks_failed + self.tasks_error != self.tasks_run: - total = f"{self.tasks_succeeded} + {self.tasks_failed} + {self.tasks_error}" + buckets = self.tasks_succeeded + self.tasks_failed + self.tasks_error + self.tasks_not_graded + if buckets != self.tasks_run: + total = f"{self.tasks_succeeded} + {self.tasks_failed} + {self.tasks_error} + {self.tasks_not_graded}" raise ValueError(f"Task count invariant violated: {total} != {self.tasks_run}") return self + @property + def tasks_graded(self) -> int: + """Tasks actually measured — ``pass_rate``'s denominator.""" + return self.tasks_run - self.tasks_not_graded + @computed_field # type: ignore[prop-decorator] @property def pass_rate(self) -> float | None: - """``tasks_succeeded / tasks_run`` as a 0-1 fraction. ``None`` on an empty variant.""" - return self.tasks_succeeded / self.tasks_run if self.tasks_run else None + """``tasks_succeeded / tasks_graded`` as a 0-1 fraction. ``None`` when nothing was graded. + + Mirrors ``RunSummary.pass_rate``: ungraded tasks leave both sides. + """ + return self.tasks_succeeded / self.tasks_graded if self.tasks_graded else None class TaskExperimentSummary(BaseModel): # noqa: CE009 -- persisted result model; round-trip leniency like models/results.py diff --git a/src/coder_eval/models/results.py b/src/coder_eval/models/results.py index 8834710a..170b868b 100644 --- a/src/coder_eval/models/results.py +++ b/src/coder_eval/models/results.py @@ -1050,6 +1050,17 @@ class RunSummary(BaseModel): tasks_succeeded: int = Field(description="Number of tasks that succeeded") tasks_failed: int = Field(description="Number of tasks that failed") tasks_error: int = Field(description="Number of tasks that encountered errors") + # Part of the task_count invariant (a fourth bucket, not a sub-counter), but + # defaulted so run.json written before `coder-eval execute` existed — where + # no task can be ungraded — still deserialises. + tasks_not_graded: int = Field( + default=0, + ge=0, + description=( + "Number of tasks executed without grading (`coder-eval execute`). " + "Excluded from BOTH sides of pass_rate — an ungraded task was never measured." + ), + ) # Informational sub-counters: subsets of tasks_failed (NOT part of the # task_count invariant). Default 0 so old serialized RunSummary JSON @@ -1097,11 +1108,17 @@ class RunSummary(BaseModel): @model_validator(mode="after") def _check_task_count_invariant(self) -> RunSummary: - if self.tasks_succeeded + self.tasks_failed + self.tasks_error != self.tasks_run: - total = f"{self.tasks_succeeded} + {self.tasks_failed} + {self.tasks_error}" + buckets = self.tasks_succeeded + self.tasks_failed + self.tasks_error + self.tasks_not_graded + if buckets != self.tasks_run: + total = f"{self.tasks_succeeded} + {self.tasks_failed} + {self.tasks_error} + {self.tasks_not_graded}" raise ValueError(f"Task count invariant violated: {total} != {self.tasks_run}") return self + @property + def tasks_graded(self) -> int: + """Tasks that were actually measured — the denominator for every rate below.""" + return self.tasks_run - self.tasks_not_graded + # Derived run metrics: computed_fields over the stored counts and # ``task_results``, so they serialize into run.json while staying impossible to # set to something the rows disagree with. Consumers should read these rather @@ -1110,18 +1127,24 @@ def _check_task_count_invariant(self) -> RunSummary: @computed_field # type: ignore[prop-decorator] @property def pass_rate(self) -> float | None: - """``tasks_succeeded / tasks_run`` as a 0-1 fraction. ``None`` on an empty run.""" - return self.tasks_succeeded / self.tasks_run if self.tasks_run else None + """``tasks_succeeded / tasks_graded`` as a 0-1 fraction. ``None`` on an empty run. + + The denominator excludes ungraded tasks (``coder-eval execute``), which were + never measured — counting them as misses would report a clean execute run as + 0% pass. Identical to ``tasks_run`` for every graded run. + """ + return self.tasks_succeeded / self.tasks_graded if self.tasks_graded else None @computed_field # type: ignore[prop-decorator] @property def error_share(self) -> float | None: - """``tasks_error / tasks_run`` as a 0-1 fraction. ``None`` on an empty run. + """``tasks_error / tasks_graded`` as a 0-1 fraction. ``None`` on an empty run. Diagnostic only, never adjusts the rate: a drop at a high error share is an - infrastructure night, the same drop at a normal share is the model. + infrastructure night, the same drop at a normal share is the model. Shares + ``pass_rate``'s denominator so the two are directly comparable. """ - return self.tasks_error / self.tasks_run if self.tasks_run else None + return self.tasks_error / self.tasks_graded if self.tasks_graded else None @computed_field # type: ignore[prop-decorator] @property diff --git a/src/coder_eval/orchestration/batch.py b/src/coder_eval/orchestration/batch.py index 8576cdfc..d79e6d96 100644 --- a/src/coder_eval/orchestration/batch.py +++ b/src/coder_eval/orchestration/batch.py @@ -164,6 +164,7 @@ async def run_single(rt: ResolvedTask) -> TaskResult: preservation_mode=preservation_mode, stream_callback=task_callback, verbose=config.verbose, + grade=config.grade, ).run() # The in-container _finalize_result can't emit task telemetry # (connection-string env vars aren't forwarded into the @@ -185,6 +186,7 @@ async def run_single(rt: ResolvedTask) -> TaskResult: source_yaml=rt.source_yaml, config_lineage=rt.config_lineage, replicate_index=rt.replicate_index, + grade=config.grade, ) result = await orchestrator.run() tr = TaskResult( @@ -665,6 +667,7 @@ def build_run_summary( tasks_succeeded=sum(1 for s in statuses if s.category == "succeeded"), tasks_failed=sum(1 for s in statuses if s.category == "failed"), tasks_error=sum(1 for s in statuses if s.category == "error"), + tasks_not_graded=sum(1 for s in statuses if s.category == "ungraded"), tasks_token_budget_exceeded=sum(1 for s in statuses if s == FinalStatus.TOKEN_BUDGET_EXCEEDED), tasks_cost_budget_exceeded=sum(1 for s in statuses if s == FinalStatus.COST_BUDGET_EXCEEDED), skipped_tasks=skipped_tasks or [], diff --git a/src/coder_eval/orchestration/config.py b/src/coder_eval/orchestration/config.py index 695b5594..80a3d812 100644 --- a/src/coder_eval/orchestration/config.py +++ b/src/coder_eval/orchestration/config.py @@ -94,6 +94,20 @@ class BatchRunConfig(BaseModel): description="CLI override for replicates per (task, variant). None = defer to experiment layers.", ) + # Grading switch: `coder-eval run` (True) vs `coder-eval execute` (False). + # It lives HERE and nowhere else on purpose — it is deliberately NOT part of + # the 5-layer task merge, so there is no `-D grade=...` path and no + # MergeField (CE014 does not apply to a scalar bool outside the merged + # roots). A task YAML must never be able to declare itself ungraded; only + # the invoking command decides. + grade: bool = Field( + default=True, + description=( + "Evaluate success criteria after execution. False = `coder-eval execute`: " + "run and capture the trajectory, score nothing, finalize as NOT_GRADED." + ), + ) + # Logging verbose: bool = Field(default=False, description="Enable verbose (DEBUG level) logging for Docker output") diff --git a/src/coder_eval/orchestration/experiment.py b/src/coder_eval/orchestration/experiment.py index 25d4ed58..3e8c1500 100644 --- a/src/coder_eval/orchestration/experiment.py +++ b/src/coder_eval/orchestration/experiment.py @@ -819,11 +819,22 @@ def _pick_worst_status(statuses: list[FinalStatus]) -> FinalStatus: Unknown categories fall back to priority -1 so they sort as worst-of-all (fail-closed: a new unrecognised status becomes the most urgent). + + "ungraded" sorts LEAST urgent (above "succeeded") — it carries no verdict, so + any replicate that does have one must win. It therefore survives only when + every replicate is ungraded, which is the only case reachable today anyway + (``grade`` is run-level, so replicates never mix). """ - priority = {"error": 0, "failed": 1, "succeeded": 2} + priority = {"error": 0, "failed": 1, "succeeded": 2, "ungraded": 3} return min(statuses, key=lambda s: priority.get(s.category, -1)) +def _mean_graded_score(vr_list: list[VariantResult]) -> float: + """Mean ``weighted_score`` over the graded rows; ``0.0`` when none were graded.""" + graded = [v.weighted_score for v in vr_list if v.final_status.category != "ungraded"] + return sum(graded) / len(graded) if graded else 0.0 + + def _mean_reference_similarity(reps: list[TaskResult]) -> float | None: """Return the mean reference_comparison score across replicates that have one.""" scores = [ @@ -938,9 +949,14 @@ def aggregate_results( tasks_succeeded=sum(1 for v in vr_list if v.final_status.category == "succeeded"), tasks_failed=sum(1 for v in vr_list if v.final_status.category == "failed"), tasks_error=sum(1 for v in vr_list if v.final_status.category == "error"), + tasks_not_graded=sum(1 for v in vr_list if v.final_status.category == "ungraded"), tasks_token_budget_exceeded=sum(1 for v in vr_list if v.final_status == FinalStatus.TOKEN_BUDGET_EXCEEDED), tasks_cost_budget_exceeded=sum(1 for v in vr_list if v.final_status == FinalStatus.COST_BUDGET_EXCEEDED), - average_score=sum(v.weighted_score for v in vr_list) / len(vr_list), + # Mean over GRADED rows only. An ungraded row has no score (it + # arrives here as 0.0 because VariantResult.weighted_score is a + # plain float), so including it would report a clean execute run as + # average_score 0.0 — a number indistinguishable from "scored zero". + average_score=_mean_graded_score(vr_list), average_duration=sum(v.duration_seconds / v.replicate_count for v in vr_list) / len(vr_list), total_tokens=total_tokens, replicate_count=vr_list[0].replicate_count if vr_list else 1, diff --git a/src/coder_eval/orchestrator.py b/src/coder_eval/orchestrator.py index a2697bc0..22092c17 100644 --- a/src/coder_eval/orchestrator.py +++ b/src/coder_eval/orchestrator.py @@ -363,6 +363,7 @@ def __init__( config_lineage: dict[str, ConfigLineageEntry] | None = None, replicate_index: int = 0, workspace_dir: Path | None = None, + grade: bool = True, ): """Initialize the orchestrator. @@ -385,6 +386,13 @@ def __init__( run_dir/artifacts/, and the workspace is copied out to run_dir/artifacts/ at cleanup. Resolved host-side by DockerRunner; None keeps standard behavior. Takes precedence over preservation_mode when set. + grade: Whether to evaluate success criteria after execution. False is + `coder-eval execute`: the agent runs and the full trajectory is + captured, but no criterion is checked, ``weighted_score`` stays + None, and the row finalizes as ``FinalStatus.NOT_GRADED``. It is + deliberately NOT a task-config field — a task YAML must never be + able to declare itself ungraded — so it arrives only from + ``BatchRunConfig.grade``, never from the 5-layer merge or -D. """ self.task = task self.run_dir = run_dir @@ -403,6 +411,7 @@ def __init__( self.source_yaml = source_yaml self.config_lineage = config_lineage or {} self.replicate_index = replicate_index + self.grade = grade # Derived paths self.report_path = self.run_dir / "task.json" @@ -583,11 +592,19 @@ def _kill_agent_subprocess_sync() -> None: elapsed_seconds=time.time() - start_time, ) - # Update final status + # Update final status. The NOT_GRADED arm sits between the + # execution facts and FAILURE deliberately: under grade=False no + # criterion ran, so `success` is always False and FAILURE would be + # a verdict we never actually reached — but MAX_TURNS_EXHAUSTED + # (like TIMEOUT / BUILD_FAILED / the budget stops on the except + # branches below) is a fact about the RUN, not about grading, and + # still applies. With grade=True the chain is unchanged. if success: self.result.final_status = FinalStatus.SUCCESS elif self.result.max_turns_exhausted: self.result.final_status = FinalStatus.MAX_TURNS_EXHAUSTED + elif not self.grade: + self.result.final_status = FinalStatus.NOT_GRADED else: self.result.final_status = FinalStatus.FAILURE @@ -683,6 +700,9 @@ def _kill_agent_subprocess_sync() -> None: # but BEFORE _finalize_result so task.json includes the field. # Allowlist non-success terminal statuses; SUCCESS and # MAX_TURNS_EXHAUSTED skip the tail to keep task.json compact. + # NOT_GRADED is deliberately absent: like SUCCESS and + # MAX_TURNS_EXHAUSTED it is not a diagnosis of something going + # wrong, so it keeps task.json compact. if self.result.final_status in { FinalStatus.ERROR, FinalStatus.TIMEOUT, @@ -785,6 +805,11 @@ async def _evaluate_post_failure_criteria(self) -> None: """ if self.result is None: return + if not self.grade: + # Grading site 4 of 4. Under `execute` no criterion is checked on any + # path, diagnostics included — recording a not_evaluated vector here + # would imply criteria we were supposed to run and couldn't. + return if self.success_checker is None or self.sandbox is None: self._record_post_failure_not_evaluated("the sandbox or success checker was unavailable") return @@ -879,7 +904,14 @@ def _finalize_result(self, start_time: float) -> None: # path) run inside run()'s try, whose broad `except Exception` already converts # a raise into a populated ERROR result, so they intentionally stay unwrapped. try: - self.result.calculate_weighted_score(self.task.success_criteria) + if self.grade: + self.result.calculate_weighted_score(self.task.success_criteria) + else: + # Explicit None, NOT the 0.0 calculate_weighted_score writes for an + # empty results list — that value is indistinguishable from a task + # that was graded and scored zero, and every downstream `score or + # 0.0` would launder it into a real-looking failure. + self.result.weighted_score = None except ValueError as e: logger.error("Weighted-score computation failed; marking row ERROR: %s", e, exc_info=True) self.result.weighted_score = None @@ -1284,7 +1316,18 @@ async def _setup(self) -> None: # armed evaluate-only re-grade builds an inert (never-fed) watcher — # harmless, and keeps a single creation point. if early_stop_active(self.task): - self._early_stop_watcher = EarlyStopWatcher.for_task(self.task) + if self.grade: + self._early_stop_watcher = EarlyStopWatcher.for_task(self.task) + else: + # Early stop cuts the run once coder-eval's own criteria decide the + # outcome. Under `execute` there is no outcome to decide and the + # trajectory is the deliverable (an external harness grades it), so + # an armed criterion must not truncate it. Same effect as the + # run_limits.stop_early kill switch, decided one layer up. + logger.info( + "Grading disabled (execute mode): early-stop is armed but stays disabled; " + + "the full trajectory is the deliverable." + ) # Stage the reference BEFORE either branch returns: judge criteria with # include_reference=true (and any $REFERENCE_DIR/... file entry) expect it @@ -1888,6 +1931,14 @@ async def _evaluation_loop(self) -> bool: assert self.task.agent is not None if self.agent is None: + # Grading site 1 of 4. Evaluate-only with grading off would neither + # run an agent nor check anything — a no-op that still writes a + # task.json. Refuse instead of producing an empty row. + if not self.grade: + raise ValueError( + "grade=False is meaningless on the evaluate-only path (no agent attached): " + + "the run would neither execute nor grade." + ) # No agent attached: evaluate-only re-grade of a completed sandbox. # (No-op tasks have a NoOpAgent here, so they take the normal path # below.) Check the criteria directly against the sandbox. @@ -1955,6 +2006,15 @@ async def _evaluation_loop(self) -> bool: logger.debug(f"Agent response received ({len(turn_record.agent_output)} chars)") + # Grading site 2 of 4. `execute` stops here: the trajectory is captured + # and persisted exactly as on a graded run, but nothing is scored. + # Returning False keeps FinalStatus off SUCCESS; run()'s status chain + # turns it into NOT_GRADED. The reference-integrity check is skipped too + # — it exists to protect a grade that is not happening. + if not self.grade: + logger.info("Grading disabled (execute mode): skipping success criteria.") + return False + # Check success criteria (reference_dir feeds reference_comparison + judges) logger.debug("Checking success criteria") await self._verify_reference_integrity() @@ -2073,6 +2133,15 @@ async def _run_dialog_criteria_check( """ assert self.result is not None assert self.success_checker is not None + # Grading site 3 of 4. Unreachable today — `execute` rejects simulation + # tasks at the CLI, because the dialog's turn-continuation logic reads + # criteria results to decide whether to keep talking, so an ungraded + # dialog would silently change its own stopping behavior. Kept as a + # correct, defensive no-op so the gate holds if that restriction lifts. + if not self.grade: + self.result.success_criteria_results = [] + self.result.weighted_score = None + return [] await self._verify_reference_integrity() criteria_results = await self.success_checker.check_all_async( self.task.success_criteria, diff --git a/src/coder_eval/reports.py b/src/coder_eval/reports.py index 88345636..ea6d5897 100644 --- a/src/coder_eval/reports.py +++ b/src/coder_eval/reports.py @@ -237,8 +237,16 @@ def early_stop_gate_note(reason: str) -> str: def _pass_rate_lines(summary: RunSummary) -> list[str]: - """The pass rate over every dispatched task, plus the error share when non-zero.""" - lines = [f"- **Pass Rate**: {_fmt_rate(summary.pass_rate)} ({summary.tasks_succeeded}/{summary.tasks_run})"] + """The pass rate over every GRADED task, plus the error share when non-zero. + + An ungraded run (``coder-eval execute``) has no pass rate at all, so it says + so rather than rendering ``0.0% (0/N)`` — which reads as a total failure. + """ + # Only an ungraded run gets the explanatory line. An ordinary EMPTY run keeps + # its original "n/a (0/0)" rendering — the two are different facts. + if summary.tasks_not_graded and not summary.tasks_graded: + return [f"- **Pass Rate**: n/a — {summary.tasks_not_graded} task(s) executed without grading"] + lines = [f"- **Pass Rate**: {_fmt_rate(summary.pass_rate)} ({summary.tasks_succeeded}/{summary.tasks_graded})"] if summary.tasks_error: lines.append( f"- **Error Share**: {_fmt_rate(summary.error_share)} of tasks never produced a " @@ -389,6 +397,7 @@ def _summary_section_lines(summary: RunSummary) -> list[str]: f"- **Succeeded**: {summary.tasks_succeeded}", failed_line, f"- **Errors**: {summary.tasks_error}", + *([f"- **Not Graded**: {summary.tasks_not_graded}"] if summary.tasks_not_graded else []), *_pass_rate_lines(summary), ] diff --git a/src/coder_eval/reports_experiment.py b/src/coder_eval/reports_experiment.py index e141174b..e3cdd95f 100644 --- a/src/coder_eval/reports_experiment.py +++ b/src/coder_eval/reports_experiment.py @@ -626,7 +626,10 @@ def generate_variant_report(variant_id: str, result: ExperimentResult, run_dir: f"- **Succeeded**: {agg.tasks_succeeded}", failed_line, f"- **Errors**: {agg.tasks_error}", - f"- **Pass Rate**: {pass_rate_str} ({agg.tasks_succeeded}/{agg.tasks_run})", + *([f"- **Not Graded**: {agg.tasks_not_graded}"] if agg.tasks_not_graded else []), + # Denominator is the GRADED count, matching VariantAggregate.pass_rate — + # an ungraded task was never measured and belongs on neither side. + f"- **Pass Rate**: {pass_rate_str} ({agg.tasks_succeeded}/{agg.tasks_graded})", f"- **Average Score**: {agg.average_score:.3f}", f"- **Average Duration**: {agg.average_duration:.1f}s", f"- **Total Tokens**: {tokens_str}", diff --git a/src/coder_eval/reports_html.py b/src/coder_eval/reports_html.py index def6591f..7f5d4468 100644 --- a/src/coder_eval/reports_html.py +++ b/src/coder_eval/reports_html.py @@ -283,7 +283,10 @@ def _status_badge(status: Any) -> str: status_str = getattr(status, "value", None) or str(status) try: fs = status if isinstance(status, FinalStatus) else FinalStatus(str(status)) - cls = {"succeeded": "success", "failed": "failure", "error": "error"}[fs.category] + # "ungraded" -> neutral: the row carries no verdict, so it must render as + # neither green nor red. Same class an unrecognised status falls back to, + # reached deliberately here rather than by accident. + cls = {"succeeded": "success", "failed": "failure", "error": "error", "ungraded": "neutral"}[fs.category] except (ValueError, KeyError): cls = "neutral" # unknown / non-FinalStatus input return f'{_esc(status_str)}' diff --git a/src/coder_eval/reports_junit.py b/src/coder_eval/reports_junit.py index 133f9187..acc609a7 100644 --- a/src/coder_eval/reports_junit.py +++ b/src/coder_eval/reports_junit.py @@ -54,7 +54,7 @@ def _xml_safe(text: str) -> str: return _ILLEGAL_XML.sub("", text) -def _category_of(status: str) -> Literal["succeeded", "failed", "error"]: +def _category_of(status: str) -> Literal["succeeded", "failed", "error", "ungraded"]: """Map a serialized status string to a reporting category via the SSOT. Goes through ``FinalStatus(value).category`` (an explicit allowlist, CE018); @@ -303,6 +303,14 @@ def _task_case(row: dict[str, Any], run_dir: Path) -> ET.Element: if category == "succeeded": return case + if category == "ungraded": + # `coder-eval execute`: the task ran but was deliberately not scored. + # is JUnit's only "no verdict" element — reporting it as a + # would turn a healthy ungraded run red in CI, and reporting + # it as a pass would invent a verdict. _set_counts already counts these. + ET.SubElement(case, "skipped", {"message": "not graded (coder-eval execute)"}) + return case + message = status if status in _KNOWN_STATUSES else f"unknown status: {status}" tag = "failure" if category == "failed" else "error" child = ET.SubElement(case, tag, {"message": _xml_safe(message)}) diff --git a/tests/lint/rules/ce018_no_final_status_name_denylist.py b/tests/lint/rules/ce018_no_final_status_name_denylist.py index 2e9c12a1..1a3b4f96 100644 --- a/tests/lint/rules/ce018_no_final_status_name_denylist.py +++ b/tests/lint/rules/ce018_no_final_status_name_denylist.py @@ -35,6 +35,7 @@ "MAX_TURNS_EXHAUSTED", "TOKEN_BUDGET_EXCEEDED", "COST_BUDGET_EXCEEDED", + "NOT_GRADED", } ) diff --git a/tests/test_execute_command.py b/tests/test_execute_command.py new file mode 100644 index 00000000..5f0f02da --- /dev/null +++ b/tests/test_execute_command.py @@ -0,0 +1,293 @@ +"""`coder-eval execute` — run without grading. + +Three layers, deliberately: + +* **End-to-end** against the agentless task (`agent: {type: none}`), which needs + no API key and is fully deterministic. This is the only layer that proves the + whole chain — CLI → batch → Orchestrator → task.json → run.json — actually + withholds the verdict while still executing. +* **Contrast** — the same task under `run` must still produce SUCCESS with a real + score. Without it, a totally broken `execute` (or a broken fixture) would pass + the assertions above by accident. +* **Wiring** — the two commands share one body, so a signature or `grade` drift + is caught mechanically rather than by a human noticing. +""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any +from unittest.mock import patch + +import pytest +from typer.testing import CliRunner + +from coder_eval.cli import app +from coder_eval.models import FinalStatus, RunSummary + + +runner = CliRunner() + +# The agentless smoke task: no agent, no model call, and a pre_run that writes a +# file its criteria read back. Executing it must still write that file (proving +# the run really happened) while scoring nothing. +AGENTLESS_TASK = Path("tasks/agentless_smoke_test.yaml") + + +def _invoke(command: str, run_dir: Path) -> Any: + return runner.invoke( + app, + [command, str(AGENTLESS_TASK), "--run-dir", str(run_dir), "--preservation-mode", "MOVE_ON_WRITE"], + ) + + +def _task_json(run_dir: Path) -> dict[str, Any]: + matches = sorted(run_dir.glob("**/task.json")) + assert len(matches) == 1, f"expected exactly one task.json under {run_dir}, got {matches}" + return json.loads(matches[0].read_text(encoding="utf-8")) + + +# -------------------------------------------------------------------------- +# The status itself +# -------------------------------------------------------------------------- + + +def test_not_graded_is_its_own_category() -> None: + """NOT_GRADED must not fold into succeeded/failed/error — each would lie.""" + assert FinalStatus.NOT_GRADED.category == "ungraded" + assert FinalStatus.NOT_GRADED.icon == "?" + + +def test_ungraded_leaves_both_sides_of_the_pass_rate() -> None: + """An all-ungraded run has NO pass rate — not a 0% one.""" + summary = RunSummary( + run_id="r", + start_time="2026-01-01T00:00:00", # type: ignore[arg-type] + end_time="2026-01-01T00:01:00", # type: ignore[arg-type] + total_duration_seconds=60.0, + tasks_run=2, + tasks_succeeded=0, + tasks_failed=0, + tasks_error=0, + tasks_not_graded=2, + task_results=[], + framework_version="test", + ) + assert summary.tasks_graded == 0 + assert summary.pass_rate is None + assert summary.error_share is None + + +def test_ungraded_does_not_dilute_a_partially_graded_run() -> None: + """One pass out of one graded task is 100%, even alongside three ungraded ones.""" + summary = RunSummary( + run_id="r", + start_time="2026-01-01T00:00:00", # type: ignore[arg-type] + end_time="2026-01-01T00:01:00", # type: ignore[arg-type] + total_duration_seconds=60.0, + tasks_run=4, + tasks_succeeded=1, + tasks_failed=0, + tasks_error=0, + tasks_not_graded=3, + task_results=[], + framework_version="test", + ) + assert summary.pass_rate == 1.0 + + +def test_task_count_invariant_counts_the_ungraded_bucket() -> None: + """The fourth bucket is part of the invariant, not a free-floating sub-counter.""" + with pytest.raises(ValueError, match="Task count invariant violated"): + RunSummary( + run_id="r", + start_time="2026-01-01T00:00:00", # type: ignore[arg-type] + end_time="2026-01-01T00:01:00", # type: ignore[arg-type] + total_duration_seconds=1.0, + tasks_run=2, + tasks_succeeded=0, + tasks_failed=0, + tasks_error=0, + tasks_not_graded=1, # 0+0+0+1 != 2 + task_results=[], + framework_version="test", + ) + + +# -------------------------------------------------------------------------- +# End to end +# -------------------------------------------------------------------------- + + +@pytest.mark.skipif(not AGENTLESS_TASK.is_file(), reason="needs a source checkout (tasks/ is not in the wheel)") +def test_execute_runs_the_task_but_grades_nothing(tmp_path: Path) -> None: + run_dir = tmp_path / "run" + result = _invoke("execute", run_dir) + + assert result.exit_code == 0, result.output + + row = _task_json(run_dir) + # The verdict is withheld ... + assert row["final_status"] == FinalStatus.NOT_GRADED.value + assert row["weighted_score"] is None, "must be None, never 0.0 — 0.0 reads as 'graded and scored zero'" + assert row["success_criteria_results"] == [] + # ... but the run itself demonstrably happened: pre_run wrote its file into + # the preserved sandbox. Without this the test would also pass if `execute` + # had simply skipped the task. + proof = sorted(run_dir.glob("**/artifacts/**/proof.txt")) + assert proof, f"pre_run did not run — no proof.txt under {run_dir}" + assert "coder-eval-ran-without-a-coder" in proof[0].read_text(encoding="utf-8") + + +@pytest.mark.skipif(not AGENTLESS_TASK.is_file(), reason="needs a source checkout (tasks/ is not in the wheel)") +def test_execute_run_json_reports_ungraded_not_failed(tmp_path: Path) -> None: + run_dir = tmp_path / "run" + assert _invoke("execute", run_dir).exit_code == 0 + + summary = json.loads((run_dir / "run.json").read_text(encoding="utf-8")) + assert summary["tasks_run"] == 1 + assert summary["tasks_not_graded"] == 1 + # The whole point: an ungraded task is NOT a failure and NOT an error. + assert summary["tasks_failed"] == 0 + assert summary["tasks_error"] == 0 + assert summary["tasks_succeeded"] == 0 + assert summary["pass_rate"] is None + + +@pytest.mark.skipif(not AGENTLESS_TASK.is_file(), reason="needs a source checkout (tasks/ is not in the wheel)") +def test_run_still_grades_the_same_task(tmp_path: Path) -> None: + """The negative control: `run` must still score this task, or the assertions + above prove nothing about grading being *deliberately* skipped.""" + run_dir = tmp_path / "run" + result = _invoke("run", run_dir) + + assert result.exit_code == 0, result.output + row = _task_json(run_dir) + assert row["final_status"] == FinalStatus.SUCCESS.value + assert row["weighted_score"] == 1.0 + assert len(row["success_criteria_results"]) == 2 + + summary = json.loads((run_dir / "run.json").read_text(encoding="utf-8")) + assert summary["tasks_not_graded"] == 0 + assert summary["tasks_succeeded"] == 1 + assert summary["pass_rate"] == 1.0 + + +# -------------------------------------------------------------------------- +# Wiring: one shared body, two commands +# -------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + ("command", "module", "expected_grade"), + [("run", "run_command", True), ("execute", "execute_command", False)], +) +def test_both_commands_call_the_shared_pipeline(command: str, module: str, expected_grade: bool) -> None: + """`run` and `execute` differ ONLY in `grade` — no third code path. + + Patched per module because each command imported ``run_pipeline`` into its own + namespace; patching the defining module would silently miss ``execute``. + """ + with patch(f"coder_eval.cli.{module}.run_pipeline") as pipeline: + result = runner.invoke(app, [command, "a.yaml"]) + assert result.exit_code == 0, result.output + pipeline.assert_called_once() + assert pipeline.call_args.kwargs["grade"] is expected_grade + + +def _option_names(command: str) -> set[str]: + import typer.main + + click_app = typer.main.get_command(app) + cmd = click_app.commands[command] # type: ignore[attr-defined] + return {opt for param in cmd.params for opt in getattr(param, "opts", [])} + + +# `execute` restates `run`'s Typer signature because Typer builds its parser from +# the signature and there is no way to share one. That duplication is the drift +# risk this test exists to close: a flag added to `run` must be added here too, +# or consciously listed below as a deliberate omission. +_DELIBERATELY_ABSENT_FROM_EXECUTE = { + "--resume", # partition_for_resume would treat a NOT_GRADED row as finalized + "--junit-xml", # a report of verdicts, and there are none +} + + +def test_execute_exposes_run_flags_minus_the_two_refused_ones() -> None: + run_opts = _option_names("run") + execute_opts = _option_names("execute") + + missing = run_opts - execute_opts - _DELIBERATELY_ABSENT_FROM_EXECUTE + assert not missing, ( + f"`run` has flag(s) {sorted(missing)} that `execute` lacks. Add them to " + "execute_command's signature, or list them in _DELIBERATELY_ABSENT_FROM_EXECUTE " + "with the reason they are refused." + ) + assert not execute_opts - run_opts, "`execute` must not grow flags of its own" + # The omissions must be real, not stale entries masking a genuine gap. + assert not _DELIBERATELY_ABSENT_FROM_EXECUTE & execute_opts + + +# -------------------------------------------------------------------------- +# The docker boundary +# -------------------------------------------------------------------------- + + +async def _staged_context(tmp_path: Path, *, grade: bool) -> dict[str, Any]: + """Stage a docker task's inputs and read back the context.json the container sees.""" + from coder_eval.isolation.docker_runner import DockerRunner + from coder_eval.models import ResolvedTask, TaskDefinition + + task = TaskDefinition( + task_id="t", + description="d", + initial_prompt="p", + agent={"type": "claude-code"}, + sandbox={"driver": "docker"}, + success_criteria=[{"type": "file_exists", "path": "x.txt", "description": "x"}], + ) + rt = ResolvedTask( + task=task, + task_file=tmp_path / "t.yaml", + run_dir=tmp_path / "run", + variant_id="default", + original_task_id="t", + ) + staged = tmp_path / "input" + staged.mkdir() + await DockerRunner(rt, grade=grade)._stage_inputs(staged) + return json.loads((staged / "context.json").read_text(encoding="utf-8")) + + +@pytest.mark.parametrize("grade", [True, False]) +async def test_docker_forwards_grade_to_the_container(tmp_path: Path, grade: bool) -> None: + """`grade` is a run-level CLI decision, so it is NOT recoverable from the staged + task.yaml on the container side — it has to cross the boundary in context.json. + Without this, `execute --driver docker` would silently grade after all.""" + assert (await _staged_context(tmp_path, grade=grade))["grade"] is grade + + +def test_container_defaults_to_grading_when_the_host_sends_no_key() -> None: + """A host predating `execute` writes no `grade` key; the container must keep + its original (grading) behavior rather than silently withholding verdicts.""" + # The parse is inline in a Typer command that cannot run outside a container, + # so this reads its source. Resolved off the function object because + # `coder_eval.cli` rebinds the submodule's name to the function it exports. + import inspect + + from coder_eval.cli.run_task_internal_command import run_task_internal_command + + source = inspect.getsource(inspect.getmodule(run_task_internal_command)) # type: ignore[arg-type] + assert 'context.get("grade", True)' in source, "the in-container default must be True (grade)" + + +def test_execute_help_explains_the_refused_flags() -> None: + """The two omissions are documented in the help, not silently absent — a user + who reaches for `--resume` needs to learn why it is refused, not just that it + is unrecognised. (Presence as a real *flag* is covered by the option-set test + above; here we only require the help text to mention them.)""" + result = runner.invoke(app, ["execute", "--help"]) + assert result.exit_code == 0 + for flag in _DELIBERATELY_ABSENT_FROM_EXECUTE: + assert flag in result.output, f"execute's help should explain why {flag} is unavailable" diff --git a/tests/test_reports_html.py b/tests/test_reports_html.py index 77e5886a..17c77255 100644 --- a/tests/test_reports_html.py +++ b/tests/test_reports_html.py @@ -37,16 +37,21 @@ ) -_CATEGORY_TO_CLASS = {"succeeded": "success", "failed": "failure", "error": "error"} +# "ungraded" is the one category whose badge is legitimately neutral: the row +# carries no verdict, so it must render as neither green nor red. Listing it +# explicitly (rather than dropping the negative assertion below) keeps the guard +# that no OTHER member falls through to the neutral fallback. +_CATEGORY_TO_CLASS = {"succeeded": "success", "failed": "failure", "error": "error", "ungraded": "neutral"} @pytest.mark.parametrize("status", list(FinalStatus)) def test_status_badge_dispatches_on_category(status: FinalStatus): - """Every FinalStatus member renders a non-neutral, category-correct badge.""" + """Every FinalStatus member renders a category-correct badge, neutral only when intended.""" badge = _status_badge(status) expected_cls = _CATEGORY_TO_CLASS[status.category] assert f'class="badge {expected_cls}"' in badge - assert "neutral" not in badge + if expected_cls != "neutral": + assert "neutral" not in badge # The human-readable label is the status value. assert status.value in badge @@ -1229,16 +1234,20 @@ def test_generation_metrics_breaks_down_crashed_partials(): FinalStatus.MAX_TURNS_EXHAUSTED: "failure", FinalStatus.TOKEN_BUDGET_EXCEEDED: "failure", FinalStatus.COST_BUDGET_EXCEEDED: "failure", + # Neutral on purpose — an ungraded row has no verdict to colour. See + # _CATEGORY_TO_CLASS above. + FinalStatus.NOT_GRADED: "neutral", } @pytest.mark.parametrize("status", list(FinalStatus)) def test_status_badge_maps_every_member_to_its_category(status: FinalStatus): - """Every FinalStatus member gets a non-neutral badge matching its category.""" + """Every FinalStatus member gets a badge matching its category, neutral only when intended.""" badge = _status_badge(status) expected_cls = _EXPECTED_BADGE_CLASS[status] assert f'class="badge {expected_cls}"' in badge - assert "neutral" not in badge + if expected_cls != "neutral": + assert "neutral" not in badge assert status.value in badge From 39db36108c5f421aea0ec02e2ca6b559605a3843 Mon Sep 17 00:00:00 2001 From: Akshaya Shanbhogue Date: Thu, 3 Sep 2026 13:42:41 -0700 Subject: [PATCH 2/6] =?UTF-8?q?feat(cli):=20grade=20an=20executed=20run=20?= =?UTF-8?q?afterwards=20=E2=80=94=20`evaluate=20`=20+=20`Sandbo?= =?UTF-8?q?x.adopt`?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `coder-eval execute` withholds the verdict; this closes the loop by letting `coder-eval evaluate` supply it later, and fixes a pre-existing bug that made the copy-based grading path score real files as missing. ## `evaluate` takes two shapes Told apart by a pure resolver (`cli/evaluate_target.py`) on one probe: a target holding `task.json` is a run directory. coder-eval evaluate tasks/hello.yaml ./my_solution # unchanged coder-eval evaluate ./r/default/hello/00 # re-grade a finished run coder-eval execute tasks/hello.yaml --run-dir ./r coder-eval evaluate ./r/default/hello/00 coder-eval aggregate ./r # run.json now reports the verdict Passing a task file OVER a run directory re-grades it with different criteria, reusing the trajectory and workspace of a run you already paid for. ## Re-grading must describe the run that happened Run-dir mode rebuilds the task from the run's own `task_config.resolved`, NOT by re-reading the YAML. `resolved` is post-merge, so variant overrides, -D flags and dataset row expansion are already baked in; re-loading the source would silently grade a different task. Falling back to `source_file` happens only when `resolved` no longer validates, and says so loudly. `Orchestrator(prior_result=...)` seeds the fresh result via `_seed_from_prior_result`, which carries: - the trajectory — every derived figure (tokens, cost, command_stats, model_used, assistant turns) recomputes from `iterations`, so seeding it reproduces them exactly; - `iteration_count`, which evaluate-only used to flatten to 1; - `early_stop` — LOAD-BEARING. Gate selection is FIRED-ONLY: when it is set the checker gates on the weighted ARMED subset instead of strict-AND. Dropping it would re-grade a truncated trajectory under the full-run gate and flip the verdict; - execution facts (max_turns_exhausted, error_message/details, sdk_options). Grader-host `environment_info` is preserved under a `graded_by` sub-dict rather than overwriting the run's — showing the grader's tool versions as the run's is worse than showing neither. Two further parity fixes, both closing gaps the code already knew about: - `command_base_path` is now persisted by `_sync_sandbox_command_path_with_ agent` and restored in the evaluate-only branch. That method's docstring named "evaluate-only mode" as a known PATH gap; without it a detached grade resolves `run_command` binaries against ambient PATH and can disagree with the run it claims to grade. - `_join_litellm_actual_cost` skips when `prior_result` is set. It keys on a per-Orchestrator nonce the prior turns were never tagged with, so it would match nothing and overwrite already-correct per-turn costs. A re-grade refuses outright on a `reference_digest` mismatch: grading then would score the agent's old work against a new answer key. The verdict is written back into the run's `task.json`, keeping the pre-grade record as `task.execute.json`. That in-place write is what makes plain `coder-eval aggregate ` rebuild a graded run.json with zero new code. ## `Sandbox.adopt` — and the bug it fixes `adopt(workspace)` reuses `setup`'s adoption half but skips every MATERIALIZING step (`_setup_template`, `_generate_cli_recorders`, venv/package installs, the destructive $HOME remediation), running only non-mutating derivation: mock-dir +x, venv *discovery*, plugin-tools pin. `_cleanup_on_exit` stays False, so an adopted tree is never moved or deleted. In-place is MORE CORRECT, not merely faster. `_setup_template` filters its copy through `_should_ignore_template_file`, whose default list drops node_modules, dist, build, .venv and .git. So `evaluate` today scores a file that is plainly there as missing: copy path: Score 0.00 "File 'node_modules/x/a.js' does not exist" in place: Score 1.00 "File 'node_modules/x/a.js' exists" That is a pre-existing defect independent of `execute`. Defaults: in-place for a run directory (it is the run's own output), copy for a bare work directory (criteria can mutate it and it is the user's tree); `--in-place` / `--copy` override. `adopt` hard-errors on `driver: docker` — a container workspace is unreachable from the host, so adopting one would grade whatever happens to sit at that host path. ## Also The Typer command is now a thin wrapper over `run_evaluation(...)`, which has real Python defaults — the same split `run`/`execute` use. Calling a Typer command function in-process hands unspecified options an `OptionInfo` sentinel, which silently made `in_place=None` truthy; the existing test_evaluate_command.py calls were the ones that surfaced it. ## Verification `make verify` green (4602 passed, 92.06%). The headline test asserts `execute` + `evaluate` reaches the same status, score and per-criterion results as a single `run` — compared against a real `run` rather than hardcoded values, so a change breaking both paths still fails. Plus: aggregate rebuilds a graded run.json unaided; the trajectory survives the re-grade; the adopted workspace is not moved or deleted; task.execute.json preserves the ungraded record; the original two-argument form still works; adopt writes nothing, deletes nothing, and exposes the filtered directories; and the target resolver is table-tested over every (one arg / two args) x (run dir / plain dir / file / missing) combination. Co-Authored-By: Claude Opus 5 (1M context) --- CLAUDE.md | 3 + docs/USER_GUIDE.md | 53 +++- src/coder_eval/cli/evaluate_command.py | 367 ++++++++++++++++++++++--- src/coder_eval/cli/evaluate_target.py | 99 +++++++ src/coder_eval/orchestrator.py | 84 +++++- src/coder_eval/sandbox.py | 77 ++++++ tests/test_evaluate_command.py | 32 +-- tests/test_evaluate_target.py | 102 +++++++ tests/test_execute_evaluate_loop.py | 147 ++++++++++ tests/test_litellm_cost.py | 6 + tests/test_sandbox_adopt.py | 108 ++++++++ 11 files changed, 1007 insertions(+), 71 deletions(-) create mode 100644 src/coder_eval/cli/evaluate_target.py create mode 100644 tests/test_evaluate_target.py create mode 100644 tests/test_execute_evaluate_loop.py create mode 100644 tests/test_sandbox_adopt.py diff --git a/CLAUDE.md b/CLAUDE.md index 896c2e4b..20f8f5e0 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -92,6 +92,8 @@ coder_eval/ │ ├── run_command.py # `coder-eval run` + `run_pipeline` (the body BOTH run and execute share) │ ├── execute_command.py # `coder-eval execute` — Typer signature only; delegates to run_pipeline(grade=False) │ ├── plan_command.py # `coder-eval plan` +│ ├── evaluate_command.py # `coder-eval evaluate` (grade a dir, or re-grade a run dir) + `run_evaluation` +│ ├── evaluate_target.py # PURE shape detection for evaluate's positionals (run dir ⟺ holds task.json) │ ├── report_command.py # `coder-eval report` │ ├── run_helpers.py # CLI helper functions │ ├── console.py # Rich console instance @@ -148,6 +150,7 @@ action.yml # Published composite GitHub Action (coder-ev - **Harness run-limit parity**: a shared `BaseAgentConfig` field must mean the same thing on every backend, so a divergence is either fixed or documented — never silent. **`run_limits.max_turns` on Codex/Antigravity counts VISIBLE turns** (resolved tool calls, read live off the shared `EventCollector.visible_turn_count`, the same list `TurnRecord.commands` holds) because one `communicate()` is a single SDK turn on both, so a native counter would clamp at 1; claude-code keeps its native SDK cap, whose unit (an agent-loop turn) absorbs arbitrarily many parallel calls — the same number is NOT the same budget across harnesses. OpenCode likewise keeps a native unit — the CLI streams a real multi-step loop per `communicate()` (`step_start`/`step_finish`), so `max_turns: N` allows N complete steps and cuts cleanly when step N+1 begins. The cap is enforced on the same loop boundary as the cooperative early stop and finalizes cleanly as `max_turns_exhausted` (no crash, no retry); on Antigravity that boundary lives in `_drain()`, so the background-work poll loop honors it too. Known unfixed divergences: `permission_mode` on Codex and Antigravity (both run unconfined — the sandbox driver is the isolation boundary), `disallowed_tools` on Codex (forwarded, not SDK-enforced), `allowed_tools`/`disallowed_tools` on Antigravity (not read at all), `turn_timeout` on Antigravity (bounded by an earlier internal poll deadline at 80% of it), `allowed_tools`/`disallowed_tools`/`system_prompt` on OpenCode (no CLI knob; warned at `start()`, not enforced), and **`agent.plugins[].path` depth** — claude-code REQUIRES a plugin root holding `skills/` and silently loads NOTHING from a bare skills directory, while Codex and Antigravity scan both depths and accept either. That is the costly direction: the wrong depth produces no error, every positive row of an activation suite scores 0, and the suite reports recall 0.0, which reads exactly like a skill that never triggers. Held to the plugin-root shape (for `SKILL_SOURCE_PATH` only) by lint rule CE045. `plugins` on OpenCode is **honored for skills**: each local plugin root is mapped to the `skills` dir its `.claude-plugin/plugin.json` declares (default `/skills`, never the root itself — `skills.paths` is scanned recursively and a plugin root can hold a self-referential symlink) and injected via `OPENCODE_CONFIG_CONTENT`, which `--pure` does not suppress; a plugin's agents/hooks/commands/MCP servers are still dropped. Full table + rationale: docs/agents/HARNESS_PARITY.md. - **sandbox isolation**: Tasks that don't need MCP servers should set `setting_sources: []` in their `agent:` block to isolate the sandbox from the host project's CLAUDE.md and settings. Without this, the host project's CLAUDE.md (often 20 KB+) is injected into every API call, inflating cache-creation tokens and cost significantly. - **Execute vs. run (the grading switch)**: `coder-eval execute` is `coder-eval run` with grading removed — the agent runs and the full trajectory is captured, but no criterion is checked, `weighted_score` is `None` (never `0.0`, which would be indistinguishable from "graded and scored zero"), and the row finalizes as **`FinalStatus.NOT_GRADED`**, whose `category` is a **fourth** bucket, `"ungraded"`. Ungraded rows leave BOTH sides of every rate: `RunSummary.pass_rate` / `error_share` and `VariantAggregate.pass_rate` divide by `tasks_graded` (`tasks_run - tasks_not_graded`), and `tasks_not_graded` is part of the sum-to-`tasks_run` invariant, not a `tasks_failed` sub-counter. **Only SUCCESS/FAILURE collapse into it** — `ERROR`, `TIMEOUT`, `BUILD_FAILED`, `MAX_TURNS_EXHAUSTED` and the budget stops are facts about the *run*, not about grading, and still apply (so `execute` still exits non-zero on a crash). The switch is `BatchRunConfig.grade` → `Orchestrator(grade=...)` → the **four** grading call sites (single-shot, evaluate-only, the simulation dialog check, and post-failure diagnostics); it crosses the docker boundary in `context.json` (defaulting to `True` in-container, so a host predating `execute` keeps grading). It is **deliberately not a task-config field** — no 5-layer merge, no `-D` path — because a task YAML must never declare itself ungraded; only the invoking command decides. `run` and `execute` share one body (`run_command.run_pipeline`) and differ solely in that flag, so there is no third code path. Three things are refused rather than degraded: `--junit-xml` (a report of verdicts, and there are none — though `reports_junit` still emits `` for an ungraded row it encounters), `--resume` (`partition_for_resume` treats "has any final status" as finalized, so a `NOT_GRADED` row would be *skipped* by a later `run --resume` instead of graded), and simulation tasks (their turn-continuation logic reads criteria results, so an ungraded dialog would silently change its own stopping behavior). `stop_early:` blocks are inert under `execute` for the same reason the kill switch exists: the full trajectory is the deliverable. Motivating consumer: an external harness (Harbor / Terminal-Bench 2.0) that builds its own container, calls coder-eval as the agent, and grades with its own tests. +- **Detached grading (`evaluate` over a run dir) + `Sandbox.adopt`**: `coder-eval evaluate` takes two shapes, told apart by a **pure** resolver (`cli/evaluate_target.py`) on one probe — a target holding `task.json` is a run directory. Run-dir mode rebuilds the task from the run's own `task_config.resolved`, **not** by re-loading the YAML: `resolved` is post-merge, so variant overrides / `-D` / dataset expansion are already baked in and re-loading the source would silently grade a *different* task (fallback to `source_file` only when `resolved` no longer validates, and loudly). It seeds the fresh result from the prior one via `Orchestrator(prior_result=...)` → `_seed_from_prior_result`, which carries the trajectory (every derived figure — tokens, cost, `command_stats`, `model_used` — recomputes from `iterations`), `iteration_count`, execution facts, and **`early_stop` — load-bearing, because gate selection is FIRED-ONLY**: dropping it re-grades a truncated trajectory under the full-run strict-AND gate and can flip the verdict. Grader-host `environment_info` is preserved under a `graded_by` sub-dict rather than overwriting the run's. Two other parity fixes: `command_base_path` is now persisted into `environment_info` by `_sync_sandbox_command_path_with_agent` and restored in the evaluate-only branch (closing the PATH gap that method's docstring already named), and `_join_litellm_actual_cost` **skips** when `prior_result` is set (its join keys on a per-Orchestrator nonce the prior turns never carried, so it would clobber already-correct costs). The verdict is written back into the run's `task.json`, with the pre-grade record kept as `task.execute.json` — that in-place write is what makes plain `coder-eval aggregate ` rebuild a graded `run.json` with **zero** new code. **`Sandbox.adopt(workspace)`** is the grade-in-place primitive: it reuses `setup`'s adoption half but skips every *materializing* step (`_setup_template`, `_generate_cli_recorders`, venv/package installs, the destructive `$HOME` remediation), running only non-mutating derivation (mock-dir `+x`, venv *discovery*, plugin-tools pin); `_cleanup_on_exit` stays False so an adopted tree is never moved or deleted. In-place is **more correct**, not merely faster: `_setup_template` filters the copy through `_should_ignore_template_file`, which drops `node_modules` / `dist` / `build` / `.venv` / `.git`, so on the copy path a criterion like `test -f dist/bundle.js` fails as a *copying artifact* rather than as a verdict (verified: 0.00 "does not exist" on copy vs 1.00 in place). Defaults: in-place for a run dir, copy for a bare work dir (criteria can mutate it and it is the user's own tree); `--in-place`/`--copy` override. `adopt` hard-errors on `driver: docker` (a container workspace is unreachable from the host) and a re-grade refuses on a `reference_digest` mismatch. NOTE the Typer command is a thin wrapper over `run_evaluation(...)`, which has real Python defaults — calling a Typer command function in-process hands unspecified options an `OptionInfo` sentinel, which silently made `in_place=None` truthy. - **Run-time caps (non-criterion enforcement)**: `TaskDefinition.run_limits` (`RunLimits` model) is the single namespace for all *task-level* run-time caps — `max_turns` / `task_timeout` / `turn_timeout` (structural) and `max_input_tokens` / `max_output_tokens` / `max_total_tokens` / `max_usd` (cumulative budget). Token/USD breaches abort with `FinalStatus.TOKEN_BUDGET_EXCEEDED` or `COST_BUDGET_EXCEEDED` (both `category == "failed"`). Structural caps are set from the CLI via `-D run_limits.max_turns=…` / `-D run_limits.task_timeout=…` / `-D run_limits.turn_timeout=…` (field-merged into `run_limits`); budget caps via `-D run_limits.max_usd=…` etc. or YAML. Layered config uses field-merge — a variant block overrides individual keys without replacing the task's block. The one *per-criterion* cap, `stop_early.decide_within`, deliberately lives on `LiveSuccessCriterion` instead (see below) — the watcher must attribute a decision-step timeout to a specific criterion, which `RunLimits` (task-scoped, criterion-agnostic) cannot express. - **Early stop on criterion (opt-in, per-criterion arming)**: a `stop_early:` block (`StopEarlyPolicy`) on a criterion ends a single-shot run early once the run's **armed** criteria decide the outcome, so a raised `max_turns` isn't wasted on the smoke flavor. The block's PRESENCE is the arming and alone activates the watcher — there is **no run-level master switch**: `run_limits.stop_early: false` is the run-level KILL SWITCH that force-disarms every block (the one-line experiment-variant/`-D` override for an authoritative full run), and `run_limits.stop_early: true` (the removed master arm) is a hard `EarlyStopConfigError` at resolution. The block exists on `LiveSuccessCriterion` only (currently `skill_triggered`, `command_executed` — so arming an unobservable criterion is unrepresentable, a pydantic extra-forbid error). Arming carries one implicit trigger (a native live-fail may fail-stop the run); its keys refine it: `on_pass: stop` (pass-stop the moment the criterion live-passes; default `continue` just latches) and `decide_within: N` (still undecided after N tool-call steps latches an **effective fail**, fed through the same fail-stop rule, reported as `decision_budget_exceeded` — an ordinary weighted fail, NOT a gate-bypassing force-fail; cumulative across retry attempts of the same turn). A trigger whose polarity the instance can't decide (per the abstract, checker-independent `live_decidable_polarities()`, a pure function of the criterion's own fields, paired with the checker's `live_verdict` override by lint rule CE025, a registry-based whole-tree check) is **inert by design** — one dataset-fanned YAML line serves both positive rows (pass/timeout live) and distractor rows (fail live). Verdicts **latch**: once a criterion decides, its `live_verdict` is never polled again. Stop rule is weighted, not strict-boolean: `run_limits.stop_early_gate_threshold` (default `1.0`, reproducing strict-AND behavior exactly) is the minimum weighted score (`Σ weight·score / Σ weight` over the armed subset) required to pass; a fail-stop fires once the armed set's **ceiling** (best case for everything still undecided) can no longer reach the threshold — so a low-weight fail or timeout that can't doom the gate is absorbed and the run continues — and is **deferred while any pass-capable armed criterion is undecided** (a distractor misfire never truncates a positive row's recall signal); a pass-stop fires once the `on_pass: stop` subset's **floor** (worst case) already meets the threshold, and is symmetrically **deferred while any pass-capable armed criterion outside the `on_pass: stop` subset is undecided** (so an early pass never freezes a sibling `on_pass: continue` criterion's signal out of the trajectory). A fail-stop is therefore verdict-preserving; a pass-stop can miss a *later* distractor misfire, so authoritative P/R/F1 comes from a kill-switched (`stop_early: false`) run. Driven by `orchestration/early_stop.py::EarlyStopWatcher` (built when `early_stop_active(task)`: ≥1 armed criterion, kill switch not thrown) through the agent's cooperative `should_stop` seam (tool-call granularity, no SIGKILL); live verdicts only *trigger* the stop — the standard `check_all_async` on the frozen trajectory is authoritative. Gating is **FIRED-ONLY**: a run the watcher actually cut gates on the **armed subset** via the weighted `EvaluationResult.armed_criteria_passed`; a run that completes naturally — armed or not — gates strict-AND via `all_criteria_passed`, so adding a block never changes the verdict of a run it didn't cut. Note the gate keys on the watcher having FIRED (`result.early_stop is not None`), not on confirmed truncation — an agent that ignores `should_stop`, or a stop firing on the final message, still gates armed-only. Every resolution-time guardrail violation is a hard error at resolution (plan *and* run); the one load-time case — a `stop_early:` block on a non-live criterion — is a pydantic schema error at task load, which the run surface reports as a skipped task like any other malformed task. A runtime verdict bug **fails open** to a full run. Surfaces: `EarlyStopInfo` (incl. `gate_threshold` at stop time), report notes/badges, `stopped_early` run.json rows, `EarlyStopped`/`EarlyStopReason` telemetry dims. Worked rationale: docs/TASK_DEFINITION_GUIDE.md § `stop_early`. No blocks anywhere ⇒ behavior byte-for-byte unchanged. diff --git a/docs/USER_GUIDE.md b/docs/USER_GUIDE.md index 878e72a9..8c4454f3 100644 --- a/docs/USER_GUIDE.md +++ b/docs/USER_GUIDE.md @@ -74,7 +74,7 @@ share one implementation, so they cannot drift apart. Use it when something *else* owns the verdict — an external harness that builds its own container and runs its own tests — or to separate one expensive agent run from grading you want to iterate on afterwards. Grade the results later with -[`coder-eval evaluate`](#coder-eval-evaluate--test-criteria-without-an-agent). +[`coder-eval evaluate`](#coder-eval-evaluate--grade-without-running-an-agent). **Only the verdict is withheld, never the facts of the run.** A crash, timeout, or budget breach still reports `ERROR` / `TIMEOUT` / `TOKEN_BUDGET_EXCEEDED` and still @@ -105,23 +105,58 @@ Checks task syntax, required CLI tools, API keys, and schema validity without ex | --- | --- | | `--experiment, -e` | Experiment definition YAML to resolve variants against (default: `experiments/default.yaml`). | -### `coder-eval evaluate` — test criteria without an agent +### `coder-eval evaluate` — grade without running an agent + +Two shapes, told apart by whether the target holds a `task.json`: ```bash -coder-eval evaluate tasks/hello_date.yaml ./my_solution # evaluate a directory -coder-eval evaluate tasks/hello_date.yaml ./my_solution --preserve # keep the sandbox +# 1. Grade a directory against a task +coder-eval evaluate tasks/hello_date.yaml ./my_solution + +# 2. Re-grade a finished run — including one left NOT_GRADED by `execute` +coder-eval execute tasks/hello_date.yaml --run-dir ./r +coder-eval evaluate ./r/default/hello_date/00 +coder-eval aggregate ./r # run.json now reports the verdict ``` -Runs a task's success criteria against a directory without an agent — useful for -testing criterion definitions, validating task configs, or scoring code that was -already written. +**Run-directory mode** rebuilds the task from the run's own recorded +`task_config.resolved`, not by re-reading the YAML. That is what makes the grade +describe the run that happened: variant overrides, `-D` flags and dataset row +expansion are already baked into `resolved`, so re-loading the source would +silently grade a *different* task. The run's trajectory is restored too, so +criteria that read the agent's tool calls (`command_executed`, `skill_triggered`, +judges with trajectory) score exactly as they would have during the run. + +It writes the verdict back into the run's `task.json` and keeps the pre-grade +record beside it as `task.execute.json`. Writing back in place is what makes +`aggregate` free — no new flag, no second copy of the results. + +Passing a task file **over** a run directory re-grades it with different +criteria, reusing the trajectory and workspace of a run you already paid for: + +```bash +coder-eval evaluate tasks/hello_date.edited.yaml ./r/default/hello_date/00 +``` + +**In-place vs. copy.** The two-argument form copies your directory into a fresh +sandbox (criteria can mutate the target, and it is your own tree). Run-directory +mode grades **in place**, because copying filters build output — `node_modules`, +`dist`, `build`, `.venv`, `.git` are all on the default ignore list, so a +criterion like `test -f dist/bundle.js` would fail as a *copying artifact* +rather than as a verdict. Override either default with `--in-place` / `--copy`. | Flag | Description | | --- | --- | -| `--preserve / --no-preserve` | Preserve sandbox after evaluation (default: preserve) | -| `--run-dir` | Custom run directory (default: auto-generated timestamped dir in `runs/`). | +| `--workspace` | Grade this directory instead of the run's own artifacts (run-directory mode only). | +| `--in-place / --copy` | Grade where the files are, or copy first. Default: in-place for a run directory, copy for a plain work directory. | +| `--preserve / --no-preserve` | Preserve sandbox after evaluation (default: preserve). Ignored when grading in place — an adopted directory is never moved or deleted. | +| `--run-dir` | Where the graded `task.json` lands (default: auto-generated timestamped dir in `runs/`). | | `--verbose, -v` | DEBUG-level logging | +A re-grade refuses to run if the task's `reference:` directory changed since the +run (digest mismatch) — grading then would score the agent's old work against a +new answer key. + ### `coder-eval report` — view results ```bash diff --git a/src/coder_eval/cli/evaluate_command.py b/src/coder_eval/cli/evaluate_command.py index 1e62d886..cdba3fc5 100644 --- a/src/coder_eval/cli/evaluate_command.py +++ b/src/coder_eval/cli/evaluate_command.py @@ -1,6 +1,10 @@ -"""Evaluate command - run criteria against a directory without an agent.""" +"""Evaluate command - run criteria against a directory or re-grade a finished run.""" + +from __future__ import annotations import asyncio +import logging +from dataclasses import dataclass from pathlib import Path import typer @@ -11,6 +15,7 @@ EvaluationResult, FinalStatus, PreservationMode, + TaskDefinition, TemplateDirSource, parse_agent_config, ) @@ -18,21 +23,215 @@ from ..orchestrator import Orchestrator from ..sandbox import Sandbox from .console import console +from .evaluate_target import TASK_JSON, EvaluateMode, EvaluateTarget, EvaluateTargetError, resolve_evaluate_target from .run_helpers import prepare_run_directory +logger = logging.getLogger(__name__) + +# Where a run directory keeps the workspace the agent worked in. The extra +# segment is the task id: preservation writes `artifacts//...`. +ARTIFACTS_DIRNAME = "artifacts" + + +def _load_prior_result(run_dir: Path) -> EvaluationResult: + """Read a finished run's ``task.json``.""" + raw = (run_dir / TASK_JSON).read_text(encoding="utf-8") + try: + return EvaluationResult.model_validate_json(raw) + except ValueError as e: + raise typer.BadParameter(f"{run_dir / TASK_JSON} is not a readable EvaluationResult: {e}") from e + + +def _task_from_prior(prior: EvaluationResult, run_dir: Path) -> tuple[TaskDefinition, str]: + """Rebuild the executed task from the run's own recorded config. + + Rebuilding from ``task_config.resolved`` rather than re-reading the YAML is + what makes the grade describe the run that happened: ``resolved`` is the + post-merge definition, so variant overrides, ``-D`` flags and dataset row + expansion are all already baked in. Re-loading the source YAML would silently + grade a DIFFERENT task whenever any of those were used. + + Falls back to the source YAML only when ``resolved`` will not validate (a + schema change since the run), and says so loudly — a quiet fallback would + reintroduce exactly the drift above. + """ + record = prior.task_config + if record is None: + raise typer.BadParameter( + f"{run_dir / TASK_JSON} carries no task_config, so the executed task cannot be " + + "rebuilt. Pass the task file explicitly: coder-eval evaluate " + ) + try: + return TaskDefinition.model_validate(record.resolved), record.source_yaml + except ValueError as e: + if not record.source_file or not Path(record.source_file).is_file(): + raise typer.BadParameter( + f"The resolved task config in {run_dir / TASK_JSON} no longer validates ({e}), and " + + "its source YAML is unavailable. Pass the task file explicitly." + ) from e + console.print( + f"[yellow]⚠[/] The recorded resolved config does not validate ({e}); falling back to " + + f"{record.source_file}. Variant overrides, -D flags and dataset expansion from the " + + "original run are NOT reapplied, so this grade may not match what ran." + ) + return load_task(Path(record.source_file)) + + +def _default_workspace(run_dir: Path, prior: EvaluationResult) -> Path: + """Locate the workspace a finished run left behind. + + ``sandbox_path`` is authoritative when it still exists — it is where the run + actually worked. Otherwise fall back to the preserved artifacts tree, whose + single child is named for the task. + """ + if prior.sandbox_path: + recorded = Path(prior.sandbox_path) + if recorded.is_dir(): + return recorded + + artifacts = run_dir / ARTIFACTS_DIRNAME + if not artifacts.is_dir(): + raise typer.BadParameter( + f"No workspace to grade: {artifacts} does not exist and the recorded sandbox_path " + + f"({prior.sandbox_path or 'unset'}) is gone. The run was probably made with " + + "--preservation-mode NONE. Point at one explicitly with --workspace." + ) + children = [p for p in sorted(artifacts.iterdir()) if p.is_dir()] + # Preservation nests the workspace under the task id; a flat artifacts dir + # (no subdirectory) means the workspace IS artifacts/. + return children[0] if len(children) == 1 else artifacts + + +def _verify_reference_unchanged(prior: EvaluationResult, task: TaskDefinition) -> None: + """Refuse to grade when the reference tree changed since the run. + + ``reference_comparison`` and reference-carrying judges score against + ``task.reference.directory``. If it moved since the run, the re-grade would + silently measure the agent's old work against a new answer key. + """ + recorded = prior.environment_info.get("reference_digest") + if not isinstance(recorded, str) or task.reference is None: + return + from ..orchestration.evaluation import resolve_reference_dir + from ..path_utils import digest_tree + + resolved = resolve_reference_dir(task, None) + if resolved is None or not resolved.is_dir(): + return + if digest_tree(resolved) != recorded: + raise typer.BadParameter( + f"The reference directory {resolved} changed since this run was executed " + + "(digest mismatch). Grading now would score the agent's work against a " + + "different answer key. Restore the reference, or re-run the task." + ) + + +@dataclass(frozen=True) +class _ResolvedInputs: + """Everything the two positionals + ``--workspace`` decide, resolved once.""" + + target: EvaluateTarget + task: TaskDefinition + source_yaml: str + work_dir: Path + task_file: Path | None + prior: EvaluationResult | None + + +def _resolve_inputs(task_or_run_dir: Path, work_dir: Path | None, workspace: Path | None) -> _ResolvedInputs: + """Turn the CLI positionals into a task, a workspace, and (maybe) a prior run. + + Split out of the command because it is where both shapes converge: after this + the rest of ``evaluate`` is one code path regardless of which form was used. + """ + try: + target = resolve_evaluate_target(task_or_run_dir, work_dir) + except EvaluateTargetError as e: + raise typer.BadParameter(str(e)) from e + + if workspace is not None and target.mode is not EvaluateMode.RUN_DIR: + raise typer.BadParameter( + "--workspace applies to a run directory only; in the two-argument form the " + + "directory to grade is already the second argument." + ) + + prior: EvaluationResult | None = None + if target.mode is EvaluateMode.RUN_DIR: + prior = _load_prior_result(target.target) + if target.task_file is not None: + task, source_yaml = load_task(target.task_file) + console.print(f"[dim]Grading with {target.task_file} (overrides the run's recorded config).[/dim]") + else: + task, source_yaml = _task_from_prior(prior, target.target) + work_dir = workspace or _default_workspace(target.target, prior) + recorded_source = prior.task_config.source_file if prior.task_config else None + task_file = target.task_file or (Path(recorded_source) if recorded_source else None) + else: + assert target.task_file is not None # guaranteed by resolve_evaluate_target + task_file = target.task_file + try: + task, source_yaml = load_task(task_file) + except Exception as e: + console.print(f"[red]✗ Failed to load task:[/red] {e}") + raise typer.Exit(1) from e + work_dir = target.target + + if not work_dir.is_dir(): + console.print(f"[red]✗ Work directory is not a directory:[/red] {work_dir}") + raise typer.Exit(1) + + # Evaluate-only mode bypasses experiment resolution + CLI overrides, so + # `agent` may be None or `agent.type` may be unset for tasks that defer + # those to the experiment / CLI layers. The orchestrator only uses + # `agent.type` for result labeling here (no agent is created), so a + # default is safe. + if task.agent is None: + task.agent = parse_agent_config(type=AgentKind.CLAUDE_CODE) + elif task.agent.type is None: + task.agent = parse_agent_config(**{**task.agent.model_dump(exclude_unset=True), "type": AgentKind.CLAUDE_CODE}) + + if prior is not None: + _verify_reference_unchanged(prior, task) + + return _ResolvedInputs( + target=target, + task=task, + source_yaml=source_yaml, + work_dir=work_dir, + task_file=task_file, + prior=prior, + ) + + def evaluate_command( - task_file: Path = typer.Argument( # noqa: B008 + task_or_run_dir: Path = typer.Argument( # noqa: B008 ..., - help="Path to task YAML file", + metavar="[TASK_FILE] TARGET", + help="Task YAML file, or (when it is the only argument) a finished run directory.", exists=True, ), - work_dir: Path = typer.Argument( # noqa: B008 - ..., - help="Directory containing the code to evaluate", - exists=True, - file_okay=False, - dir_okay=True, + work_dir: Path | None = typer.Argument( # noqa: B008 + None, + metavar="", + help="Directory containing the code to evaluate. Omit when TASK_FILE is a run directory.", + ), + workspace: Path | None = typer.Option( # noqa: B008 + None, + "--workspace", + help=( + "Grade this directory instead of the run's own artifacts. Run-directory mode only (e.g. a verifier's /app)." + ), + ), + in_place: bool | None = typer.Option( + None, + "--in-place/--copy", + help=( + "Grade the workspace where it is, or copy it into a fresh sandbox first. " + "Default: in-place for a run directory, copy for a plain work directory. " + "Copying filters build output (node_modules, dist, build, .venv), so a " + "criterion that reads those needs --in-place." + ), ), verbose: bool = typer.Option( False, @@ -49,43 +248,75 @@ def evaluate_command( run_dir: Path | None = typer.Option( # noqa: B008 None, "--run-dir", - help="Custom run directory (default: auto-generated timestamped directory in runs/)", + help="Where the graded task.json lands (default: auto-generated timestamped directory in runs/)", ), ) -> None: - """Evaluate criteria against a directory without running an agent. + """Evaluate criteria against a directory, or re-grade a finished run. - Runs the success criteria defined in a task against a work directory. - Artifacts are saved to a run directory when --preserve is used. + Two shapes, told apart by whether the target holds a task.json: - Examples: + \b + Grade a directory against a task (no agent runs): coder-eval evaluate tasks/hello.yaml ./my_solution - coder-eval evaluate tasks/test.yaml /path/to/code --preserve - coder-eval evaluate tasks/test.yaml /path/to/code --run-dir ./my_eval_run + + \b + Re-grade a finished run — including one produced by `coder-eval execute`, + which leaves every task NOT_GRADED. The run's own task.json supplies the + resolved config AND the trajectory, so criteria that read the agent's tool + calls score exactly as they would have during the run: + coder-eval execute tasks/hello.yaml --run-dir ./r + coder-eval evaluate ./r/default/hello/00 + + \b + Iterate on criteria against a run you already paid for, by passing a task + file over a run directory (its trajectory and workspace are still used): + coder-eval evaluate tasks/hello.edited.yaml ./r/default/hello/00 """ - setup_logging(verbose=verbose) + run_evaluation( + task_or_run_dir=task_or_run_dir, + work_dir=work_dir, + workspace=workspace, + in_place=in_place, + verbose=verbose, + preserve=preserve, + run_dir=run_dir, + ) - console.print("\n[bold]Evaluating Criteria[/bold]\n") - try: - task, source_yaml = load_task(task_file) - except Exception as e: - console.print(f"[red]✗ Failed to load task:[/red] {e}") - raise typer.Exit(1) from e +def run_evaluation( + *, + task_or_run_dir: Path, + work_dir: Path | None = None, + workspace: Path | None = None, + in_place: bool | None = None, + verbose: bool = False, + preserve: bool = True, + run_dir: Path | None = None, +) -> None: + """The body of ``coder-eval evaluate``, with real Python defaults. - # Evaluate-only mode bypasses experiment resolution + CLI overrides, so - # `agent` may be None or `agent.type` may be unset for tasks that defer - # those to the experiment / CLI layers. The orchestrator only uses - # `agent.type` for result labeling here (no agent is created), so a - # default is safe. + Split from the Typer signature so it is directly callable: invoking a Typer + command function in-process hands every unspecified option an ``OptionInfo`` + sentinel rather than its default, which silently turns ``in_place=None`` into + a truthy object. Callers (tests, and any library use) call this instead. + """ + setup_logging(verbose=verbose) - if task.agent is None: - task.agent = parse_agent_config(type=AgentKind.CLAUDE_CODE) - elif task.agent.type is None: - task.agent = parse_agent_config(**{**task.agent.model_dump(exclude_unset=True), "type": AgentKind.CLAUDE_CODE}) + console.print("\n[bold]Evaluating Criteria[/bold]\n") - if not work_dir.is_dir(): - console.print(f"[red]✗ Work directory is not a directory:[/red] {work_dir}") - raise typer.Exit(1) + inputs = _resolve_inputs(task_or_run_dir, work_dir, workspace) + task = inputs.task + source_yaml = inputs.source_yaml + graded_dir = inputs.work_dir + task_file = inputs.task_file + prior = inputs.prior + target = inputs.target + + # In-place is the default for a run directory: that workspace is the run's + # own output and copying it would filter build artifacts out of the grade. + # A plain work directory defaults to copying, because criteria can mutate the + # target and it is the user's own tree. + grade_in_place = in_place if in_place is not None else (target.mode is EvaluateMode.RUN_DIR) try: prepared_run_dir = prepare_run_directory(run_dir) @@ -93,27 +324,38 @@ def evaluate_command( console.print(f"[red]✗ Failed to prepare run directory:[/red] {e}") raise typer.Exit(1) from e - # Build a sandbox pre-loaded with the work_dir contents, then run evaluate-only sandbox_config = task.sandbox.model_copy(deep=True) - template_source = TemplateDirSource(path=str(work_dir.resolve())) - if sandbox_config.template_sources: - sandbox_config.template_sources = [template_source, *sandbox_config.template_sources] - else: - sandbox_config.template_sources = [template_source] + if not grade_in_place: + # Copy path: preload the sandbox with the work dir as a template source. + template_source = TemplateDirSource(path=str(graded_dir.resolve())) + sandbox_config.template_sources = [template_source, *(sandbox_config.template_sources or [])] + # Grading never runs a container: the docker driver dispatches through + # DockerRunner, which needs an agent. Force tempdir so a task whose YAML says + # `driver: docker` is still gradeable on the host. + sandbox_config = sandbox_config.model_copy(update={"driver": "tempdir"}) - task_dir = task_file.parent.resolve() + task_dir = task_file.parent.resolve() if task_file is not None else None sandbox = Sandbox(sandbox_config, task_id=task.task_id, task_dir=task_dir) async def _setup_and_run() -> EvaluationResult: - await asyncio.to_thread(sandbox.setup) + if grade_in_place: + await asyncio.to_thread(sandbox.adopt, graded_dir) + else: + await asyncio.to_thread(sandbox.setup) orchestrator = Orchestrator( task=task, run_dir=prepared_run_dir, - preservation_mode=PreservationMode.MOVE_ON_WRITE if preserve else PreservationMode.NONE, + # An adopted directory is the caller's; never move or delete it. + preservation_mode=( + PreservationMode.NONE + if grade_in_place + else (PreservationMode.MOVE_ON_WRITE if preserve else PreservationMode.NONE) + ), task_file=task_file, sandbox=sandbox, - variant_id="evaluate", + variant_id=prior.variant_id if prior is not None else "evaluate", source_yaml=source_yaml, + prior_result=prior, ) return await orchestrator.run() @@ -162,6 +404,13 @@ async def _setup_and_run() -> EvaluationResult: if result.sandbox_path: console.print(f"[dim]Artifacts: {result.sandbox_path}[/dim]") + if prior is not None: + console.print( + f"[dim]Re-graded {prior.final_status.value} → {result.final_status.value} " + + f"over {len(result.iterations)} recorded turn(s).[/dim]" + ) + _write_back(target.target, result) + if result.final_status == FinalStatus.ERROR: console.print(f"\n[red]✗ Evaluation error: {result.error_message}[/red]") raise typer.Exit(1) @@ -171,3 +420,31 @@ async def _setup_and_run() -> EvaluationResult: else: console.print(f"\n[red]{failed} criterion/criteria failed.[/red]") raise typer.Exit(1) + + +def _write_back(run_dir: Path, result: EvaluationResult) -> None: + """Replace the graded run's ``task.json`` with the verdict, keeping a copy of the original. + + Updating in place is what makes the rest of the toolchain free: plain + ``coder-eval aggregate `` then rebuilds ``run.json`` from these rows + with no new code, and every report and evalboard view reads the graded row. + + The pre-grade original is kept alongside as ``task.execute.json`` so the + ungraded record is auditable — the write is not a silent overwrite of the + only evidence that the run was executed separately. + """ + target = run_dir / TASK_JSON + backup = run_dir / "task.execute.json" + try: + if not backup.exists(): + backup.write_text(target.read_text(encoding="utf-8"), encoding="utf-8") + target.write_text(result.model_dump_json(indent=2), encoding="utf-8") + except OSError as e: + # Never fail the grade over the write-back: the verdict was computed and + # already printed, and the fresh run dir holds its own task.json. + console.print(f"[yellow]⚠[/] Could not update {target}: {e}") + return + console.print( + f"[dim]Updated {target} (original kept as {backup.name}); " + + "run `coder-eval aggregate` to refresh run.json.[/dim]" + ) diff --git a/src/coder_eval/cli/evaluate_target.py b/src/coder_eval/cli/evaluate_target.py new file mode 100644 index 00000000..d8039a4e --- /dev/null +++ b/src/coder_eval/cli/evaluate_target.py @@ -0,0 +1,99 @@ +"""Shape detection for ``coder-eval evaluate``'s positional arguments. + +``evaluate`` accepts two shapes that look alike on the command line: + + coder-eval evaluate tasks/hello.yaml ./my_solution # grade a directory + coder-eval evaluate runs/latest/default/hello/00 # re-grade a finished run + +Both are "a task and a place", but the second carries its own task config and +trajectory inside ``task.json``, so nothing needs to be supplied twice. Rather +than adding a ``--run-dir-mode`` flag the caller has to remember, the shape is +detected from the target: a directory holding ``task.json`` is a run directory. + +The logic lives here, apart from the Typer command, because it is pure — it does +one ``is_file`` probe and otherwise just maps arguments to a decision — so it can +be tested exhaustively without building sandboxes or invoking a CLI runner. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from enum import StrEnum +from pathlib import Path + + +TASK_JSON = "task.json" + + +class EvaluateMode(StrEnum): + """Which of the two shapes the caller asked for.""" + + WORK_DIR = "work_dir" + """Grade a plain directory against a task file. The original behavior.""" + + RUN_DIR = "run_dir" + """Re-grade a finished run: its task.json supplies config and trajectory.""" + + +@dataclass(frozen=True) +class EvaluateTarget: + """The resolved intent behind ``evaluate``'s positional arguments.""" + + mode: EvaluateMode + target: Path + """The run directory (RUN_DIR) or the directory to grade (WORK_DIR).""" + + task_file: Path | None + """Explicit task YAML. Required in WORK_DIR mode; an optional override in RUN_DIR mode.""" + + +class EvaluateTargetError(ValueError): + """The two positionals do not describe either supported shape.""" + + +def is_run_dir(path: Path) -> bool: + """Whether ``path`` is a finished task run directory (it holds ``task.json``).""" + return (path / TASK_JSON).is_file() + + +def resolve_evaluate_target(first: Path, second: Path | None) -> EvaluateTarget: + """Map ``evaluate``'s one-or-two positionals onto a mode + target. + + Args: + first: The first positional — a task file, or a run directory when it is + the only argument. + second: The second positional (the directory to grade), or None. + + Returns: + The resolved target. + + Raises: + EvaluateTargetError: If the arguments match neither shape. The message + always names what was passed and what to pass instead: a caller who + gets this wrong is one keystroke from the right command, and a bare + "invalid arguments" would not tell them which one. + """ + if second is None: + # One argument: only the run-dir shape is unambiguous. A lone task file + # names no place to grade, and a lone plain directory names no criteria. + if not first.is_dir(): + raise EvaluateTargetError( + f"{first} is not a directory. With a single argument, pass a finished run " + + f"directory (one containing {TASK_JSON}). To grade a directory against a " + + "task, pass both: coder-eval evaluate " + ) + if not is_run_dir(first): + raise EvaluateTargetError( + f"{first} holds no {TASK_JSON}, so it is not a run directory. Pass the task " + + f"file too: coder-eval evaluate {first}" + ) + return EvaluateTarget(mode=EvaluateMode.RUN_DIR, target=first, task_file=None) + + # Two arguments. The second is the place; the first is the task file. When + # that place turns out to be a run directory the caller is re-grading it with + # a DIFFERENT task file than the one it ran with — the "iterate on my + # criteria against an expensive run I already paid for" case, which is the + # main reason to keep `execute` and `evaluate` separate at all. Allow it, and + # let the caller be told which config won. + mode = EvaluateMode.RUN_DIR if second.is_dir() and is_run_dir(second) else EvaluateMode.WORK_DIR + return EvaluateTarget(mode=mode, target=second, task_file=first) diff --git a/src/coder_eval/orchestrator.py b/src/coder_eval/orchestrator.py index 22092c17..0549e7b7 100644 --- a/src/coder_eval/orchestrator.py +++ b/src/coder_eval/orchestrator.py @@ -364,6 +364,7 @@ def __init__( replicate_index: int = 0, workspace_dir: Path | None = None, grade: bool = True, + prior_result: EvaluationResult | None = None, ): """Initialize the orchestrator. @@ -393,6 +394,11 @@ def __init__( deliberately NOT a task-config field — a task YAML must never be able to declare itself ungraded — so it arrives only from ``BatchRunConfig.grade``, never from the 5-layer merge or -D. + prior_result: A completed run's ``EvaluationResult`` to re-grade + (evaluate-only mode). Its trajectory and execution facts are + carried onto the fresh result so the grade describes the run that + actually happened instead of an empty one — see + ``_seed_from_prior_result`` for the field-by-field rationale. """ self.task = task self.run_dir = run_dir @@ -412,6 +418,7 @@ def __init__( self.config_lineage = config_lineage or {} self.replicate_index = replicate_index self.grade = grade + self.prior_result = prior_result # Derived paths self.report_path = self.run_dir / "task.json" @@ -540,6 +547,8 @@ async def run(self) -> EvaluationResult: environment_info=get_version_info(), ) + self._seed_from_prior_result() + # Calculate task log path task_log_file = task_log_path(self.run_dir) task_log_file.parent.mkdir(parents=True, exist_ok=True) # noqa: CE002 — mkdir on local FS is nanoseconds @@ -717,6 +726,51 @@ def _kill_agent_subprocess_sync() -> None: return self.result + def _seed_from_prior_result(self) -> None: + """Carry a completed run's execution facts onto this re-grade's result. + + ``execute`` then ``evaluate`` must equal a single ``run``. Everything + below is a fact the AGENT phase established that the grading phase cannot + re-derive; leaving any of them at their defaults would publish a row that + silently disagrees with the run it grades. + + Deliberately NOT carried: ``final_status``, ``weighted_score`` and + ``success_criteria_results`` — those are exactly what this pass recomputes + — and the timestamps/duration, which describe the grading pass. + """ + prior = self.prior_result + if prior is None or self.result is None: + return + + # The trajectory itself. Every derived figure in _finalize_result — + # token totals, cost, command_stats, model_used, assistant turns — + # recomputes from `iterations`, so seeding it reproduces them exactly. + self.result.iterations = list(prior.iterations) + # Evaluate-only hardcodes 1; a multi-turn run must not be reported as + # single-turn just because the re-grade ran once. + self.result.iteration_count = prior.iteration_count or len(prior.iterations) + + # LOAD-BEARING for the verdict: gate selection is FIRED-ONLY. When + # early_stop is not None the checker gates on the weighted ARMED subset + # instead of strict-AND over every criterion. Dropping it would re-grade a + # truncated trajectory under the full-run gate and flip the verdict. + self.result.early_stop = prior.early_stop + + # Execution facts that outlive the agent process. + self.result.max_turns_exhausted = prior.max_turns_exhausted + self.result.error_message = prior.error_message + self.result.error_details = prior.error_details + self.result.sdk_options = prior.sdk_options + + # environment_info: the prior run's capture describes the machine that + # RAN the task (installed_tools, api route, coder_eval version). Ours + # describes the machine grading it. Prior wins on conflict, and ours is + # preserved wholesale under `graded_by` rather than being interleaved — + # a report that shows the grader's tool versions as the run's is worse + # than one that shows neither. + graded_by = dict(self.result.environment_info) + self.result.environment_info = {**graded_by, **prior.environment_info, "graded_by": graded_by} + async def _run_evaluation_with_failure_evidence( self, *, @@ -1166,6 +1220,14 @@ def _join_litellm_actual_cost(self) -> None: """ if not (isinstance(self.route, LiteLLMRoute) and settings.litellm_cost_log and self.result is not None): return + if self.prior_result is not None: + # Re-grading someone else's trajectory. The join keys on THIS + # Orchestrator's per-attempt nonce, which the original turns were + # never tagged with, so it would match nothing and overwrite the + # already-corrected per-turn costs with a warning about a missing + # bill. The prior run's cost is the real one; leave it alone. + logger.debug("Re-grade of a prior trajectory: keeping its recorded cost, skipping the LiteLLM join.") + return try: applied = apply_actual_cost( self.result, @@ -1340,6 +1402,14 @@ async def _setup(self) -> None: self.sandbox.reference_dir = self._reference_dir self.result.sandbox_path = str(self.sandbox.sandbox_dir) + # PATH parity with the run being graded. _sync_sandbox_command_path_ + # with_agent recorded the agent's effective PATH; no agent runs here, + # so restore it explicitly or `run_command` criteria resolve binaries + # against ambient PATH and can disagree with the original verdict. + restored_path = self.result.environment_info.get("command_base_path") + if isinstance(restored_path, str) and restored_path: + self.sandbox.set_command_base_path(restored_path) + self._resolve_routes() self._record_route_environment_info() return @@ -1511,6 +1581,13 @@ def _sync_sandbox_command_path_with_agent(self) -> None: path = sdk_env.get("PATH") if isinstance(path, str) and path: self.sandbox.set_command_base_path(path) + # Persist it so a LATER detached grade (`coder-eval evaluate` over a + # finished run dir) can restore the same PATH. Without this the + # "evaluate-only mode" gap named above is permanent: the re-grade + # would resolve `run_command` criteria against ambient PATH and could + # reach a different verdict than the run it claims to be grading. + if self.result is not None: + self.result.environment_info["command_base_path"] = path def _eval_route_overrides(self) -> EvalRouteOverrides: """The ``(backend, model)`` pair from ``task.checker_context.api_route``, if any. @@ -1952,7 +2029,12 @@ async def _evaluation_loop(self) -> bool: "Criteria %s require agent execution; results may be incomplete with no agent", unsupported, ) - self.result.iteration_count = 1 + # A bare `evaluate ` has no trajectory, so one nominal + # iteration stands for the single grading pass. A re-grade seeded + # from a prior result already carries the real count (and the turns + # the trajectory-reading criteria need) — do not flatten it to 1. + if self.prior_result is None: + self.result.iteration_count = 1 # Load reference in evaluate-only mode too: judge criteria with # include_reference=true expect this populated even when no agent # runs. The agent-driven branch below has the same call. diff --git a/src/coder_eval/sandbox.py b/src/coder_eval/sandbox.py index 55a24b5a..3fbead1b 100644 --- a/src/coder_eval/sandbox.py +++ b/src/coder_eval/sandbox.py @@ -266,6 +266,83 @@ def setup(self, target_dir: Path | None = None) -> Path: ) raise ValueError(f"Unsupported sandbox driver: {self.config.driver}") + def adopt(self, workspace: Path) -> Path: + """Use ``workspace`` **as** the sandbox, materializing nothing into it. + + The grade-in-place counterpart to :meth:`setup`. ``setup`` builds a + workspace: it copies template sources in, generates ``record_cli`` shims, + creates a venv, installs packages. ``adopt`` takes a workspace that + already exists — an ``execute`` run's artifacts, or a verifier's ``/app`` + — and only derives the *environment* the criteria need to run against it + (mock-dir ``+x``, venv discovery, the plugin-tools pin). + + Why not ``setup(target_dir=workspace)``: that already adopts a + caller-supplied directory and sets ``_cleanup_on_exit=False``, but it + then runs ``_setup_template()``, which would write over the very files + it was asked to grade. + + In-place is more CORRECT here, not merely faster: + + * ``_setup_template`` filters what it copies through + ``_should_ignore_template_file`` — ``node_modules``, ``dist``, + ``build``, ``venv``, ``.git`` and friends are dropped. A criterion like + ``test -f dist/bundle.js`` therefore fails as a *copying artifact* + rather than as a verdict on the agent's work. + * ``run_command`` criteria execute with ``cwd = sandbox_dir``, so on the + copy path they see the copy's paths, not the ones the agent worked at. + * Copying a real workspace costs minutes. + + The caller keeps ownership: ``_cleanup_on_exit`` stays False, so + ``cleanup()`` never deletes an adopted directory. Criteria CAN still + mutate it (a ``run_command`` that writes), which is why the copy path + remains the default for a bare user-supplied work dir. + + Args: + workspace: An existing directory to grade in place. + + Returns: + Path to the sandbox directory (``workspace``). + + Raises: + RuntimeError: If the driver is ``docker`` (a container workspace is + not reachable from the host), or ``workspace`` is not a directory. + """ + if self.config.driver == "docker": + raise RuntimeError( + "Sandbox.adopt() is host-side only; a driver='docker' workspace lives inside " + + "the container. Grade it from within the container, or copy it out first." + ) + if not workspace.is_dir(): + raise RuntimeError(f"Cannot adopt {workspace}: not an existing directory") + + self.sandbox_dir = workspace.resolve() + # Never flipped True: an adopted directory belongs to the caller. + self._cleanup_on_exit = False + + # Only NON-materializing steps below. Deliberately skipped, and why: + # _setup_template would overwrite the workspace being graded + # _generate_cli_recorders writes shims into it + # _setup_virtualenv / + # _install_*_packages the execute phase already provisioned these; + # re-running mutates the graded tree + # _maybe_remediate_home_plugins_pollution + # destructive on $HOME, and it is remediation + # rather than derivation — the execute phase + # already ran it if it was enabled + self._prepare_mock_path_dirs() + + # Discover an existing venv instead of creating one, so `run_command` + # criteria get the same VIRTUAL_ENV/PATH the agent had. Absent venv -> + # None, exactly as for a task with no python config. + candidate = self.sandbox_dir / ".venv" + if candidate.is_dir(): + self.venv_dir = candidate + + self._check_parent_node_modules_contamination() + self._refresh_plugin_tools_dir() + + return self.sandbox_dir + def _setup_tempdir(self, target_dir: Path | None = None) -> Path: """Set up a sandbox directory. diff --git a/tests/test_evaluate_command.py b/tests/test_evaluate_command.py index 67a98046..06d26f3a 100644 --- a/tests/test_evaluate_command.py +++ b/tests/test_evaluate_command.py @@ -20,7 +20,7 @@ def test_evaluate_command_success(tmp_path): (work_dir / "app.py").write_text("print('hello')") # Import and run command - from coder_eval.cli.evaluate_command import evaluate_command + from coder_eval.cli.evaluate_command import run_evaluation run_dir = tmp_path / "run" run_dir.mkdir() @@ -28,7 +28,7 @@ def test_evaluate_command_success(tmp_path): with patch("coder_eval.cli.console.console.print"), patch("coder_eval.logging_config.setup_logging"): # Should not raise - all criteria pass with pytest.raises(typer.Exit) as exc_info: - evaluate_command(task_file=task_file, work_dir=work_dir, run_dir=run_dir) + run_evaluation(task_or_run_dir=task_file, work_dir=work_dir, run_dir=run_dir) assert exc_info.value.exit_code == 0 @@ -59,14 +59,14 @@ def test_evaluate_command_defaults_agent_type_when_missing(tmp_path): work_dir.mkdir() (work_dir / "app.py").write_text("print('hello')") - from coder_eval.cli.evaluate_command import evaluate_command + from coder_eval.cli.evaluate_command import run_evaluation run_dir = tmp_path / "run" run_dir.mkdir() with patch("coder_eval.cli.console.console.print"), patch("coder_eval.logging_config.setup_logging"): with pytest.raises(typer.Exit) as exc_info: - evaluate_command(task_file=task_file, work_dir=work_dir, run_dir=run_dir) + run_evaluation(task_or_run_dir=task_file, work_dir=work_dir, run_dir=run_dir) assert exc_info.value.exit_code == 0 @@ -85,7 +85,7 @@ def test_evaluate_command_maps_preserve_to_mode(tmp_path, preserve, expected_mod run_dir = tmp_path / "run" run_dir.mkdir() - from coder_eval.cli.evaluate_command import evaluate_command + from coder_eval.cli.evaluate_command import run_evaluation captured: dict[str, PreservationMode] = {} @@ -105,7 +105,7 @@ async def run(self): patch("coder_eval.cli.evaluate_command.Orchestrator", _CapturingOrchestrator), pytest.raises(_StopError), ): - evaluate_command(task_file=task_file, work_dir=work_dir, run_dir=run_dir, preserve=preserve) + run_evaluation(task_or_run_dir=task_file, work_dir=work_dir, run_dir=run_dir, preserve=preserve) assert captured["mode"] == PreservationMode(expected_mode) @@ -119,7 +119,7 @@ def test_evaluate_command_failure(tmp_path): work_dir.mkdir() # Import and run command - from coder_eval.cli.evaluate_command import evaluate_command + from coder_eval.cli.evaluate_command import run_evaluation run_dir = tmp_path / "run" run_dir.mkdir() @@ -127,7 +127,7 @@ def test_evaluate_command_failure(tmp_path): with patch("coder_eval.cli.console.console.print"), patch("coder_eval.logging_config.setup_logging"): # Should fail - file doesn't exist with pytest.raises(typer.Exit) as exc_info: - evaluate_command(task_file=task_file, work_dir=work_dir, run_dir=run_dir) + run_evaluation(task_or_run_dir=task_file, work_dir=work_dir, run_dir=run_dir) assert exc_info.value.exit_code == 1 @@ -140,14 +140,14 @@ def test_evaluate_command_informational_criterion_does_not_fail_exit(tmp_path): work_dir.mkdir() (work_dir / "app.py").write_text("print('hello')") # gating criterion passes; missing.py absent - from coder_eval.cli.evaluate_command import evaluate_command + from coder_eval.cli.evaluate_command import run_evaluation run_dir = tmp_path / "run" run_dir.mkdir() with patch("coder_eval.cli.console.console.print"), patch("coder_eval.logging_config.setup_logging"): with pytest.raises(typer.Exit) as exc_info: - evaluate_command(task_file=task_file, work_dir=work_dir, run_dir=run_dir) + run_evaluation(task_or_run_dir=task_file, work_dir=work_dir, run_dir=run_dir) # The only gating criterion passed → exit 0, despite the weight=0 miss. assert exc_info.value.exit_code == 0 @@ -163,14 +163,14 @@ def test_evaluate_command_invalid_task_file(tmp_path): work_dir.mkdir() # Import and run command - from coder_eval.cli.evaluate_command import evaluate_command + from coder_eval.cli.evaluate_command import run_evaluation run_dir = tmp_path / "run" run_dir.mkdir() with patch("coder_eval.cli.console.console.print"), patch("coder_eval.logging_config.setup_logging"): with pytest.raises(typer.Exit) as exc_info: - evaluate_command(task_file=task_file, work_dir=work_dir, run_dir=run_dir) + run_evaluation(task_or_run_dir=task_file, work_dir=work_dir, run_dir=run_dir) assert exc_info.value.exit_code == 1 @@ -183,14 +183,14 @@ def test_evaluate_command_invalid_work_dir(tmp_path): work_dir = tmp_path / "nonexistent" # Import and run command - from coder_eval.cli.evaluate_command import evaluate_command + from coder_eval.cli.evaluate_command import run_evaluation run_dir = tmp_path / "run" run_dir.mkdir() with patch("coder_eval.cli.console.console.print"), patch("coder_eval.logging_config.setup_logging"): with pytest.raises(typer.Exit) as exc_info: - evaluate_command(task_file=task_file, work_dir=work_dir, run_dir=run_dir) + run_evaluation(task_or_run_dir=task_file, work_dir=work_dir, run_dir=run_dir) assert exc_info.value.exit_code == 1 @@ -205,14 +205,14 @@ def test_evaluate_command_multiple_criteria(tmp_path): (work_dir / "app.py").write_text("print('hello')") # Import and run command - from coder_eval.cli.evaluate_command import evaluate_command + from coder_eval.cli.evaluate_command import run_evaluation run_dir = tmp_path / "run" run_dir.mkdir() with patch("coder_eval.cli.console.console.print"), patch("coder_eval.logging_config.setup_logging"): with pytest.raises(typer.Exit) as exc_info: - evaluate_command(task_file=task_file, work_dir=work_dir, run_dir=run_dir) + run_evaluation(task_or_run_dir=task_file, work_dir=work_dir, run_dir=run_dir) # All 3 criteria should pass assert exc_info.value.exit_code == 0 diff --git a/tests/test_evaluate_target.py b/tests/test_evaluate_target.py new file mode 100644 index 00000000..1b908419 --- /dev/null +++ b/tests/test_evaluate_target.py @@ -0,0 +1,102 @@ +"""`resolve_evaluate_target` — the shape detection behind `coder-eval evaluate`. + +Pure and exhaustively testable by design: the command accepts two forms that +look alike on the command line, and picking the wrong one silently grades the +wrong thing. Every combination of (one arg / two args) x (run dir / plain dir / +file / missing) is enumerated here. +""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from coder_eval.cli.evaluate_target import ( + EvaluateMode, + EvaluateTargetError, + is_run_dir, + resolve_evaluate_target, +) + + +def _run_dir(tmp_path: Path, name: str = "run") -> Path: + d = tmp_path / name + d.mkdir() + (d / "task.json").write_text(json.dumps({"task_id": "t"}), encoding="utf-8") + return d + + +def _plain_dir(tmp_path: Path, name: str = "work") -> Path: + d = tmp_path / name + d.mkdir() + return d + + +def _file(tmp_path: Path, name: str = "task.yaml") -> Path: + f = tmp_path / name + f.write_text("task_id: t", encoding="utf-8") + return f + + +def test_is_run_dir_keys_on_task_json(tmp_path: Path) -> None: + assert is_run_dir(_run_dir(tmp_path)) + assert not is_run_dir(_plain_dir(tmp_path)) + + +# --- one argument --------------------------------------------------------- + + +def test_lone_run_dir_is_run_dir_mode(tmp_path: Path) -> None: + run = _run_dir(tmp_path) + resolved = resolve_evaluate_target(run, None) + assert resolved.mode is EvaluateMode.RUN_DIR + assert resolved.target == run + assert resolved.task_file is None, "a run dir carries its own config; nothing to supply" + + +def test_lone_task_file_is_rejected_with_the_fix(tmp_path: Path) -> None: + """A task file alone names no place to grade.""" + with pytest.raises(EvaluateTargetError, match="not a directory"): + resolve_evaluate_target(_file(tmp_path), None) + + +def test_lone_plain_dir_is_rejected_with_the_fix(tmp_path: Path) -> None: + """A plain directory alone names no criteria.""" + plain = _plain_dir(tmp_path) + with pytest.raises(EvaluateTargetError) as exc: + resolve_evaluate_target(plain, None) + # The message must name the missing piece AND the corrected command — a + # caller here is one argument away from the right invocation. + assert "task.json" in str(exc.value) + assert "" in str(exc.value) + + +# --- two arguments -------------------------------------------------------- + + +def test_task_file_plus_plain_dir_is_the_original_form(tmp_path: Path) -> None: + """The pre-existing shape must keep resolving exactly as before.""" + task, work = _file(tmp_path), _plain_dir(tmp_path) + resolved = resolve_evaluate_target(task, work) + assert resolved.mode is EvaluateMode.WORK_DIR + assert resolved.target == work + assert resolved.task_file == task + + +def test_task_file_plus_run_dir_re_grades_with_the_given_task(tmp_path: Path) -> None: + """Iterating on criteria against a run you already paid for: the run supplies + the trajectory and workspace, the explicit file supplies the criteria.""" + task, run = _file(tmp_path), _run_dir(tmp_path) + resolved = resolve_evaluate_target(task, run) + assert resolved.mode is EvaluateMode.RUN_DIR + assert resolved.target == run + assert resolved.task_file == task, "the override must survive; it is the whole point of this form" + + +def test_a_nonexistent_second_arg_stays_work_dir_mode(tmp_path: Path) -> None: + """Shape detection must not invent run-dir mode for a path that isn't there; + the command reports the missing directory itself, with a clearer message.""" + resolved = resolve_evaluate_target(_file(tmp_path), tmp_path / "nope") + assert resolved.mode is EvaluateMode.WORK_DIR diff --git a/tests/test_execute_evaluate_loop.py b/tests/test_execute_evaluate_loop.py new file mode 100644 index 00000000..c9c7264b --- /dev/null +++ b/tests/test_execute_evaluate_loop.py @@ -0,0 +1,147 @@ +"""The `execute` -> `evaluate` -> `aggregate` loop. + +`coder-eval execute` withholds the verdict; `coder-eval evaluate ` +supplies it later. The pair only earns its keep if it ends up where a single +`coder-eval run` would have: same status, same score, same criteria, and a +`run.json` the rest of the toolchain can read. + +Everything here runs against the agentless task — deterministic, no API key. +""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any + +import pytest +from typer.testing import CliRunner + +from coder_eval.cli import app +from coder_eval.models import FinalStatus + + +runner = CliRunner() + +AGENTLESS_TASK = Path("tasks/agentless_smoke_test.yaml") + +pytestmark = pytest.mark.skipif( + not AGENTLESS_TASK.is_file(), reason="needs a source checkout (tasks/ is not in the wheel)" +) + + +def _task_dir(run_dir: Path) -> Path: + matches = sorted(p.parent for p in run_dir.glob("**/task.json")) + assert len(matches) == 1, f"expected exactly one task.json under {run_dir}, got {matches}" + return matches[0] + + +def _row(task_dir: Path, name: str = "task.json") -> dict[str, Any]: + return json.loads((task_dir / name).read_text(encoding="utf-8")) + + +def _invoke(args: list[str]) -> Any: + result = runner.invoke(app, args) + assert result.exit_code == 0, f"{args} failed:\n{result.output}" + return result + + +def test_execute_then_evaluate_reaches_the_same_verdict_as_run(tmp_path: Path) -> None: + """The headline guarantee, asserted against a real `run` rather than a + hardcoded expectation — so a change that breaks BOTH paths still fails.""" + direct = tmp_path / "direct" + _invoke(["run", str(AGENTLESS_TASK), "--run-dir", str(direct)]) + expected = _row(_task_dir(direct)) + + split = tmp_path / "split" + _invoke(["execute", str(AGENTLESS_TASK), "--run-dir", str(split)]) + _invoke(["evaluate", str(_task_dir(split))]) + actual = _row(_task_dir(split)) + + assert expected["final_status"] == FinalStatus.SUCCESS.value, "the fixture must actually pass under `run`" + assert actual["final_status"] == expected["final_status"] + assert actual["weighted_score"] == expected["weighted_score"] + assert [c["criterion_type"] for c in actual["success_criteria_results"]] == [ + c["criterion_type"] for c in expected["success_criteria_results"] + ] + assert [c["score"] for c in actual["success_criteria_results"]] == [ + c["score"] for c in expected["success_criteria_results"] + ] + + +def test_evaluate_upgrades_the_row_in_place_and_keeps_the_original(tmp_path: Path) -> None: + run_dir = tmp_path / "r" + _invoke(["execute", str(AGENTLESS_TASK), "--run-dir", str(run_dir)]) + task_dir = _task_dir(run_dir) + assert _row(task_dir)["final_status"] == FinalStatus.NOT_GRADED.value + + _invoke(["evaluate", str(task_dir)]) + + assert _row(task_dir)["final_status"] == FinalStatus.SUCCESS.value + # The pre-grade record survives, so "this run was executed separately" stays + # auditable rather than being silently overwritten. + assert _row(task_dir, "task.execute.json")["final_status"] == FinalStatus.NOT_GRADED.value + + +def test_aggregate_rebuilds_a_graded_run_json_with_no_extra_step(tmp_path: Path) -> None: + """Grading in place is what makes the rest of the toolchain free: the + existing `aggregate` command sees the upgraded rows with no new code.""" + run_dir = tmp_path / "r" + _invoke(["execute", str(AGENTLESS_TASK), "--run-dir", str(run_dir)]) + assert json.loads((run_dir / "run.json").read_text(encoding="utf-8"))["tasks_not_graded"] == 1 + + _invoke(["evaluate", str(_task_dir(run_dir))]) + _invoke(["aggregate", str(run_dir)]) + + summary = json.loads((run_dir / "run.json").read_text(encoding="utf-8")) + assert summary["tasks_not_graded"] == 0 + assert summary["tasks_succeeded"] == 1 + assert summary["pass_rate"] == 1.0 + + +def test_re_grade_carries_the_trajectory_not_an_empty_one(tmp_path: Path) -> None: + """Criteria that read the agent's tool calls (command_executed, + skill_triggered, judges with trajectory) score off `iterations`. A re-grade + that dropped them would silently fail every such criterion.""" + run_dir = tmp_path / "r" + _invoke(["execute", str(AGENTLESS_TASK), "--run-dir", str(run_dir)]) + task_dir = _task_dir(run_dir) + executed = _row(task_dir) + + _invoke(["evaluate", str(task_dir)]) + graded = _row(task_dir) + + assert len(graded["iterations"]) == len(executed["iterations"]) + assert graded["iteration_count"] == executed["iteration_count"] + + +def test_evaluate_does_not_move_or_delete_the_graded_workspace(tmp_path: Path) -> None: + """Run-dir mode adopts the workspace; the caller keeps ownership.""" + run_dir = tmp_path / "r" + _invoke(["execute", str(AGENTLESS_TASK), "--run-dir", str(run_dir)]) + proof = sorted(run_dir.glob("**/artifacts/**/proof.txt")) + assert proof, "fixture precondition: execute preserved a workspace" + + _invoke(["evaluate", str(_task_dir(run_dir))]) + + assert proof[0].is_file(), "the adopted workspace was moved or deleted" + + +def test_evaluate_still_grades_a_plain_directory(tmp_path: Path) -> None: + """The original two-argument form must keep working unchanged.""" + work = tmp_path / "work" + work.mkdir() + (work / "proof.txt").write_text("coder-eval-ran-without-a-coder", encoding="utf-8") + + result = runner.invoke(app, ["evaluate", str(AGENTLESS_TASK), str(work)]) + + assert result.exit_code == 0, result.output + assert "All criteria passed" in result.output + + +def test_evaluate_rejects_workspace_flag_outside_run_dir_mode(tmp_path: Path) -> None: + work = tmp_path / "work" + work.mkdir() + result = runner.invoke(app, ["evaluate", str(AGENTLESS_TASK), str(work), "--workspace", str(work)]) + assert result.exit_code != 0 + assert "run directory only" in result.output diff --git a/tests/test_litellm_cost.py b/tests/test_litellm_cost.py index ea23c8c7..ce40a43b 100644 --- a/tests/test_litellm_cost.py +++ b/tests/test_litellm_cost.py @@ -257,6 +257,8 @@ def test_joins_on_litellm_route(self, tmp_path, monkeypatch): _cost_correlation_run_id=run_id, _cost_attempt_nonce="att1", _log_task_id="calc", + # A live run, not a re-grade: the join applies (see _join_litellm_actual_cost). + prior_result=None, ) orch_mod.Orchestrator._join_litellm_actual_cost(fake) assert fake.result.iterations[0].token_usage.total_cost_usd == 0.09 # static 0.5 overridden @@ -270,6 +272,8 @@ def test_join_never_raises_on_bad_log(self, tmp_path, monkeypatch): _cost_correlation_run_id="R", _cost_attempt_nonce="att1", _log_task_id="calc", + # A live run, not a re-grade: the join applies (see _join_litellm_actual_cost). + prior_result=None, ) orch_mod.Orchestrator._join_litellm_actual_cost(fake) # missing file → no-op, no raise assert fake.result.iterations[0].token_usage.total_cost_usd == 0.5 @@ -294,6 +298,8 @@ def test_run_total_rederives_from_actual_after_join(self, tmp_path, monkeypatch) _cost_correlation_run_id=run_id, _cost_attempt_nonce="att1", _log_task_id="calc", + # A live run, not a re-grade: the join applies (see _join_litellm_actual_cost). + prior_result=None, ) orch_mod.Orchestrator._join_litellm_actual_cost(fake) orch_mod.Orchestrator._aggregate_token_usage(fake) diff --git a/tests/test_sandbox_adopt.py b/tests/test_sandbox_adopt.py new file mode 100644 index 00000000..39fafd9b --- /dev/null +++ b/tests/test_sandbox_adopt.py @@ -0,0 +1,108 @@ +"""`Sandbox.adopt` — grade a workspace in place instead of copying it. + +The behavior that matters is what adopt does NOT do. `setup()` materializes a +workspace (copies templates in, writes shims, builds a venv); `adopt()` takes one +that already exists and only derives the environment around it. A regression that +made adopt materialize anything would overwrite the very files being graded, and +would do so silently — the criteria would still run, just against different +content. So each assertion below pins one thing staying untouched. +""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from coder_eval.models import SandboxConfig +from coder_eval.sandbox import Sandbox + + +def _workspace(tmp_path: Path) -> Path: + """A workspace shaped like an agent's output: real files plus build output.""" + ws = tmp_path / "ws" + (ws / "node_modules" / "pkg").mkdir(parents=True) + (ws / "dist").mkdir() + (ws / "src.py").write_text("print('hi')", encoding="utf-8") + (ws / "node_modules" / "pkg" / "index.js").write_text("module.exports=1", encoding="utf-8") + (ws / "dist" / "bundle.js").write_text("bundled", encoding="utf-8") + return ws + + +def _sandbox(**cfg: object) -> Sandbox: + return Sandbox(SandboxConfig(**cfg), task_id="t") # type: ignore[arg-type] + + +def test_adopt_uses_the_directory_itself(tmp_path: Path) -> None: + ws = _workspace(tmp_path) + sandbox = _sandbox() + assert sandbox.adopt(ws) == ws.resolve() + assert sandbox.sandbox_dir == ws.resolve(), "adopt must not create a copy" + + +def test_adopt_exposes_files_the_copy_path_filters_out(tmp_path: Path) -> None: + """The bug this fixes. `_should_ignore_template_file` drops node_modules, + dist, build and .venv, so on the copy path a criterion like + `test -f dist/bundle.js` fails as a COPYING artifact rather than as a + verdict on the agent's work.""" + ws = _workspace(tmp_path) + sandbox = _sandbox() + sandbox.adopt(ws) + assert sandbox.sandbox_dir is not None + assert (sandbox.sandbox_dir / "dist" / "bundle.js").is_file() + assert (sandbox.sandbox_dir / "node_modules" / "pkg" / "index.js").is_file() + + +def test_adopt_writes_nothing_into_the_workspace(tmp_path: Path) -> None: + """No shims, no venv, no template files — the graded tree is exactly as found.""" + ws = _workspace(tmp_path) + before = {p.relative_to(ws) for p in ws.rglob("*")} + _sandbox().adopt(ws) + assert {p.relative_to(ws) for p in ws.rglob("*")} == before + + +def test_adopt_never_owns_the_directory(tmp_path: Path) -> None: + """cleanup() must not delete a directory the caller handed us.""" + ws = _workspace(tmp_path) + sandbox = _sandbox() + sandbox.adopt(ws) + assert sandbox.is_persistent + sandbox.cleanup() + assert ws.is_dir(), "cleanup deleted an adopted workspace" + assert (ws / "src.py").is_file() + + +def test_adopt_discovers_an_existing_venv_without_creating_one(tmp_path: Path) -> None: + """The execute phase already built the venv; re-creating it would mutate the + graded tree. Discovery keeps `run_command` criteria on the agent's PATH.""" + ws = _workspace(tmp_path) + (ws / ".venv" / "bin").mkdir(parents=True) + sandbox = _sandbox(python={"env_packages": []}) + sandbox.adopt(ws) + assert sandbox.venv_dir == ws.resolve() / ".venv" + + +def test_adopt_leaves_venv_unset_when_there_is_none(tmp_path: Path) -> None: + sandbox = _sandbox(python={"env_packages": []}) + sandbox.adopt(_workspace(tmp_path)) + assert sandbox.venv_dir is None + + +def test_adopt_rejects_the_docker_driver(tmp_path: Path) -> None: + """A container workspace is not reachable from the host, so adopting one + would silently grade whatever happens to sit at that host path.""" + sandbox = _sandbox(driver="docker") + with pytest.raises(RuntimeError, match="host-side only"): + sandbox.adopt(_workspace(tmp_path)) + + +def test_adopt_rejects_a_missing_directory(tmp_path: Path) -> None: + with pytest.raises(RuntimeError, match="not an existing directory"): + _sandbox().adopt(tmp_path / "nope") + + +def test_adopt_rejects_a_file(tmp_path: Path) -> None: + f = tmp_path / "a.txt" + f.write_text("x", encoding="utf-8") + with pytest.raises(RuntimeError, match="not an existing directory"): + _sandbox().adopt(f) From 7c13418ca113d46bda3dc2ad34512ba0aac71121 Mon Sep 17 00:00:00 2001 From: Akshaya Shanbhogue Date: Thu, 3 Sep 2026 14:16:03 -0700 Subject: [PATCH 3/6] feat(cli): make --resume distinguish "executed" from "graded" MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `--resume` decided a task was finished by asking "does task.json carry any final_status". NOT_GRADED is a final status, so `run --resume` over a run produced by `coder-eval execute` reported the tasks already complete, graded nothing, and exited 0: after execute: NOT_GRADED $ coder-eval run --run-dir tmp/res --resume ↻ Resume: 1 task(s) already complete, running 0 remaining Results: 1/1 executed, not graded real exit code: 0 after run --resume: NOT_GRADED ## "Finished" is relative to the resuming command `partition_for_resume(tasks, *, grade)` now returns a four-way `ResumePartition` (to_run / to_grade / prior_results / prior_resolved). A NOT_GRADED row owes `execute` nothing — it finished executing — but owes `run` a grade. Under grade=True those rows route to `to_grade`, where the criteria run against the trajectory and workspace already on disk instead of paying for the agent a second time. That reuse is the entire reason `execute` and `run` are separate commands. The carve-out is ONLY for NOT_GRADED. FAILURE and ERROR stay complete under both commands — resume has never retried failures (delete a task's task.json to force that) — and a parametrized test pins that so the carve-out cannot grow into a general "retry bad rows" rule. `clear_rerun_artifacts` skips `to_grade`, whose artifacts are the very thing being graded. A per-task grading failure is warned and folded back in with its ORIGINAL ungraded result, so one bad row neither aborts the resume nor vanishes from run.json — it stays visible as tasks_not_graded. `grade` joins `_FINGERPRINT_DIFF_EXEMPT`: execute → run --resume is a supported flow, not config drift, and the warning's "already-finalized tasks keep their original-config results" text is actively wrong for it (those rows are re-graded with the current config, which is the point). `execute --resume` is consequently supported and no longer refused. ## One implementation, not two `orchestration/regrade.py` now holds the re-grading core, shared by the resume path and `evaluate`'s run-dir mode. Two copies of "how to re-grade" would drift into two different verdicts for the same run. It raises a plain `RegradeError` that the CLI wraps, since orchestration/ must not import the CLI layer (CE004). ## Fidelity fix caught by writing the test A re-graded row was reporting the GRADING pass's clock. A 10-minute agent run re-graded in 2 seconds would record 2 seconds — and duration_seconds feeds VariantAggregate.average_duration, the report tables and the evalboard, so harness-vs-harness comparisons would have been quietly wrong. A task row describes the TASK, so it now keeps the agent run's `started_at` and `duration_seconds`. The grading pass's own cost is preserved separately as `environment_info["grading_duration_seconds"]` rather than discarded, so a slow judge stays visible. ## Verification `make verify` green (4612 passed, 92.07%). End-to-end: `run --resume` grades what execute left (NOT_GRADED → SUCCESS, pass_rate 1.0) while reporting "running 0 remaining", so the agent demonstrably did not re-run; the trajectory, started_at and duration_seconds all survive; task.execute.json is preserved by this path too; `execute --resume` treats the row as done; and no config-drift warning is emitted. Unit: the four-way partition under both grade values, and the failure-retry guard. Co-Authored-By: Claude Opus 5 (1M context) --- CLAUDE.md | 6 +- docs/USER_GUIDE.md | 39 ++++- src/coder_eval/cli/evaluate_command.py | 132 ++++----------- src/coder_eval/cli/execute_command.py | 30 ++-- src/coder_eval/cli/run_command.py | 76 ++++++++- src/coder_eval/orchestration/batch.py | 61 +++++-- src/coder_eval/orchestration/regrade.py | 205 ++++++++++++++++++++++++ src/coder_eval/orchestrator.py | 18 ++- tests/test_execute_command.py | 1 - tests/test_execute_evaluate_loop.py | 68 ++++++++ tests/test_resume.py | 73 ++++++++- 11 files changed, 571 insertions(+), 138 deletions(-) create mode 100644 src/coder_eval/orchestration/regrade.py diff --git a/CLAUDE.md b/CLAUDE.md index 20f8f5e0..f4e4642e 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -80,10 +80,11 @@ coder_eval/ │ └── timeout.py # Timeout handling (TurnTimeoutError carries optional partial TurnRecord) │ ├── orchestration/ # Batch execution utilities -│ ├── batch.py # Parallel task execution (run_batch + run_batch_resolved) +│ ├── batch.py # Parallel task execution (run_batch) + partition_for_resume/ResumePartition │ ├── config.py # Batch run configuration │ ├── early_stop.py # validate_early_stop guardrails + EarlyStopWatcher (armed live-verdict observer) │ ├── evaluation.py # Reference dir resolution + per-run private staging +│ ├── regrade.py # Grade an already-executed run in place — shared by `evaluate ` and `run --resume` │ ├── experiment.py # ExperimentRunner, resolve_task_for_variant, load_experiment │ └── task_loader.py # YAML task loading │ @@ -149,8 +150,9 @@ action.yml # Published composite GitHub Action (coder-ev - **Reference solutions are directory-only, and shielded (partially) from the agent**: `task.reference` is a single required `directory:` (relative to the task YAML) — the inline `code:` / single-file `file:` forms are gone, because a directory is the only shape that can be permission-gated as a unit; a `model_validator(mode="before")` gives the removed forms a migration error. The orchestrator stages a **per-run private copy** (`orchestration/evaluation.py::stage_reference_dir`, symlinks stripped) into a tempdir, removed in `_cleanup` via `path_utils.rmtree_restrictive` (keyed on `_reference_staging_root`, recorded BEFORE the copy so a failed copy still cleans up; `rmtree(ignore_errors=True)` silently declines on a tree left at 000) and deliberately never preserved into `run_dir/artifacts`. That copy is held at mode `000` for the whole of every `agent.communicate` call via **`Sandbox.set_permissions`**, the driver-aware wrapper over `fs_permissions.py::set_permissions`. Windows **stack**: exiting restores the *enclosing* window's mode, only the outermost exit restores the pre-window mode — that is what makes a mid-turn re-grant (`mode=READ_ONLY_MODE`) expressible, and it covers two windows at the same mode so no refcount is needed. The window is enforced **only inside a docker container** (`Sandbox.enforces_permission_windows`) and is a no-op on the host, where the agent shares our uid. **That gate keys on the `CODER_EVAL_IN_CONTAINER` env var, NOT `sandbox.driver`** — `run_task_internal_command` rewrites `driver: docker` → `tempdir` before building the in-container Orchestrator, so a driver-based gate would silently disable the anti-cheat on exactly the path that needs it (regression-guarded by `TestSandboxDriverGate`); `resolve_reference_dir` gates its `/work/references` branch on the same var for the same reason. The task directory is **not** shielded (`:ro` mount → EROFS, and the same YAML is readable at `/work/input`). Criteria address reference files with the `$REFERENCE_DIR` token (same resolver as `$TASK_DIR`) and the `REFERENCE_DIR` env var for `run_command`; `reference_comparison` names one file via `reference_file`. Docker mounts a throwaway **read-write** copy at `/work/references` (a `:ro` mount cannot be chmod'd — EROFS), masks the in-task-dir original with an empty tmpfs, and drops `DAC_OVERRIDE`/`DAC_READ_SEARCH`. `FOWNER`/`CHOWN` are deliberately **NOT** dropped: the in-container orchestrator that applies the window is the same root process with the same caps, so dropping `FOWNER` breaks *the harness's own* chmod wherever the bind mount preserves a non-root owner (native Linux — verified: `chmod: Operation not permitted`), i.e. exactly where the drop would otherwise bite. A window that cannot be applied is now a hard error, not a warning: `Sandbox.set_permissions` passes `strict=True` whenever it enforces, so an unprotected run fails instead of producing a normal-looking score. **KNOWN GAP — this is defense-in-depth, not a boundary**: (a) `chmod(2)` is gated on owner-or-`CAP_FOWNER` and the container runs as root owning the copy, so a deliberate `chmod 755 /work/references` restores access; (b) the window spans `agent.communicate` only, and nothing reaps agent child processes at turn end, so a backgrounded read loop succeeds once the window closes. The **write** half of (b) is closed — `path_utils.digest_tree` hashes the tree at staging and `Orchestrator._verify_reference_integrity` re-checks before grading, raising `ReferenceTamperedError` (→ `FinalStatus.ERROR`) on a mismatch so an agent cannot overwrite the reference to drive `reference_comparison` to 1.0. Passive reads are blocked; an adversarial agent is not. Full containment requires running the agent as a non-root uid AND holding the window for the agent's whole lifetime — follow-up. `tasks/anti_cheat_reference` probes the passive-read half. - **Harness run-limit parity**: a shared `BaseAgentConfig` field must mean the same thing on every backend, so a divergence is either fixed or documented — never silent. **`run_limits.max_turns` on Codex/Antigravity counts VISIBLE turns** (resolved tool calls, read live off the shared `EventCollector.visible_turn_count`, the same list `TurnRecord.commands` holds) because one `communicate()` is a single SDK turn on both, so a native counter would clamp at 1; claude-code keeps its native SDK cap, whose unit (an agent-loop turn) absorbs arbitrarily many parallel calls — the same number is NOT the same budget across harnesses. OpenCode likewise keeps a native unit — the CLI streams a real multi-step loop per `communicate()` (`step_start`/`step_finish`), so `max_turns: N` allows N complete steps and cuts cleanly when step N+1 begins. The cap is enforced on the same loop boundary as the cooperative early stop and finalizes cleanly as `max_turns_exhausted` (no crash, no retry); on Antigravity that boundary lives in `_drain()`, so the background-work poll loop honors it too. Known unfixed divergences: `permission_mode` on Codex and Antigravity (both run unconfined — the sandbox driver is the isolation boundary), `disallowed_tools` on Codex (forwarded, not SDK-enforced), `allowed_tools`/`disallowed_tools` on Antigravity (not read at all), `turn_timeout` on Antigravity (bounded by an earlier internal poll deadline at 80% of it), `allowed_tools`/`disallowed_tools`/`system_prompt` on OpenCode (no CLI knob; warned at `start()`, not enforced), and **`agent.plugins[].path` depth** — claude-code REQUIRES a plugin root holding `skills/` and silently loads NOTHING from a bare skills directory, while Codex and Antigravity scan both depths and accept either. That is the costly direction: the wrong depth produces no error, every positive row of an activation suite scores 0, and the suite reports recall 0.0, which reads exactly like a skill that never triggers. Held to the plugin-root shape (for `SKILL_SOURCE_PATH` only) by lint rule CE045. `plugins` on OpenCode is **honored for skills**: each local plugin root is mapped to the `skills` dir its `.claude-plugin/plugin.json` declares (default `/skills`, never the root itself — `skills.paths` is scanned recursively and a plugin root can hold a self-referential symlink) and injected via `OPENCODE_CONFIG_CONTENT`, which `--pure` does not suppress; a plugin's agents/hooks/commands/MCP servers are still dropped. Full table + rationale: docs/agents/HARNESS_PARITY.md. - **sandbox isolation**: Tasks that don't need MCP servers should set `setting_sources: []` in their `agent:` block to isolate the sandbox from the host project's CLAUDE.md and settings. Without this, the host project's CLAUDE.md (often 20 KB+) is injected into every API call, inflating cache-creation tokens and cost significantly. -- **Execute vs. run (the grading switch)**: `coder-eval execute` is `coder-eval run` with grading removed — the agent runs and the full trajectory is captured, but no criterion is checked, `weighted_score` is `None` (never `0.0`, which would be indistinguishable from "graded and scored zero"), and the row finalizes as **`FinalStatus.NOT_GRADED`**, whose `category` is a **fourth** bucket, `"ungraded"`. Ungraded rows leave BOTH sides of every rate: `RunSummary.pass_rate` / `error_share` and `VariantAggregate.pass_rate` divide by `tasks_graded` (`tasks_run - tasks_not_graded`), and `tasks_not_graded` is part of the sum-to-`tasks_run` invariant, not a `tasks_failed` sub-counter. **Only SUCCESS/FAILURE collapse into it** — `ERROR`, `TIMEOUT`, `BUILD_FAILED`, `MAX_TURNS_EXHAUSTED` and the budget stops are facts about the *run*, not about grading, and still apply (so `execute` still exits non-zero on a crash). The switch is `BatchRunConfig.grade` → `Orchestrator(grade=...)` → the **four** grading call sites (single-shot, evaluate-only, the simulation dialog check, and post-failure diagnostics); it crosses the docker boundary in `context.json` (defaulting to `True` in-container, so a host predating `execute` keeps grading). It is **deliberately not a task-config field** — no 5-layer merge, no `-D` path — because a task YAML must never declare itself ungraded; only the invoking command decides. `run` and `execute` share one body (`run_command.run_pipeline`) and differ solely in that flag, so there is no third code path. Three things are refused rather than degraded: `--junit-xml` (a report of verdicts, and there are none — though `reports_junit` still emits `` for an ungraded row it encounters), `--resume` (`partition_for_resume` treats "has any final status" as finalized, so a `NOT_GRADED` row would be *skipped* by a later `run --resume` instead of graded), and simulation tasks (their turn-continuation logic reads criteria results, so an ungraded dialog would silently change its own stopping behavior). `stop_early:` blocks are inert under `execute` for the same reason the kill switch exists: the full trajectory is the deliverable. Motivating consumer: an external harness (Harbor / Terminal-Bench 2.0) that builds its own container, calls coder-eval as the agent, and grades with its own tests. +- **Execute vs. run (the grading switch)**: `coder-eval execute` is `coder-eval run` with grading removed — the agent runs and the full trajectory is captured, but no criterion is checked, `weighted_score` is `None` (never `0.0`, which would be indistinguishable from "graded and scored zero"), and the row finalizes as **`FinalStatus.NOT_GRADED`**, whose `category` is a **fourth** bucket, `"ungraded"`. Ungraded rows leave BOTH sides of every rate: `RunSummary.pass_rate` / `error_share` and `VariantAggregate.pass_rate` divide by `tasks_graded` (`tasks_run - tasks_not_graded`), and `tasks_not_graded` is part of the sum-to-`tasks_run` invariant, not a `tasks_failed` sub-counter. **Only SUCCESS/FAILURE collapse into it** — `ERROR`, `TIMEOUT`, `BUILD_FAILED`, `MAX_TURNS_EXHAUSTED` and the budget stops are facts about the *run*, not about grading, and still apply (so `execute` still exits non-zero on a crash). The switch is `BatchRunConfig.grade` → `Orchestrator(grade=...)` → the **four** grading call sites (single-shot, evaluate-only, the simulation dialog check, and post-failure diagnostics); it crosses the docker boundary in `context.json` (defaulting to `True` in-container, so a host predating `execute` keeps grading). It is **deliberately not a task-config field** — no 5-layer merge, no `-D` path — because a task YAML must never declare itself ungraded; only the invoking command decides. `run` and `execute` share one body (`run_command.run_pipeline`) and differ solely in that flag, so there is no third code path. Two things are refused rather than degraded: `--junit-xml` (a report of verdicts, and there are none — though `reports_junit` still emits `` for an ungraded row it encounters) and simulation tasks (their turn-continuation logic reads criteria results, so an ungraded dialog would silently change its own stopping behavior). `stop_early:` blocks are inert under `execute` for the same reason the kill switch exists: the full trajectory is the deliverable. Motivating consumer: an external harness (Harbor / Terminal-Bench 2.0) that builds its own container, calls coder-eval as the agent, and grades with its own tests. - **Detached grading (`evaluate` over a run dir) + `Sandbox.adopt`**: `coder-eval evaluate` takes two shapes, told apart by a **pure** resolver (`cli/evaluate_target.py`) on one probe — a target holding `task.json` is a run directory. Run-dir mode rebuilds the task from the run's own `task_config.resolved`, **not** by re-loading the YAML: `resolved` is post-merge, so variant overrides / `-D` / dataset expansion are already baked in and re-loading the source would silently grade a *different* task (fallback to `source_file` only when `resolved` no longer validates, and loudly). It seeds the fresh result from the prior one via `Orchestrator(prior_result=...)` → `_seed_from_prior_result`, which carries the trajectory (every derived figure — tokens, cost, `command_stats`, `model_used` — recomputes from `iterations`), `iteration_count`, execution facts, and **`early_stop` — load-bearing, because gate selection is FIRED-ONLY**: dropping it re-grades a truncated trajectory under the full-run strict-AND gate and can flip the verdict. Grader-host `environment_info` is preserved under a `graded_by` sub-dict rather than overwriting the run's. Two other parity fixes: `command_base_path` is now persisted into `environment_info` by `_sync_sandbox_command_path_with_agent` and restored in the evaluate-only branch (closing the PATH gap that method's docstring already named), and `_join_litellm_actual_cost` **skips** when `prior_result` is set (its join keys on a per-Orchestrator nonce the prior turns never carried, so it would clobber already-correct costs). The verdict is written back into the run's `task.json`, with the pre-grade record kept as `task.execute.json` — that in-place write is what makes plain `coder-eval aggregate ` rebuild a graded `run.json` with **zero** new code. **`Sandbox.adopt(workspace)`** is the grade-in-place primitive: it reuses `setup`'s adoption half but skips every *materializing* step (`_setup_template`, `_generate_cli_recorders`, venv/package installs, the destructive `$HOME` remediation), running only non-mutating derivation (mock-dir `+x`, venv *discovery*, plugin-tools pin); `_cleanup_on_exit` stays False so an adopted tree is never moved or deleted. In-place is **more correct**, not merely faster: `_setup_template` filters the copy through `_should_ignore_template_file`, which drops `node_modules` / `dist` / `build` / `.venv` / `.git`, so on the copy path a criterion like `test -f dist/bundle.js` fails as a *copying artifact* rather than as a verdict (verified: 0.00 "does not exist" on copy vs 1.00 in place). Defaults: in-place for a run dir, copy for a bare work dir (criteria can mutate it and it is the user's own tree); `--in-place`/`--copy` override. `adopt` hard-errors on `driver: docker` (a container workspace is unreachable from the host) and a re-grade refuses on a `reference_digest` mismatch. NOTE the Typer command is a thin wrapper over `run_evaluation(...)`, which has real Python defaults — calling a Typer command function in-process hands unspecified options an `OptionInfo` sentinel, which silently made `in_place=None` truthy. +- **`--resume` is command-relative**: `partition_for_resume(tasks, *, grade)` returns a four-way `ResumePartition` (`to_run` / `to_grade` / `prior_results` / `prior_resolved`), because **"finished" is not absolute — it depends on what the resuming command still owes the task**. A `NOT_GRADED` row carries a final status, so the original "has any final status" test called it complete: right for `execute --resume` (it finished executing), and wrong for `run --resume`, which was asked to grade and would instead report "already complete", grade nothing, and **exit 0**. Under `grade=True` those rows route to `to_grade`, where `_grade_resumed_tasks` runs the criteria against the trajectory and workspace already on disk via `orchestration/regrade.py::regrade_in_place` — reusing the agent spend, which is the entire reason `execute` and `run` are separate. The carve-out is **only** for `NOT_GRADED`: `FAILURE`/`ERROR` stay complete under both commands (resume has never retried failures — delete the task.json), and `clear_rerun_artifacts` deliberately skips `to_grade`, whose artifacts are the very thing being graded. A per-task grading failure is warned and folded back in with its ORIGINAL ungraded result, so one bad row neither aborts the resume nor vanishes from run.json. `grade` is in `_FINGERPRINT_DIFF_EXEMPT` because `execute` → `run --resume` is a supported flow, not config drift — and the warning's "already-finalized tasks keep their original-config results" text is actively wrong for it. **`orchestration/regrade.py` is the single implementation** shared by that path and `evaluate`'s run-dir mode (two copies would drift into two verdicts for the same run); it raises plain `RegradeError`, which the CLI wraps, since `orchestration/` must not import the CLI layer (CE004). One fidelity rule it enforces: a re-graded row keeps the **agent run's** `started_at`/`duration_seconds`, not the grading pass's — a 10-minute run re-graded in 2s would otherwise report 2s into `average_duration`, the report tables and the evalboard; the grading cost is preserved separately as `environment_info["grading_duration_seconds"]`. - **Run-time caps (non-criterion enforcement)**: `TaskDefinition.run_limits` (`RunLimits` model) is the single namespace for all *task-level* run-time caps — `max_turns` / `task_timeout` / `turn_timeout` (structural) and `max_input_tokens` / `max_output_tokens` / `max_total_tokens` / `max_usd` (cumulative budget). Token/USD breaches abort with `FinalStatus.TOKEN_BUDGET_EXCEEDED` or `COST_BUDGET_EXCEEDED` (both `category == "failed"`). Structural caps are set from the CLI via `-D run_limits.max_turns=…` / `-D run_limits.task_timeout=…` / `-D run_limits.turn_timeout=…` (field-merged into `run_limits`); budget caps via `-D run_limits.max_usd=…` etc. or YAML. Layered config uses field-merge — a variant block overrides individual keys without replacing the task's block. The one *per-criterion* cap, `stop_early.decide_within`, deliberately lives on `LiveSuccessCriterion` instead (see below) — the watcher must attribute a decision-step timeout to a specific criterion, which `RunLimits` (task-scoped, criterion-agnostic) cannot express. - **Early stop on criterion (opt-in, per-criterion arming)**: a `stop_early:` block (`StopEarlyPolicy`) on a criterion ends a single-shot run early once the run's **armed** criteria decide the outcome, so a raised `max_turns` isn't wasted on the smoke flavor. The block's PRESENCE is the arming and alone activates the watcher — there is **no run-level master switch**: `run_limits.stop_early: false` is the run-level KILL SWITCH that force-disarms every block (the one-line experiment-variant/`-D` override for an authoritative full run), and `run_limits.stop_early: true` (the removed master arm) is a hard `EarlyStopConfigError` at resolution. The block exists on `LiveSuccessCriterion` only (currently `skill_triggered`, `command_executed` — so arming an unobservable criterion is unrepresentable, a pydantic extra-forbid error). Arming carries one implicit trigger (a native live-fail may fail-stop the run); its keys refine it: `on_pass: stop` (pass-stop the moment the criterion live-passes; default `continue` just latches) and `decide_within: N` (still undecided after N tool-call steps latches an **effective fail**, fed through the same fail-stop rule, reported as `decision_budget_exceeded` — an ordinary weighted fail, NOT a gate-bypassing force-fail; cumulative across retry attempts of the same turn). A trigger whose polarity the instance can't decide (per the abstract, checker-independent `live_decidable_polarities()`, a pure function of the criterion's own fields, paired with the checker's `live_verdict` override by lint rule CE025, a registry-based whole-tree check) is **inert by design** — one dataset-fanned YAML line serves both positive rows (pass/timeout live) and distractor rows (fail live). Verdicts **latch**: once a criterion decides, its `live_verdict` is never polled again. Stop rule is weighted, not strict-boolean: `run_limits.stop_early_gate_threshold` (default `1.0`, reproducing strict-AND behavior exactly) is the minimum weighted score (`Σ weight·score / Σ weight` over the armed subset) required to pass; a fail-stop fires once the armed set's **ceiling** (best case for everything still undecided) can no longer reach the threshold — so a low-weight fail or timeout that can't doom the gate is absorbed and the run continues — and is **deferred while any pass-capable armed criterion is undecided** (a distractor misfire never truncates a positive row's recall signal); a pass-stop fires once the `on_pass: stop` subset's **floor** (worst case) already meets the threshold, and is symmetrically **deferred while any pass-capable armed criterion outside the `on_pass: stop` subset is undecided** (so an early pass never freezes a sibling `on_pass: continue` criterion's signal out of the trajectory). A fail-stop is therefore verdict-preserving; a pass-stop can miss a *later* distractor misfire, so authoritative P/R/F1 comes from a kill-switched (`stop_early: false`) run. Driven by `orchestration/early_stop.py::EarlyStopWatcher` (built when `early_stop_active(task)`: ≥1 armed criterion, kill switch not thrown) through the agent's cooperative `should_stop` seam (tool-call granularity, no SIGKILL); live verdicts only *trigger* the stop — the standard `check_all_async` on the frozen trajectory is authoritative. Gating is **FIRED-ONLY**: a run the watcher actually cut gates on the **armed subset** via the weighted `EvaluationResult.armed_criteria_passed`; a run that completes naturally — armed or not — gates strict-AND via `all_criteria_passed`, so adding a block never changes the verdict of a run it didn't cut. Note the gate keys on the watcher having FIRED (`result.early_stop is not None`), not on confirmed truncation — an agent that ignores `should_stop`, or a stop firing on the final message, still gates armed-only. Every resolution-time guardrail violation is a hard error at resolution (plan *and* run); the one load-time case — a `stop_early:` block on a non-live criterion — is a pydantic schema error at task load, which the run surface reports as a skipped task like any other malformed task. A runtime verdict bug **fails open** to a full run. Surfaces: `EarlyStopInfo` (incl. `gate_threshold` at stop time), report notes/badges, `stopped_early` run.json rows, `EarlyStopped`/`EarlyStopReason` telemetry dims. Worked rationale: docs/TASK_DEFINITION_GUIDE.md § `stop_early`. No blocks anywhere ⇒ behavior byte-for-byte unchanged. diff --git a/docs/USER_GUIDE.md b/docs/USER_GUIDE.md index 8c4454f3..c37fa46f 100644 --- a/docs/USER_GUIDE.md +++ b/docs/USER_GUIDE.md @@ -41,7 +41,7 @@ coder-eval run tasks/hello_date.yaml --stream full # live LLM output | `--driver` | Shorthand alias for `-D sandbox.driver=…` (`tempdir` or `docker`) | | `--type, -T` | Override agent type for all tasks (`claude-code`, `codex`, `antigravity`, `opencode`, or a plugin kind). | | `--repeats` | Run each `(task, variant)` N times (≥1); overrides experiment/variant `repeats:`. See [Replicates](#replicates). | -| `--resume` | Resume an interrupted run: skip tasks already finalized in `--run-dir` and run the rest, folding prior results into `run.json`. Requires `--run-dir`. A task with *any* final status (incl. FAILED/ERROR) counts as finalized, so resume does **not** retry failures — delete a task's `task.json` to force a re-run. A config mismatch is warned, not refused. | +| `--resume` | Resume an interrupted run: skip tasks already finalized in `--run-dir` and run the rest, folding prior results into `run.json`. Requires `--run-dir`. See [Resuming a run](#resuming-a-run). | | `--sample N` | For dataset-backed tasks, run a fixed-seed random N-row sample (reproducible; cheap smoke test). See [Bring Your Own Dataset](DATASETS.md). | | `--sample-per-stratum N` | For dataset-backed tasks, keep up to N rows per stratum (`stratify_field`). Overridden by `--sample`. Nondeterministic unless `dataset.sample_seed` is set — see [Bring Your Own Dataset](DATASETS.md). | | `--include-skipped` | Also run tasks marked `skip: true` in their YAML (off by default so CI keeps excluding them). | @@ -86,12 +86,47 @@ degraded: | Not supported | Why | | --- | --- | | `--junit-xml` | A JUnit report reports verdicts, and there are none. | -| `--resume` | Resume treats "has any final status" as finalized, so a `NOT_GRADED` row would be *skipped* by a later `run --resume` rather than graded. | | Simulation tasks | The dialog loop reads criteria results to decide whether to keep talking, so an ungraded dialog would silently change its own stopping behavior. Rejected by name at startup. | `stop_early:` blocks are also inert here: early stop exists to cut a run once the criteria decide the outcome, and under `execute` the full trajectory is the deliverable. +`--resume` **is** supported, and `run --resume` pairs with it (see below). + +### Resuming a run + +`--resume` continues an interrupted run without re-paying for finished work. It +requires `--run-dir` (an auto-generated directory is always fresh). + +What it owes each task depends on what it finds in that task's `task.json`: + +| On disk | `run --resume` | `execute --resume` | +| --- | --- | --- | +| No `task.json`, unreadable, or no `final_status` | re-run | re-run | +| `NOT_GRADED` | **grade in place** | already complete | +| Any other status, **including `FAILURE` / `ERROR`** | already complete | already complete | + +**"Finished" is relative to the resuming command.** A `NOT_GRADED` row owes +`execute` nothing — it finished executing — but owes `run` a grade. So +`run --resume` runs the criteria against the trajectory and workspace already on +disk instead of re-running the agent, which is the whole reason to split the two +commands: + +```bash +coder-eval execute tasks/*.yaml --run-dir ./r # expensive half +coder-eval run tasks/*.yaml --run-dir ./r --resume # grades what execute left +``` + +**Resume never retries failures.** `FAILURE` and `ERROR` count as complete under +both commands — delete a task's `task.json` to force a re-run. A task about to +re-run has its stale `artifacts/` cleared first, so leftover files from +a killed container cannot satisfy a file-based criterion. + +A run-config mismatch is **warned, not refused**: resumed tasks keep their +original-config results, so the run genuinely mixes configs. The `grade` flag is +exempt from that warning, because `execute` → `run --resume` is a supported flow +rather than a config mistake. + ### `coder-eval plan` — validate tasks ```bash diff --git a/src/coder_eval/cli/evaluate_command.py b/src/coder_eval/cli/evaluate_command.py index cdba3fc5..7de52fc3 100644 --- a/src/coder_eval/cli/evaluate_command.py +++ b/src/coder_eval/cli/evaluate_command.py @@ -19,113 +19,26 @@ TemplateDirSource, parse_agent_config, ) +from ..orchestration.regrade import ( + PRE_GRADE_JSON, + TASK_JSON, + RegradeError, + back_up_pre_grade_record, + default_workspace, + load_prior_result, + task_from_prior, + verify_reference_unchanged, +) from ..orchestration.task_loader import load_task from ..orchestrator import Orchestrator from ..sandbox import Sandbox from .console import console -from .evaluate_target import TASK_JSON, EvaluateMode, EvaluateTarget, EvaluateTargetError, resolve_evaluate_target +from .evaluate_target import EvaluateMode, EvaluateTarget, EvaluateTargetError, resolve_evaluate_target from .run_helpers import prepare_run_directory logger = logging.getLogger(__name__) -# Where a run directory keeps the workspace the agent worked in. The extra -# segment is the task id: preservation writes `artifacts//...`. -ARTIFACTS_DIRNAME = "artifacts" - - -def _load_prior_result(run_dir: Path) -> EvaluationResult: - """Read a finished run's ``task.json``.""" - raw = (run_dir / TASK_JSON).read_text(encoding="utf-8") - try: - return EvaluationResult.model_validate_json(raw) - except ValueError as e: - raise typer.BadParameter(f"{run_dir / TASK_JSON} is not a readable EvaluationResult: {e}") from e - - -def _task_from_prior(prior: EvaluationResult, run_dir: Path) -> tuple[TaskDefinition, str]: - """Rebuild the executed task from the run's own recorded config. - - Rebuilding from ``task_config.resolved`` rather than re-reading the YAML is - what makes the grade describe the run that happened: ``resolved`` is the - post-merge definition, so variant overrides, ``-D`` flags and dataset row - expansion are all already baked in. Re-loading the source YAML would silently - grade a DIFFERENT task whenever any of those were used. - - Falls back to the source YAML only when ``resolved`` will not validate (a - schema change since the run), and says so loudly — a quiet fallback would - reintroduce exactly the drift above. - """ - record = prior.task_config - if record is None: - raise typer.BadParameter( - f"{run_dir / TASK_JSON} carries no task_config, so the executed task cannot be " - + "rebuilt. Pass the task file explicitly: coder-eval evaluate " - ) - try: - return TaskDefinition.model_validate(record.resolved), record.source_yaml - except ValueError as e: - if not record.source_file or not Path(record.source_file).is_file(): - raise typer.BadParameter( - f"The resolved task config in {run_dir / TASK_JSON} no longer validates ({e}), and " - + "its source YAML is unavailable. Pass the task file explicitly." - ) from e - console.print( - f"[yellow]⚠[/] The recorded resolved config does not validate ({e}); falling back to " - + f"{record.source_file}. Variant overrides, -D flags and dataset expansion from the " - + "original run are NOT reapplied, so this grade may not match what ran." - ) - return load_task(Path(record.source_file)) - - -def _default_workspace(run_dir: Path, prior: EvaluationResult) -> Path: - """Locate the workspace a finished run left behind. - - ``sandbox_path`` is authoritative when it still exists — it is where the run - actually worked. Otherwise fall back to the preserved artifacts tree, whose - single child is named for the task. - """ - if prior.sandbox_path: - recorded = Path(prior.sandbox_path) - if recorded.is_dir(): - return recorded - - artifacts = run_dir / ARTIFACTS_DIRNAME - if not artifacts.is_dir(): - raise typer.BadParameter( - f"No workspace to grade: {artifacts} does not exist and the recorded sandbox_path " - + f"({prior.sandbox_path or 'unset'}) is gone. The run was probably made with " - + "--preservation-mode NONE. Point at one explicitly with --workspace." - ) - children = [p for p in sorted(artifacts.iterdir()) if p.is_dir()] - # Preservation nests the workspace under the task id; a flat artifacts dir - # (no subdirectory) means the workspace IS artifacts/. - return children[0] if len(children) == 1 else artifacts - - -def _verify_reference_unchanged(prior: EvaluationResult, task: TaskDefinition) -> None: - """Refuse to grade when the reference tree changed since the run. - - ``reference_comparison`` and reference-carrying judges score against - ``task.reference.directory``. If it moved since the run, the re-grade would - silently measure the agent's old work against a new answer key. - """ - recorded = prior.environment_info.get("reference_digest") - if not isinstance(recorded, str) or task.reference is None: - return - from ..orchestration.evaluation import resolve_reference_dir - from ..path_utils import digest_tree - - resolved = resolve_reference_dir(task, None) - if resolved is None or not resolved.is_dir(): - return - if digest_tree(resolved) != recorded: - raise typer.BadParameter( - f"The reference directory {resolved} changed since this run was executed " - + "(digest mismatch). Grading now would score the agent's work against a " - + "different answer key. Restore the reference, or re-run the task." - ) - @dataclass(frozen=True) class _ResolvedInputs: @@ -156,15 +69,25 @@ def _resolve_inputs(task_or_run_dir: Path, work_dir: Path | None, workspace: Pat + "directory to grade is already the second argument." ) + try: + return _resolve_run_dir_or_work_dir(target, workspace) + except RegradeError as e: + # The shared core raises a plain exception (orchestration/ must not + # depend on the CLI layer, CE004); surface it as a CLI error here. + raise typer.BadParameter(str(e)) from e + + +def _resolve_run_dir_or_work_dir(target: EvaluateTarget, workspace: Path | None) -> _ResolvedInputs: + """The mode-specific half of :func:`_resolve_inputs`.""" prior: EvaluationResult | None = None if target.mode is EvaluateMode.RUN_DIR: - prior = _load_prior_result(target.target) + prior = load_prior_result(target.target) if target.task_file is not None: task, source_yaml = load_task(target.task_file) console.print(f"[dim]Grading with {target.task_file} (overrides the run's recorded config).[/dim]") else: - task, source_yaml = _task_from_prior(prior, target.target) - work_dir = workspace or _default_workspace(target.target, prior) + task, source_yaml = task_from_prior(prior, target.target) + work_dir = workspace or default_workspace(target.target, prior) recorded_source = prior.task_config.source_file if prior.task_config else None task_file = target.task_file or (Path(recorded_source) if recorded_source else None) else: @@ -192,7 +115,7 @@ def _resolve_inputs(task_or_run_dir: Path, work_dir: Path | None, workspace: Pat task.agent = parse_agent_config(**{**task.agent.model_dump(exclude_unset=True), "type": AgentKind.CLAUDE_CODE}) if prior is not None: - _verify_reference_unchanged(prior, task) + verify_reference_unchanged(prior, task) return _ResolvedInputs( target=target, @@ -434,10 +357,9 @@ def _write_back(run_dir: Path, result: EvaluationResult) -> None: only evidence that the run was executed separately. """ target = run_dir / TASK_JSON - backup = run_dir / "task.execute.json" + backup = run_dir / PRE_GRADE_JSON + back_up_pre_grade_record(run_dir) try: - if not backup.exists(): - backup.write_text(target.read_text(encoding="utf-8"), encoding="utf-8") target.write_text(result.model_dump_json(indent=2), encoding="utf-8") except OSError as e: # Never fail the grade over the write-back: the verdict was computed and diff --git a/src/coder_eval/cli/execute_command.py b/src/coder_eval/cli/execute_command.py index b90fa0e3..79388003 100644 --- a/src/coder_eval/cli/execute_command.py +++ b/src/coder_eval/cli/execute_command.py @@ -12,14 +12,13 @@ be worse than not grading at all: coder-eval's verdict would be reported alongside Harbor's without being the one that counts. -Every flag on ``run`` is available here except two, and both omissions are -deliberate: +Every flag on ``run`` is available here except ``--junit-xml``, which is a report +of verdicts and there are none. -* ``--junit-xml`` — a JUnit report is a report of verdicts, and there are none. -* ``--resume`` — ``partition_for_resume`` treats "has any final status" as - finalized, so a ``NOT_GRADED`` row would be skipped by a later ``run --resume`` - rather than graded. Supporting it needs resume to distinguish "done" from - "executed but unscored"; until then, refusing is the honest option. +``--resume`` IS supported, because ``partition_for_resume`` now takes the +resuming command into account: a ``NOT_GRADED`` row owes ``execute`` nothing (it +finished executing) but owes ``run`` a grade, so ``run --resume`` grades those +rows in place rather than skipping them as "already complete". The command shares ``run``'s entire body (``run_command.run_pipeline``); only the Typer signature is restated, because Typer builds its parser from the signature. @@ -55,6 +54,16 @@ def execute_command( "--run-dir", help="Custom run directory (default: auto-generated timestamped directory in runs/)", ), + resume: bool = typer.Option( + False, + "--resume", + help=( + "Resume an interrupted execute: skip tasks already executed in --run-dir " + "and run only the rest. Requires --run-dir. A NOT_GRADED row counts as " + "done here (it finished executing); a later `coder-eval run --resume` on " + "the same directory grades those rows instead of skipping them." + ), + ), max_parallel: int = typer.Option( 1, "--max-parallel", @@ -187,8 +196,7 @@ def execute_command( ERROR / TIMEOUT / TOKEN_BUDGET_EXCEEDED and exits non-zero exactly as under `run`. Only the verdict is withheld, never the facts of the run. - Not supported here: --junit-xml (no verdicts to report), --resume (a - NOT_GRADED row would be mistaken for a finalized one), and simulation tasks + Not supported here: --junit-xml (no verdicts to report) and simulation tasks (their turn-continuation logic reads criteria results). Examples: @@ -202,8 +210,8 @@ def execute_command( task_files=task_files, preservation_mode=preservation_mode, run_dir=run_dir, - # Not exposed as flags — see the module docstring for why each is refused. - resume=False, + resume=resume, + # Not exposed as a flag — a JUnit report reports verdicts, and there are none. junit_xml=None, max_parallel=max_parallel, verbose=verbose, diff --git a/src/coder_eval/cli/run_command.py b/src/coder_eval/cli/run_command.py index d820e301..65337c65 100644 --- a/src/coder_eval/cli/run_command.py +++ b/src/coder_eval/cli/run_command.py @@ -657,6 +657,70 @@ def _on_task_complete(result: Any) -> None: return result +async def _grade_resumed_tasks(to_grade: list[ResolvedTask]) -> list[tuple[ResolvedTask, TaskResult]]: + """Grade the rows ``coder-eval execute`` left NOT_GRADED, in place. + + Each task's trajectory and workspace are already on disk, so this runs the + criteria against them instead of re-running the agent — that reuse is the + whole reason to split ``execute`` from ``run``. + + The task config comes from ``rt.task`` (this run's own 5-layer resolution), + not from the recorded one: ``--resume`` re-resolves the same task files, and + a config that drifted since the execute is already surfaced by the run + fingerprint warning above. + + A task that cannot be graded is reported and folded back in with its ORIGINAL + ungraded result, so one bad row neither aborts the resume nor silently + vanishes from run.json — it stays visible as ``tasks_not_graded``. + """ + from ..orchestration.regrade import ( + RegradeError, + back_up_pre_grade_record, + default_workspace, + load_prior_result, + regrade_in_place, + verify_reference_unchanged, + ) + + graded: list[tuple[ResolvedTask, TaskResult]] = [] + for rt in to_grade: + prior = load_prior_result(rt.run_dir) + try: + verify_reference_unchanged(prior, rt.task) + workspace = default_workspace(rt.run_dir, prior) + # Preserve the ungraded record BEFORE the orchestrator overwrites + # task.json in this same directory. + back_up_pre_grade_record(rt.run_dir) + result = await regrade_in_place( + task=rt.task, + prior=prior, + workspace=workspace, + run_dir=rt.run_dir, + task_file=rt.task_file, + source_yaml=rt.source_yaml, + variant_id=rt.variant_id, + replicate_index=rt.replicate_index, + ) + except (RegradeError, OSError, RuntimeError, ValueError) as e: + console.print(f"[yellow]⚠[/] Could not grade {rt.task.task_id}: {e}") + result = prior + graded.append( + ( + rt, + TaskResult( + task_id=rt.task.task_id, + variant_id=rt.variant_id, + result=result, + duration=result.duration_seconds or 0.0, + suite_id=rt.task.suite_id, + row_id=rt.task.row_id, + replicate_index=rt.replicate_index, + ), + ) + ) + return graded + + async def _run_with_experiment( all_task_files: list[Path], config: BatchRunConfig, @@ -787,16 +851,26 @@ async def _run_with_experiment( prior_results: list[TaskResult] = [] prior_resolved: list[ResolvedTask] = [] if resume: - to_run, prior_results, prior_resolved = partition_for_resume(resolved) + part = partition_for_resume(resolved, grade=grade) + to_run, prior_results, prior_resolved = part.to_run, part.prior_results, part.prior_resolved # A re-run task re-executes from scratch, so any leftover artifacts (only # DIRECT_WRITE writes them live; a container killed mid-run leaves partials) # are stale and could let a file-based criterion pass on the old output. + # to_grade is deliberately NOT cleared: its artifacts are the run's output + # and the very thing being graded. cleared = clear_rerun_artifacts(to_run) console.print( f"[cyan]↻ Resume:[/] {len(prior_results)} task(s) already complete, " + f"running {len(to_run)} remaining" + + (f", grading {len(part.to_grade)} executed-but-ungraded" if part.to_grade else "") + (f" (cleared {cleared} stale artifact dir(s))" if cleared else "") ) + # Grade the rows `execute` left behind, reusing the trajectory and + # workspace already on disk rather than paying for the agent twice. + # Folded in as prior_results so the summary covers them like any other. + for rt, tr in await _grade_resumed_tasks(part.to_grade): + prior_results.append(tr) + prior_resolved.append(rt) # Print execution mode print_execution_mode(len(to_run), max_parallel) diff --git a/src/coder_eval/orchestration/batch.py b/src/coder_eval/orchestration/batch.py index d79e6d96..d9fe0098 100644 --- a/src/coder_eval/orchestration/batch.py +++ b/src/coder_eval/orchestration/batch.py @@ -15,7 +15,7 @@ from collections.abc import Callable, Iterator from datetime import datetime from pathlib import Path -from typing import Any +from typing import Any, NamedTuple from ..models import ( AgentKind, @@ -324,10 +324,24 @@ def _create_error_task_result( ) -def partition_for_resume( - resolved_tasks: list[ResolvedTask], -) -> tuple[list[ResolvedTask], list[TaskResult], list[ResolvedTask]]: - """Split resolved tasks into (to_run, prior_results, prior_resolved) for --resume. +class ResumePartition(NamedTuple): + """How ``--resume`` splits a run's tasks over what each one still needs.""" + + to_run: list[ResolvedTask] + """Never finished executing — re-run from scratch.""" + + to_grade: list[ResolvedTask] + """Executed but ungraded (NOT_GRADED). Needs criteria, NOT another agent run.""" + + prior_results: list[TaskResult] + """Genuinely finished — reloaded so run.json covers the whole suite.""" + + prior_resolved: list[ResolvedTask] + """The ResolvedTask for each entry of prior_results, same order.""" + + +def partition_for_resume(resolved_tasks: list[ResolvedTask], *, grade: bool = True) -> ResumePartition: + """Split resolved tasks over what ``--resume`` still owes each one. A task is already-complete when its task.json exists, parses, and carries a final_status. task.json is written atomically at end-of-run, so any parseable @@ -335,26 +349,40 @@ def partition_for_resume( tasks are reloaded into TaskResults (to fold into run.json) and excluded from to_run; everything else — including failed-to-parse — re-runs. + **"Finished" is relative to the resuming command, not absolute.** A + ``NOT_GRADED`` row (written by ``coder-eval execute``) has a final status, so + the naive test calls it complete. That is right for ``execute --resume``, + which owes it nothing — and wrong for ``run --resume``, which was asked to + grade: skipping it would report "already complete", grade nothing, and exit + 0. So under ``grade=True`` those rows go to ``to_grade`` instead, where the + caller runs the criteria against the trajectory and workspace already on + disk rather than paying for the agent a second time. + + Note the asymmetry is only for NOT_GRADED. FAILURE and ERROR stay complete + under both commands — resume has never retried failures (delete a task's + task.json to force that), and this does not change it. + Args: resolved_tasks: Fully-resolved tasks for the whole run. + grade: Whether the resuming command grades (``run``) or not (``execute``). Returns: - (to_run, prior_results, prior_resolved): - - to_run: tasks still needing execution - - prior_results: reloaded results for already-complete tasks - - prior_resolved: the ResolvedTask for each prior_result (same order) + The four-way :class:`ResumePartition`. """ to_run: list[ResolvedTask] = [] + to_grade: list[ResolvedTask] = [] prior_results: list[TaskResult] = [] prior_resolved: list[ResolvedTask] = [] for rt in resolved_tasks: tr = _load_completed_result(rt) if tr is None: to_run.append(rt) + elif grade and tr.result.final_status.category == "ungraded": + to_grade.append(rt) else: prior_results.append(tr) prior_resolved.append(rt) - return to_run, prior_results, prior_resolved + return ResumePartition(to_run, to_grade, prior_results, prior_resolved) def clear_rerun_artifacts(to_run: list[ResolvedTask]) -> int: @@ -511,13 +539,24 @@ def read_run_fingerprint(run_dir: Path) -> dict[str, object] | None: return data if isinstance(data, dict) else None +# `grade` is excluded from the drift warning: `execute` then `run --resume` is a +# SUPPORTED flow, not a config mistake, and the warning's text ("already-finalized +# tasks keep their original-config results") is actively wrong for it — those rows +# are re-graded with the current config, which is the entire point. +_FINGERPRINT_DIFF_EXEMPT = frozenset({"grade"}) + + def fingerprint_diff(prior: dict[str, object], current: dict[str, object]) -> dict[str, tuple[object, object]]: """Keys present in BOTH stamps that disagree, as ``{key: (prior, current)}``. Only keys present in ``prior`` are compared, so adding fingerprint fields in a later version never false-flags a resume of an older run. """ - return {k: (prior[k], current[k]) for k in current if k in prior and prior[k] != current[k]} + return { + k: (prior[k], current[k]) + for k in current + if k in prior and prior[k] != current[k] and k not in _FINGERPRINT_DIFF_EXEMPT + } def _override_uip_versions_from_tasks(version_info: dict[str, Any], task_results: list[TaskResult]) -> None: diff --git a/src/coder_eval/orchestration/regrade.py b/src/coder_eval/orchestration/regrade.py new file mode 100644 index 00000000..aa6b15c7 --- /dev/null +++ b/src/coder_eval/orchestration/regrade.py @@ -0,0 +1,205 @@ +"""Grade a run that already executed — the shared core behind two callers. + +``coder-eval execute`` leaves every row ``NOT_GRADED``. Two commands can supply +the verdict afterwards, and both must do it identically: + +* ``coder-eval evaluate `` — grade one finished task explicitly. +* ``coder-eval run --resume`` — grade the ungraded rows it finds in the run dir + instead of re-executing them (see ``partition_for_resume``). + +The logic lives here rather than in ``cli/`` because the resume path is not a CLI +concern, and because two copies of "how to re-grade" would drift into two +different verdicts for the same run. Errors surface as :class:`RegradeError`, a +plain exception the CLI wraps into its own error type — ``orchestration/`` must +not depend on the CLI layer (CE004). +""" + +from __future__ import annotations + +import asyncio +import logging +from pathlib import Path + +from coder_eval.models import EvaluationResult, PreservationMode, TaskDefinition +from coder_eval.sandbox import Sandbox + + +logger = logging.getLogger(__name__) + +TASK_JSON = "task.json" +PRE_GRADE_JSON = "task.execute.json" +ARTIFACTS_DIRNAME = "artifacts" + + +class RegradeError(Exception): + """A finished run cannot be re-graded as asked.""" + + +def load_prior_result(run_dir: Path) -> EvaluationResult: + """Read a finished run's ``task.json``.""" + path = run_dir / TASK_JSON + try: + raw = path.read_text(encoding="utf-8") + except OSError as e: + raise RegradeError(f"Cannot read {path}: {e}") from e + try: + return EvaluationResult.model_validate_json(raw) + except ValueError as e: + raise RegradeError(f"{path} is not a readable EvaluationResult: {e}") from e + + +def task_from_prior(prior: EvaluationResult, run_dir: Path) -> tuple[TaskDefinition, str]: + """Rebuild the executed task from the run's own recorded config. + + Rebuilding from ``task_config.resolved`` rather than re-reading the YAML is + what makes the grade describe the run that happened: ``resolved`` is the + post-merge definition, so variant overrides, ``-D`` flags and dataset row + expansion are all already baked in. Re-loading the source YAML would silently + grade a DIFFERENT task whenever any of those were used. + + Falls back to the source YAML only when ``resolved`` will not validate (a + schema change since the run), and says so loudly — a quiet fallback would + reintroduce exactly the drift above. + """ + from .task_loader import load_task + + record = prior.task_config + if record is None: + raise RegradeError( + f"{run_dir / TASK_JSON} carries no task_config, so the executed task cannot be " + + "rebuilt. Pass the task file explicitly: coder-eval evaluate " + ) + try: + return TaskDefinition.model_validate(record.resolved), record.source_yaml + except ValueError as e: + if not record.source_file or not Path(record.source_file).is_file(): + raise RegradeError( + f"The resolved task config in {run_dir / TASK_JSON} no longer validates ({e}), and " + + "its source YAML is unavailable. Pass the task file explicitly." + ) from e + logger.warning( + "The recorded resolved config does not validate (%s); falling back to %s. Variant " + + "overrides, -D flags and dataset expansion from the original run are NOT reapplied, " + + "so this grade may not match what ran.", + e, + record.source_file, + ) + return load_task(Path(record.source_file)) + + +def default_workspace(run_dir: Path, prior: EvaluationResult) -> Path: + """Locate the workspace a finished run left behind. + + ``sandbox_path`` is authoritative when it still exists — it is where the run + actually worked. Otherwise fall back to the preserved artifacts tree, whose + single child is named for the task. + """ + if prior.sandbox_path: + recorded = Path(prior.sandbox_path) + if recorded.is_dir(): + return recorded + + artifacts = run_dir / ARTIFACTS_DIRNAME + if not artifacts.is_dir(): + raise RegradeError( + f"No workspace to grade: {artifacts} does not exist and the recorded sandbox_path " + + f"({prior.sandbox_path or 'unset'}) is gone. The run was probably made with " + + "--preservation-mode NONE." + ) + children = [p for p in sorted(artifacts.iterdir()) if p.is_dir()] + # Preservation nests the workspace under the task id; a flat artifacts dir + # (no subdirectory) means the workspace IS artifacts/. + return children[0] if len(children) == 1 else artifacts + + +def verify_reference_unchanged(prior: EvaluationResult, task: TaskDefinition) -> None: + """Refuse to grade when the reference tree changed since the run. + + ``reference_comparison`` and reference-carrying judges score against + ``task.reference.directory``. If it moved since the run, the re-grade would + silently measure the agent's old work against a new answer key. + """ + recorded = prior.environment_info.get("reference_digest") + if not isinstance(recorded, str) or task.reference is None: + return + from coder_eval.path_utils import digest_tree + + from .evaluation import resolve_reference_dir + + resolved = resolve_reference_dir(task, None) + if resolved is None or not resolved.is_dir(): + return + if digest_tree(resolved) != recorded: + raise RegradeError( + f"The reference directory {resolved} changed since this run was executed " + + "(digest mismatch). Grading now would score the agent's work against a " + + "different answer key. Restore the reference, or re-run the task." + ) + + +def back_up_pre_grade_record(run_dir: Path) -> None: + """Keep the ungraded ``task.json`` beside the graded one, once. + + The write-back replaces the only on-disk evidence that this run was executed + separately from grading. Copying it first keeps that auditable. Written once: + a second grade must not overwrite the ORIGINAL execute record with an + already-graded one. + """ + source, backup = run_dir / TASK_JSON, run_dir / PRE_GRADE_JSON + if backup.exists() or not source.is_file(): + return + try: + backup.write_text(source.read_text(encoding="utf-8"), encoding="utf-8") + except OSError as e: + # Never fail a grade over the audit copy. + logger.warning("Could not preserve the pre-grade record at %s: %s", backup, e) + + +async def regrade_in_place( + *, + task: TaskDefinition, + prior: EvaluationResult, + workspace: Path, + run_dir: Path, + task_file: Path | None, + source_yaml: str, + variant_id: str, + replicate_index: int = 0, +) -> EvaluationResult: + """Run ``task``'s criteria against an already-executed ``workspace``. + + The workspace is *adopted*, never copied: it is the run's own output, and the + template-copy path filters out ``node_modules`` / ``dist`` / ``build`` / + ``.venv``, which would make a criterion reading those fail as a copying + artifact rather than as a verdict. + + ``prior`` supplies the trajectory and the run's execution facts (see + ``Orchestrator._seed_from_prior_result``), so criteria that read the agent's + tool calls score exactly as they would have during the run. + """ + from coder_eval.orchestrator import Orchestrator + + # Grading never runs a container: the docker driver dispatches through + # DockerRunner, which needs an agent. Force tempdir so a task whose YAML says + # `driver: docker` is still gradeable on the host. + sandbox_config = task.sandbox.model_copy(deep=True).model_copy(update={"driver": "tempdir"}) + sandbox = Sandbox( + sandbox_config, + task_id=task.task_id, + task_dir=task_file.parent.resolve() if task_file is not None else None, + ) + await asyncio.to_thread(sandbox.adopt, workspace) + + orchestrator = Orchestrator( + task=task, + run_dir=run_dir, + # The workspace belongs to the run being graded; never move or delete it. + preservation_mode=PreservationMode.NONE, + task_file=task_file, + sandbox=sandbox, + variant_id=variant_id, + source_yaml=source_yaml, + replicate_index=replicate_index, + prior_result=prior, + ) + return await orchestrator.run() diff --git a/src/coder_eval/orchestrator.py b/src/coder_eval/orchestrator.py index 0549e7b7..e9c991ea 100644 --- a/src/coder_eval/orchestrator.py +++ b/src/coder_eval/orchestrator.py @@ -735,13 +735,20 @@ def _seed_from_prior_result(self) -> None: silently disagrees with the run it grades. Deliberately NOT carried: ``final_status``, ``weighted_score`` and - ``success_criteria_results`` — those are exactly what this pass recomputes - — and the timestamps/duration, which describe the grading pass. + ``success_criteria_results`` — those are exactly what this pass recomputes. """ prior = self.prior_result if prior is None or self.result is None: return + # A task row describes the TASK, so its clock is the agent run's, not the + # grading pass's. Left alone, a 10-minute run re-graded in 2 seconds would + # report 2 seconds — and that figure feeds average_duration, the report + # tables and the evalboard, so harness-vs-harness comparisons would be + # quietly wrong. _finalize_result restores the duration after its own + # timing write; the grading pass's cost is recorded separately there. + self.result.started_at = prior.started_at + # The trajectory itself. Every derived figure in _finalize_result — # token totals, cost, command_stats, model_used, assistant turns — # recomputes from `iterations`, so seeding it reproduces them exactly. @@ -952,6 +959,13 @@ def _finalize_result(self, start_time: float) -> None: self.result.completed_at = datetime.now() self.result.duration_seconds = time.time() - start_time + # Re-grade: the row keeps the agent run's duration (see + # _seed_from_prior_result). The grading pass's own cost is preserved + # alongside rather than discarded, so a slow judge is still visible. + if self.prior_result is not None: + self.result.environment_info["grading_duration_seconds"] = round(self.result.duration_seconds, 3) + self.result.duration_seconds = self.prior_result.duration_seconds + # Weighted score. This call site is wrapped because _finalize_result runs # inside run()'s finally — an unguarded raise here would skip persistence and # lose task.json. The other calculate_weighted_score calls (the simulation diff --git a/tests/test_execute_command.py b/tests/test_execute_command.py index 5f0f02da..be27c2a8 100644 --- a/tests/test_execute_command.py +++ b/tests/test_execute_command.py @@ -209,7 +209,6 @@ def _option_names(command: str) -> set[str]: # risk this test exists to close: a flag added to `run` must be added here too, # or consciously listed below as a deliberate omission. _DELIBERATELY_ABSENT_FROM_EXECUTE = { - "--resume", # partition_for_resume would treat a NOT_GRADED row as finalized "--junit-xml", # a report of verdicts, and there are none } diff --git a/tests/test_execute_evaluate_loop.py b/tests/test_execute_evaluate_loop.py index c9c7264b..9ab17528 100644 --- a/tests/test_execute_evaluate_loop.py +++ b/tests/test_execute_evaluate_loop.py @@ -145,3 +145,71 @@ def test_evaluate_rejects_workspace_flag_outside_run_dir_mode(tmp_path: Path) -> result = runner.invoke(app, ["evaluate", str(AGENTLESS_TASK), str(work), "--workspace", str(work)]) assert result.exit_code != 0 assert "run directory only" in result.output + + +# -------------------------------------------------------------------------- +# `run --resume` over an executed run dir +# -------------------------------------------------------------------------- + + +def test_run_resume_grades_the_ungraded_rows_it_finds(tmp_path: Path) -> None: + """The whole point of the resume fix: `run --resume` over an executed run + must GRADE those rows, not report "already complete" and exit 0.""" + run_dir = tmp_path / "r" + _invoke(["execute", str(AGENTLESS_TASK), "--run-dir", str(run_dir)]) + assert _row(_task_dir(run_dir))["final_status"] == FinalStatus.NOT_GRADED.value + + result = _invoke(["run", str(AGENTLESS_TASK), "--run-dir", str(run_dir), "--resume"]) + + assert "grading 1" in result.output + assert _row(_task_dir(run_dir))["final_status"] == FinalStatus.SUCCESS.value + summary = json.loads((run_dir / "run.json").read_text(encoding="utf-8")) + assert summary["tasks_not_graded"] == 0 + assert summary["tasks_succeeded"] == 1 + assert summary["pass_rate"] == 1.0 + + +def test_run_resume_does_not_re_execute_the_agent(tmp_path: Path) -> None: + """Grading must reuse the trajectory on disk. Re-executing would discard the + expensive half — the reason `execute` and `run` were split at all.""" + run_dir = tmp_path / "r" + _invoke(["execute", str(AGENTLESS_TASK), "--run-dir", str(run_dir)]) + executed = _row(_task_dir(run_dir)) + + result = _invoke(["run", str(AGENTLESS_TASK), "--run-dir", str(run_dir), "--resume"]) + + assert "running 0 remaining" in result.output, "the task was re-executed instead of graded" + graded = _row(_task_dir(run_dir)) + assert len(graded["iterations"]) == len(executed["iterations"]) + # The row still describes the TASK, not the grading pass: a re-execution + # would restamp these, and reporting the grading pass's 2s as the task's + # duration would corrupt average_duration and every harness comparison. + assert graded["started_at"] == executed["started_at"] + assert graded["duration_seconds"] == executed["duration_seconds"] + # The grading pass's own cost is kept alongside, not discarded. + assert "grading_duration_seconds" in graded["environment_info"] + # The pre-grade record is preserved by this path too, not just by `evaluate`. + assert _row(_task_dir(run_dir), "task.execute.json")["final_status"] == FinalStatus.NOT_GRADED.value + + +def test_execute_resume_treats_an_executed_row_as_done(tmp_path: Path) -> None: + """`execute --resume` owes a NOT_GRADED row nothing — it finished executing.""" + run_dir = tmp_path / "r" + _invoke(["execute", str(AGENTLESS_TASK), "--run-dir", str(run_dir)]) + + result = _invoke(["execute", str(AGENTLESS_TASK), "--run-dir", str(run_dir), "--resume"]) + + assert "1 task(s) already complete" in result.output + assert "grading" not in result.output + assert _row(_task_dir(run_dir))["final_status"] == FinalStatus.NOT_GRADED.value + + +def test_execute_to_run_resume_emits_no_config_drift_warning(tmp_path: Path) -> None: + """`grade` is exempt from the fingerprint diff: this flow is supported, and + the warning's "keeps their original-config results" text is wrong for it.""" + run_dir = tmp_path / "r" + _invoke(["execute", str(AGENTLESS_TASK), "--run-dir", str(run_dir)]) + + result = _invoke(["run", str(AGENTLESS_TASK), "--run-dir", str(run_dir), "--resume"]) + + assert "run config changed" not in result.output diff --git a/tests/test_resume.py b/tests/test_resume.py index 6f46084c..21282154 100644 --- a/tests/test_resume.py +++ b/tests/test_resume.py @@ -84,7 +84,7 @@ def test_partition_splits_finalized_from_pending(tmp_path): partial.run_dir.mkdir(parents=True, exist_ok=True) (partial.run_dir / "task.json").write_text(json.dumps({"task_id": "partial_task"}), encoding="utf-8") - to_run, prior_results, prior_resolved = partition_for_resume([done, pending, partial]) + to_run, _to_grade, prior_results, prior_resolved = partition_for_resume([done, pending, partial]) assert {rt.task.task_id for rt in to_run} == {"pending_task", "partial_task"} assert [tr.task_id for tr in prior_results] == ["done_task"] @@ -94,10 +94,77 @@ def test_partition_splits_finalized_from_pending(tmp_path): assert prior_results[0].duration == 12.5 +def test_partition_sends_ungraded_rows_to_grading_not_to_rerun(tmp_path): + """`run --resume` owes a NOT_GRADED row a GRADE, not another agent run. + + A NOT_GRADED row (written by `coder-eval execute`) carries a final status, so + the naive "has any final status" test calls it complete — which made + `run --resume` report "already complete", grade nothing, and exit 0. + """ + ungraded = _resolved(tmp_path, "ungraded_task") + graded = _resolved(tmp_path, "graded_task") + _write_task_json(ungraded, FinalStatus.NOT_GRADED) + _write_task_json(graded, FinalStatus.SUCCESS) + + part = partition_for_resume([ungraded, graded], grade=True) + + assert [rt.task.task_id for rt in part.to_grade] == ["ungraded_task"] + assert part.to_run == [], "an executed row must not be re-executed — that discards the agent spend" + assert [tr.task_id for tr in part.prior_results] == ["graded_task"] + + +def test_partition_treats_ungraded_as_done_for_execute(tmp_path): + """`execute --resume` owes a NOT_GRADED row nothing: it finished executing.""" + ungraded = _resolved(tmp_path, "ungraded_task") + _write_task_json(ungraded, FinalStatus.NOT_GRADED) + + part = partition_for_resume([ungraded], grade=False) + + assert part.to_grade == [] + assert part.to_run == [] + assert [tr.task_id for tr in part.prior_results] == ["ungraded_task"] + + +@pytest.mark.parametrize("status", [FinalStatus.FAILURE, FinalStatus.ERROR, FinalStatus.TIMEOUT]) +def test_partition_still_never_retries_failures(tmp_path, status): + """The NOT_GRADED carve-out must not become a general 'retry bad rows' rule. + + Resume has never retried failures (delete a task's task.json to force that), + and both commands must keep treating them as complete. + """ + failed = _resolved(tmp_path, "failed_task") + _write_task_json(failed, status) + + for grade in (True, False): + part = partition_for_resume([failed], grade=grade) + assert part.to_run == [], f"grade={grade} re-ran a {status.value} row" + assert part.to_grade == [], f"grade={grade} tried to re-grade a {status.value} row" + assert len(part.prior_results) == 1 + + +def test_grade_flag_is_exempt_from_the_resume_drift_warning(tmp_path): + """`execute` then `run --resume` is a supported flow, not a config mistake. + + The warning's text ("already-finalized tasks keep their original-config + results") is actively wrong for it: those rows are re-graded with the current + config, which is the whole point. + """ + executed = BatchRunConfig(run_dir=tmp_path, grade=False) + write_run_fingerprint(tmp_path, compute_run_fingerprint(executed, "exp1", "direct", None)) + prior = read_run_fingerprint(tmp_path) + + grading = BatchRunConfig(run_dir=tmp_path, grade=True) + assert fingerprint_diff(prior, compute_run_fingerprint(grading, "exp1", "direct", None)) == {} + + # ...but a real difference alongside it is still reported. + both = BatchRunConfig(run_dir=tmp_path, grade=True, overrides={"agent.model": "opus"}) + assert "overrides" in fingerprint_diff(prior, compute_run_fingerprint(both, "exp1", "direct", None)) + + def test_partition_no_run_dir_yields_all_pending(tmp_path): """--resume on a fresh dir degrades to a normal run (everything to_run).""" tasks = [_resolved(tmp_path, f"t{i}") for i in range(3)] - to_run, prior_results, prior_resolved = partition_for_resume(tasks) + to_run, _to_grade, prior_results, prior_resolved = partition_for_resume(tasks) assert len(to_run) == 3 assert prior_results == [] assert prior_resolved == [] @@ -114,7 +181,7 @@ async def test_run_batch_folds_prior_into_run_json(tmp_path): _write_task_json(p_success, FinalStatus.SUCCESS) _write_task_json(p_fail, FinalStatus.FAILURE) - _, prior_results, prior_resolved = partition_for_resume([p_success, p_fail]) + _, _to_grade, prior_results, prior_resolved = partition_for_resume([p_success, p_fail]) assert len(prior_results) == 2 # Nothing left to run — exercises the merge + run.json write in isolation. From e831e92946ae80b45b17780c96e2c6b318dc0a57 Mon Sep 17 00:00:00 2001 From: Akshaya Shanbhogue Date: Thu, 3 Sep 2026 15:14:17 -0700 Subject: [PATCH 4/6] fix(eval): close the verdict-correctness gaps in detached grading MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Full code review of the branch found two criticals and twelve highs. Every one of them is invisible to ruff/pyright/pytest/bandit/CodeQL, and every one of the worst produces a plausible number that is wrong rather than a crash. Verdict correctness * Gate selection is FIRED-ONLY, but only the AGENT path implemented it. The evaluate-only branch — the one a detached grade actually takes — called `all_criteria_passed` unconditionally, so `evaluate ` over an early-stopped run applied the full-run strict-AND gate to a truncated trajectory and could flip SUCCESS to FAILURE, then persist it. `early_stop` was seeded and read by nothing. Both paths now go through one `Orchestrator._select_gate()`. * `run()` calls the pre/post-run hooks unconditionally with `cwd = sandbox_dir`. On an adopted sandbox that is the agent's own output, and in-tree tasks stage fixtures there (`cp -a /app/[!.]* "$PWD/"`), so a detached grade overwrote the deliverables before the criteria read them. `Sandbox.was_adopted` now skips both, and their recorded results are carried from the prior run. * Grading may only move NOT_GRADED to SUCCESS/FAILURE. A prior TIMEOUT / ERROR / budget stop is an execution fact this pass neither repeated nor observed; `FinalStatus.is_execution_fact` (explicit map, no catch-all) preserves it. * The `reference_digest` guard was dead code — one grep hit in the whole tree, the read itself. The digest is now persisted at staging, resolves against the real task file, and RAISES on a vanished reference instead of returning. Counting and reporting * The evalboard rendered a clean `execute` run as 0% pass, N failed: every rate helper is `else failed++`, so an ungraded row was counted as a failure AND kept in the denominator. `StatusCategory` gains an explicit "ungraded" member; run-view, trends and watchlist exclude it from both sides. * `VariantResult.weighted_score` is `float | None`; `or 0.0` was laundering the ungraded None into a real-looking 0.000 that `_pick_best_variant` then ranked. * `SuiteRollup` gets the fourth bucket its two siblings have, plus the row-count invariant it was missing. `tasks_graded` is serialized on both aggregates. * `run --resume` exited 0 when every grade failed. The gate counts `tasks_not_graded` when grade is True; the reason is stamped on the row. Other * `evaluate`'s run-dir mode delegates to `regrade_in_place` instead of restating it. The copies had already drifted (hardcoded `replicate_index=0`). * `execute --driver docker` against an image predating `execute` silently graded; the returned row is now asserted NOT_GRADED. * A PATH restored from a run's own task.json is prepended ahead of the host's, so entries that do not exist or lie inside the graded workspace are dropped; shell commands rebuilt from a run dir's recorded config are announced. * `_seed_from_prior_result` also carries `agent_config`, `error_log_tail`, `expected_commands`, `simulation` and `sandbox_path`, which it was dropping. Tests: `test_seed_from_prior_result.py` partitions every `EvaluationResult` field as CARRIED or RECOMPUTED and fails closed on a new one; `test_regrade.py` covers the refusal branches (the digest guard's own test never reached it — the fixture had no reference, which is why the missing writer went unnoticed); `status.test.ts` covers the evalboard mirror, which had no test at all. make verify green (4654 passed, 92.14%); evalboard 621 tests green. Co-Authored-By: Claude Opus 5 (1M context) --- CLAUDE.md | 4 +- evalboard/app/runs/[id]/run-view.tsx | 13 +- evalboard/lib/__tests__/status.test.ts | 70 ++++++ evalboard/lib/runs.ts | 5 + evalboard/lib/status.ts | 24 +- evalboard/lib/trends.ts | 6 +- evalboard/lib/watchlist.ts | 9 +- src/coder_eval/cli/evaluate_command.py | 31 ++- src/coder_eval/cli/run_command.py | 16 +- src/coder_eval/isolation/docker_runner.py | 21 ++ src/coder_eval/models/enums.py | 32 +++ src/coder_eval/models/experiment.py | 13 +- src/coder_eval/models/results.py | 41 +++- src/coder_eval/orchestration/experiment.py | 23 +- src/coder_eval/orchestration/regrade.py | 95 +++++-- src/coder_eval/orchestrator.py | 214 ++++++++++++---- src/coder_eval/reports.py | 12 +- src/coder_eval/reports_experiment.py | 11 +- src/coder_eval/reports_html.py | 17 +- src/coder_eval/reports_stats.py | 19 ++ src/coder_eval/sandbox.py | 7 + tests/test_cleanup_preservation_guard.py | 4 + tests/test_cli_telemetry.py | 4 +- tests/test_execute_evaluate_loop.py | 49 ++++ tests/test_post_run.py | 20 +- tests/test_pre_run.py | 32 +-- tests/test_regrade.py | 259 ++++++++++++++++++++ tests/test_run_command_junit.py | 2 +- tests/test_run_metrics.py | 4 +- tests/test_seed_from_prior_result.py | 272 +++++++++++++++++++++ 30 files changed, 1192 insertions(+), 137 deletions(-) create mode 100644 evalboard/lib/__tests__/status.test.ts create mode 100644 tests/test_regrade.py create mode 100644 tests/test_seed_from_prior_result.py diff --git a/CLAUDE.md b/CLAUDE.md index f4e4642e..2b8ac212 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -151,8 +151,8 @@ action.yml # Published composite GitHub Action (coder-ev - **Harness run-limit parity**: a shared `BaseAgentConfig` field must mean the same thing on every backend, so a divergence is either fixed or documented — never silent. **`run_limits.max_turns` on Codex/Antigravity counts VISIBLE turns** (resolved tool calls, read live off the shared `EventCollector.visible_turn_count`, the same list `TurnRecord.commands` holds) because one `communicate()` is a single SDK turn on both, so a native counter would clamp at 1; claude-code keeps its native SDK cap, whose unit (an agent-loop turn) absorbs arbitrarily many parallel calls — the same number is NOT the same budget across harnesses. OpenCode likewise keeps a native unit — the CLI streams a real multi-step loop per `communicate()` (`step_start`/`step_finish`), so `max_turns: N` allows N complete steps and cuts cleanly when step N+1 begins. The cap is enforced on the same loop boundary as the cooperative early stop and finalizes cleanly as `max_turns_exhausted` (no crash, no retry); on Antigravity that boundary lives in `_drain()`, so the background-work poll loop honors it too. Known unfixed divergences: `permission_mode` on Codex and Antigravity (both run unconfined — the sandbox driver is the isolation boundary), `disallowed_tools` on Codex (forwarded, not SDK-enforced), `allowed_tools`/`disallowed_tools` on Antigravity (not read at all), `turn_timeout` on Antigravity (bounded by an earlier internal poll deadline at 80% of it), `allowed_tools`/`disallowed_tools`/`system_prompt` on OpenCode (no CLI knob; warned at `start()`, not enforced), and **`agent.plugins[].path` depth** — claude-code REQUIRES a plugin root holding `skills/` and silently loads NOTHING from a bare skills directory, while Codex and Antigravity scan both depths and accept either. That is the costly direction: the wrong depth produces no error, every positive row of an activation suite scores 0, and the suite reports recall 0.0, which reads exactly like a skill that never triggers. Held to the plugin-root shape (for `SKILL_SOURCE_PATH` only) by lint rule CE045. `plugins` on OpenCode is **honored for skills**: each local plugin root is mapped to the `skills` dir its `.claude-plugin/plugin.json` declares (default `/skills`, never the root itself — `skills.paths` is scanned recursively and a plugin root can hold a self-referential symlink) and injected via `OPENCODE_CONFIG_CONTENT`, which `--pure` does not suppress; a plugin's agents/hooks/commands/MCP servers are still dropped. Full table + rationale: docs/agents/HARNESS_PARITY.md. - **sandbox isolation**: Tasks that don't need MCP servers should set `setting_sources: []` in their `agent:` block to isolate the sandbox from the host project's CLAUDE.md and settings. Without this, the host project's CLAUDE.md (often 20 KB+) is injected into every API call, inflating cache-creation tokens and cost significantly. - **Execute vs. run (the grading switch)**: `coder-eval execute` is `coder-eval run` with grading removed — the agent runs and the full trajectory is captured, but no criterion is checked, `weighted_score` is `None` (never `0.0`, which would be indistinguishable from "graded and scored zero"), and the row finalizes as **`FinalStatus.NOT_GRADED`**, whose `category` is a **fourth** bucket, `"ungraded"`. Ungraded rows leave BOTH sides of every rate: `RunSummary.pass_rate` / `error_share` and `VariantAggregate.pass_rate` divide by `tasks_graded` (`tasks_run - tasks_not_graded`), and `tasks_not_graded` is part of the sum-to-`tasks_run` invariant, not a `tasks_failed` sub-counter. **Only SUCCESS/FAILURE collapse into it** — `ERROR`, `TIMEOUT`, `BUILD_FAILED`, `MAX_TURNS_EXHAUSTED` and the budget stops are facts about the *run*, not about grading, and still apply (so `execute` still exits non-zero on a crash). The switch is `BatchRunConfig.grade` → `Orchestrator(grade=...)` → the **four** grading call sites (single-shot, evaluate-only, the simulation dialog check, and post-failure diagnostics); it crosses the docker boundary in `context.json` (defaulting to `True` in-container, so a host predating `execute` keeps grading). It is **deliberately not a task-config field** — no 5-layer merge, no `-D` path — because a task YAML must never declare itself ungraded; only the invoking command decides. `run` and `execute` share one body (`run_command.run_pipeline`) and differ solely in that flag, so there is no third code path. Two things are refused rather than degraded: `--junit-xml` (a report of verdicts, and there are none — though `reports_junit` still emits `` for an ungraded row it encounters) and simulation tasks (their turn-continuation logic reads criteria results, so an ungraded dialog would silently change its own stopping behavior). `stop_early:` blocks are inert under `execute` for the same reason the kill switch exists: the full trajectory is the deliverable. Motivating consumer: an external harness (Harbor / Terminal-Bench 2.0) that builds its own container, calls coder-eval as the agent, and grades with its own tests. -- **Detached grading (`evaluate` over a run dir) + `Sandbox.adopt`**: `coder-eval evaluate` takes two shapes, told apart by a **pure** resolver (`cli/evaluate_target.py`) on one probe — a target holding `task.json` is a run directory. Run-dir mode rebuilds the task from the run's own `task_config.resolved`, **not** by re-loading the YAML: `resolved` is post-merge, so variant overrides / `-D` / dataset expansion are already baked in and re-loading the source would silently grade a *different* task (fallback to `source_file` only when `resolved` no longer validates, and loudly). It seeds the fresh result from the prior one via `Orchestrator(prior_result=...)` → `_seed_from_prior_result`, which carries the trajectory (every derived figure — tokens, cost, `command_stats`, `model_used` — recomputes from `iterations`), `iteration_count`, execution facts, and **`early_stop` — load-bearing, because gate selection is FIRED-ONLY**: dropping it re-grades a truncated trajectory under the full-run strict-AND gate and can flip the verdict. Grader-host `environment_info` is preserved under a `graded_by` sub-dict rather than overwriting the run's. Two other parity fixes: `command_base_path` is now persisted into `environment_info` by `_sync_sandbox_command_path_with_agent` and restored in the evaluate-only branch (closing the PATH gap that method's docstring already named), and `_join_litellm_actual_cost` **skips** when `prior_result` is set (its join keys on a per-Orchestrator nonce the prior turns never carried, so it would clobber already-correct costs). The verdict is written back into the run's `task.json`, with the pre-grade record kept as `task.execute.json` — that in-place write is what makes plain `coder-eval aggregate ` rebuild a graded `run.json` with **zero** new code. **`Sandbox.adopt(workspace)`** is the grade-in-place primitive: it reuses `setup`'s adoption half but skips every *materializing* step (`_setup_template`, `_generate_cli_recorders`, venv/package installs, the destructive `$HOME` remediation), running only non-mutating derivation (mock-dir `+x`, venv *discovery*, plugin-tools pin); `_cleanup_on_exit` stays False so an adopted tree is never moved or deleted. In-place is **more correct**, not merely faster: `_setup_template` filters the copy through `_should_ignore_template_file`, which drops `node_modules` / `dist` / `build` / `.venv` / `.git`, so on the copy path a criterion like `test -f dist/bundle.js` fails as a *copying artifact* rather than as a verdict (verified: 0.00 "does not exist" on copy vs 1.00 in place). Defaults: in-place for a run dir, copy for a bare work dir (criteria can mutate it and it is the user's own tree); `--in-place`/`--copy` override. `adopt` hard-errors on `driver: docker` (a container workspace is unreachable from the host) and a re-grade refuses on a `reference_digest` mismatch. NOTE the Typer command is a thin wrapper over `run_evaluation(...)`, which has real Python defaults — calling a Typer command function in-process hands unspecified options an `OptionInfo` sentinel, which silently made `in_place=None` truthy. -- **`--resume` is command-relative**: `partition_for_resume(tasks, *, grade)` returns a four-way `ResumePartition` (`to_run` / `to_grade` / `prior_results` / `prior_resolved`), because **"finished" is not absolute — it depends on what the resuming command still owes the task**. A `NOT_GRADED` row carries a final status, so the original "has any final status" test called it complete: right for `execute --resume` (it finished executing), and wrong for `run --resume`, which was asked to grade and would instead report "already complete", grade nothing, and **exit 0**. Under `grade=True` those rows route to `to_grade`, where `_grade_resumed_tasks` runs the criteria against the trajectory and workspace already on disk via `orchestration/regrade.py::regrade_in_place` — reusing the agent spend, which is the entire reason `execute` and `run` are separate. The carve-out is **only** for `NOT_GRADED`: `FAILURE`/`ERROR` stay complete under both commands (resume has never retried failures — delete the task.json), and `clear_rerun_artifacts` deliberately skips `to_grade`, whose artifacts are the very thing being graded. A per-task grading failure is warned and folded back in with its ORIGINAL ungraded result, so one bad row neither aborts the resume nor vanishes from run.json. `grade` is in `_FINGERPRINT_DIFF_EXEMPT` because `execute` → `run --resume` is a supported flow, not config drift — and the warning's "already-finalized tasks keep their original-config results" text is actively wrong for it. **`orchestration/regrade.py` is the single implementation** shared by that path and `evaluate`'s run-dir mode (two copies would drift into two verdicts for the same run); it raises plain `RegradeError`, which the CLI wraps, since `orchestration/` must not import the CLI layer (CE004). One fidelity rule it enforces: a re-graded row keeps the **agent run's** `started_at`/`duration_seconds`, not the grading pass's — a 10-minute run re-graded in 2s would otherwise report 2s into `average_duration`, the report tables and the evalboard; the grading cost is preserved separately as `environment_info["grading_duration_seconds"]`. +- **Detached grading (`evaluate` over a run dir) + `Sandbox.adopt`**: `coder-eval evaluate` takes two shapes, told apart by a **pure** resolver (`cli/evaluate_target.py`) on one probe — a target holding `task.json` is a run directory. Run-dir mode rebuilds the task from the run's own `task_config.resolved`, **not** by re-loading the YAML: `resolved` is post-merge, so variant overrides / `-D` / dataset expansion are already baked in and re-loading the source would silently grade a *different* task (fallback to `source_file` only when `resolved` no longer validates, and loudly). It seeds the fresh result from the prior one via `Orchestrator(prior_result=...)` → `_seed_from_prior_result`, which carries the trajectory (every derived figure — tokens, cost, `command_stats`, `model_used` — recomputes from `iterations`), `iteration_count`, execution facts, and **`early_stop` — load-bearing, because gate selection is FIRED-ONLY**: dropping it re-grades a truncated trajectory under the full-run strict-AND gate and can flip the verdict. Carrying it is only half the fix — **both** grading paths select the gate through the single `Orchestrator._select_gate()`; the evaluate-only branch a detached grade actually takes originally called `all_criteria_passed` inline, so the seeded field was written and never read. `tests/test_seed_from_prior_result.py` partitions every `EvaluationResult` field as CARRIED or RECOMPUTED and fails closed on a new one, and asserts the two `_select_gate()` call sites. A prior status that `FinalStatus.is_execution_fact` (TIMEOUT / ERROR / BUILD_FAILED / the budget stops) is **preserved**, never overwritten: grading may only move `NOT_GRADED` to SUCCESS/FAILURE, since it neither repeated nor observed the agent phase. Grader-host `environment_info` is preserved under a `graded_by` sub-dict rather than overwriting the run's. Two other parity fixes: `command_base_path` is now persisted into `environment_info` by `_sync_sandbox_command_path_with_agent` and restored in the evaluate-only branch (closing the PATH gap that method's docstring already named), and `_join_litellm_actual_cost` **skips** when `prior_result` is set (its join keys on a per-Orchestrator nonce the prior turns never carried, so it would clobber already-correct costs). The verdict is written back into the run's `task.json`, with the pre-grade record kept as `task.execute.json` — that in-place write is what makes plain `coder-eval aggregate ` rebuild a graded `run.json` with **zero** new code. **`Sandbox.adopt(workspace)`** is the grade-in-place primitive: it reuses `setup`'s adoption half but skips every *materializing* step (`_setup_template`, `_generate_cli_recorders`, venv/package installs, the destructive `$HOME` remediation), running only non-mutating derivation (mock-dir `+x`, venv *discovery*, plugin-tools pin); `_cleanup_on_exit` stays False so an adopted tree is never moved or deleted, and `Sandbox.was_adopted` is set — the Orchestrator reads it to SKIP the `pre_run`/`post_run` hooks (`run()` calls them unconditionally with `cwd = sandbox_dir`, and several in-tree tasks stage fixtures there with `cp -a /app/[!.]* "$PWD/"`, which would overwrite the agent's deliverables before the criteria read them) and to KEEP `sandbox_path` in the `PreservationMode.NONE` cleanup arm (an adopted tree survives cleanup, so the path is not stale). The hooks' recorded results are carried from the prior run instead. In-place is **more correct**, not merely faster: `_setup_template` filters the copy through `_should_ignore_template_file`, which drops `node_modules` / `dist` / `build` / `.venv` / `.git`, so on the copy path a criterion like `test -f dist/bundle.js` fails as a *copying artifact* rather than as a verdict (verified: 0.00 "does not exist" on copy vs 1.00 in place). Defaults: in-place for a run dir, copy for a bare work dir (criteria can mutate it and it is the user's own tree); `--in-place`/`--copy` override. `adopt` hard-errors on `driver: docker` (a container workspace is unreachable from the host) and a re-grade refuses on a `reference_digest` mismatch — the digest is persisted into `environment_info` at staging time by `_stage_reference` (it shipped once as a read with no writer anywhere, so the guard was dead code), and `verify_reference_unchanged` now takes the task file it resolves against and RAISES on a vanished or unresolvable reference instead of returning silently. NOTE the Typer command is a thin wrapper over `run_evaluation(...)`, which has real Python defaults — calling a Typer command function in-process hands unspecified options an `OptionInfo` sentinel, which silently made `in_place=None` truthy. +- **`--resume` is command-relative**: `partition_for_resume(tasks, *, grade)` returns a four-way `ResumePartition` (`to_run` / `to_grade` / `prior_results` / `prior_resolved`), because **"finished" is not absolute — it depends on what the resuming command still owes the task**. A `NOT_GRADED` row carries a final status, so the original "has any final status" test called it complete: right for `execute --resume` (it finished executing), and wrong for `run --resume`, which was asked to grade and would instead report "already complete", grade nothing, and **exit 0**. Under `grade=True` those rows route to `to_grade`, where `_grade_resumed_tasks` runs the criteria against the trajectory and workspace already on disk via `orchestration/regrade.py::regrade_in_place` — reusing the agent spend, which is the entire reason `execute` and `run` are separate. The carve-out is **only** for `NOT_GRADED`: `FAILURE`/`ERROR` stay complete under both commands (resume has never retried failures — delete the task.json), and `clear_rerun_artifacts` deliberately skips `to_grade`, whose artifacts are the very thing being graded. A per-task grading failure is warned, STAMPED onto the folded-back row's `error_message` (the console line alone is not durable), and folded back in with its ORIGINAL ungraded result, so one bad row neither aborts the resume nor vanishes from run.json — and the exit gate counts `tasks_not_graded` **when `grade` is True**, so a `run` that graded nothing exits non-zero instead of telling CI the suite is fine. Under `execute` an ungraded row is the expected outcome and never fails the command. `grade` is in `_FINGERPRINT_DIFF_EXEMPT` because `execute` → `run --resume` is a supported flow, not config drift — and the warning's "already-finalized tasks keep their original-config results" text is actively wrong for it. **`orchestration/regrade.py` is the single implementation** shared by that path and `evaluate`'s run-dir mode, which DELEGATES to `regrade_in_place` rather than restating it (it originally hand-built its own Sandbox + Orchestrator and had already drifted — hardcoding `replicate_index=0`, so every replicate but the first was relabelled — which is exactly how two copies become two verdicts for the same run); it raises plain `RegradeError`, which the CLI wraps, since `orchestration/` must not import the CLI layer (CE004). One fidelity rule it enforces: a re-graded row keeps the **agent run's** `started_at`/`duration_seconds`, not the grading pass's — a 10-minute run re-graded in 2s would otherwise report 2s into `average_duration`, the report tables and the evalboard; the grading cost is preserved separately as `environment_info["grading_duration_seconds"]`. - **Run-time caps (non-criterion enforcement)**: `TaskDefinition.run_limits` (`RunLimits` model) is the single namespace for all *task-level* run-time caps — `max_turns` / `task_timeout` / `turn_timeout` (structural) and `max_input_tokens` / `max_output_tokens` / `max_total_tokens` / `max_usd` (cumulative budget). Token/USD breaches abort with `FinalStatus.TOKEN_BUDGET_EXCEEDED` or `COST_BUDGET_EXCEEDED` (both `category == "failed"`). Structural caps are set from the CLI via `-D run_limits.max_turns=…` / `-D run_limits.task_timeout=…` / `-D run_limits.turn_timeout=…` (field-merged into `run_limits`); budget caps via `-D run_limits.max_usd=…` etc. or YAML. Layered config uses field-merge — a variant block overrides individual keys without replacing the task's block. The one *per-criterion* cap, `stop_early.decide_within`, deliberately lives on `LiveSuccessCriterion` instead (see below) — the watcher must attribute a decision-step timeout to a specific criterion, which `RunLimits` (task-scoped, criterion-agnostic) cannot express. - **Early stop on criterion (opt-in, per-criterion arming)**: a `stop_early:` block (`StopEarlyPolicy`) on a criterion ends a single-shot run early once the run's **armed** criteria decide the outcome, so a raised `max_turns` isn't wasted on the smoke flavor. The block's PRESENCE is the arming and alone activates the watcher — there is **no run-level master switch**: `run_limits.stop_early: false` is the run-level KILL SWITCH that force-disarms every block (the one-line experiment-variant/`-D` override for an authoritative full run), and `run_limits.stop_early: true` (the removed master arm) is a hard `EarlyStopConfigError` at resolution. The block exists on `LiveSuccessCriterion` only (currently `skill_triggered`, `command_executed` — so arming an unobservable criterion is unrepresentable, a pydantic extra-forbid error). Arming carries one implicit trigger (a native live-fail may fail-stop the run); its keys refine it: `on_pass: stop` (pass-stop the moment the criterion live-passes; default `continue` just latches) and `decide_within: N` (still undecided after N tool-call steps latches an **effective fail**, fed through the same fail-stop rule, reported as `decision_budget_exceeded` — an ordinary weighted fail, NOT a gate-bypassing force-fail; cumulative across retry attempts of the same turn). A trigger whose polarity the instance can't decide (per the abstract, checker-independent `live_decidable_polarities()`, a pure function of the criterion's own fields, paired with the checker's `live_verdict` override by lint rule CE025, a registry-based whole-tree check) is **inert by design** — one dataset-fanned YAML line serves both positive rows (pass/timeout live) and distractor rows (fail live). Verdicts **latch**: once a criterion decides, its `live_verdict` is never polled again. Stop rule is weighted, not strict-boolean: `run_limits.stop_early_gate_threshold` (default `1.0`, reproducing strict-AND behavior exactly) is the minimum weighted score (`Σ weight·score / Σ weight` over the armed subset) required to pass; a fail-stop fires once the armed set's **ceiling** (best case for everything still undecided) can no longer reach the threshold — so a low-weight fail or timeout that can't doom the gate is absorbed and the run continues — and is **deferred while any pass-capable armed criterion is undecided** (a distractor misfire never truncates a positive row's recall signal); a pass-stop fires once the `on_pass: stop` subset's **floor** (worst case) already meets the threshold, and is symmetrically **deferred while any pass-capable armed criterion outside the `on_pass: stop` subset is undecided** (so an early pass never freezes a sibling `on_pass: continue` criterion's signal out of the trajectory). A fail-stop is therefore verdict-preserving; a pass-stop can miss a *later* distractor misfire, so authoritative P/R/F1 comes from a kill-switched (`stop_early: false`) run. Driven by `orchestration/early_stop.py::EarlyStopWatcher` (built when `early_stop_active(task)`: ≥1 armed criterion, kill switch not thrown) through the agent's cooperative `should_stop` seam (tool-call granularity, no SIGKILL); live verdicts only *trigger* the stop — the standard `check_all_async` on the frozen trajectory is authoritative. Gating is **FIRED-ONLY**: a run the watcher actually cut gates on the **armed subset** via the weighted `EvaluationResult.armed_criteria_passed`; a run that completes naturally — armed or not — gates strict-AND via `all_criteria_passed`, so adding a block never changes the verdict of a run it didn't cut. Note the gate keys on the watcher having FIRED (`result.early_stop is not None`), not on confirmed truncation — an agent that ignores `should_stop`, or a stop firing on the final message, still gates armed-only. Every resolution-time guardrail violation is a hard error at resolution (plan *and* run); the one load-time case — a `stop_early:` block on a non-live criterion — is a pydantic schema error at task load, which the run surface reports as a skipped task like any other malformed task. A runtime verdict bug **fails open** to a full run. Surfaces: `EarlyStopInfo` (incl. `gate_threshold` at stop time), report notes/badges, `stopped_early` run.json rows, `EarlyStopped`/`EarlyStopReason` telemetry dims. Worked rationale: docs/TASK_DEFINITION_GUIDE.md § `stop_early`. No blocks anywhere ⇒ behavior byte-for-byte unchanged. diff --git a/evalboard/app/runs/[id]/run-view.tsx b/evalboard/app/runs/[id]/run-view.tsx index a5171fe5..40374b03 100644 --- a/evalboard/app/runs/[id]/run-view.tsx +++ b/evalboard/app/runs/[id]/run-view.tsx @@ -38,6 +38,9 @@ export interface RunMetrics { passed: number; failed: number; errored: number; + // Rows that ran but were never scored (`coder-eval execute`). Excluded from + // both sides of `pct`, so a fully ungraded run reports 0 of 0, not 0%. + ungraded: number; failedTotal: number; pct: number; // Per-task view of pass rate for repeated runs: distinct task_ids, and how @@ -72,6 +75,7 @@ export function computeRunMetrics(tasks: TaskResultSummary[]): RunMetrics { let passed = 0; let failed = 0; let errored = 0; + let ungraded = 0; let cost = 0; let durationSum = 0; const costSamples: number[] = []; @@ -80,6 +84,11 @@ export function computeRunMetrics(tasks: TaskResultSummary[]): RunMetrics { const cat = statusCategory(t.status); if (cat === "passed") passed++; else if (cat === "error") errored++; + // An ungraded row (`coder-eval execute`) was never scored. It leaves + // BOTH sides of the rate — the `else failed++` below would otherwise + // count it as a failure AND keep it in the denominator, rendering a + // clean execute run as 0% pass, N failed. + else if (cat === "ungraded") ungraded++; else failed++; if (t.matureSkipped) continue; if (t.totalCostUsd != null) { @@ -91,13 +100,15 @@ export function computeRunMetrics(tasks: TaskResultSummary[]): RunMetrics { durSamples.push(t.durationSeconds); } } + const graded = total - ungraded; return { total, passed, failed, errored, + ungraded, failedTotal: failed + errored, - pct: total ? (passed / total) * 100 : 0, + pct: graded ? (passed / graded) * 100 : 0, ...(() => { // Per-task rollup (any replicate passed → task passed) via the shared // helper, so the run tile and the grid badge apply the same rule. diff --git a/evalboard/lib/__tests__/status.test.ts b/evalboard/lib/__tests__/status.test.ts new file mode 100644 index 00000000..a91db0a6 --- /dev/null +++ b/evalboard/lib/__tests__/status.test.ts @@ -0,0 +1,70 @@ +import { describe, expect, it } from "vitest"; + +import { + isGraded, + isPassStatus, + statusCategory, + statusSortRank, + type StatusCategory, +} from "../status"; + +// This module mirrors coder_eval's FinalStatus.category (models/enums.py), which +// is guarded there by `assert set(_STATUS_CATEGORIES) == set(FinalStatus)`. The +// mirror had no test at all, which is how NOT_GRADED came to be categorized as +// "unknown" and then counted as a failure by every rate helper downstream. +const EVERY_FINAL_STATUS: Record = { + SUCCESS: "passed", + FAILURE: "failed", + ERROR: "error", + BUILD_FAILED: "error", + TIMEOUT: "failed", + MAX_TURNS_EXHAUSTED: "failed", + TOKEN_BUDGET_EXCEEDED: "failed", + COST_BUDGET_EXCEEDED: "failed", + NOT_GRADED: "ungraded", +}; + +describe("statusCategory", () => { + it.each(Object.entries(EVERY_FINAL_STATUS))( + "maps %s to %s", + (status, expected) => { + expect(statusCategory(status)).toBe(expected); + }, + ); + + it("treats a missing status as unknown, distinct from ungraded", () => { + expect(statusCategory(null)).toBe("unknown"); + expect(statusCategory(null)).not.toBe(statusCategory("NOT_GRADED")); + }); + + it("does not classify an ungraded row as a pass or a failure", () => { + // The whole point of the fourth category: folding it into either side + // of a rate misreports a run that was never scored. + expect(statusCategory("NOT_GRADED")).not.toBe("passed"); + expect(statusCategory("NOT_GRADED")).not.toBe("failed"); + expect(isPassStatus("NOT_GRADED")).toBe(false); + }); +}); + +describe("isGraded", () => { + it("is false only for an ungraded row", () => { + expect(isGraded("NOT_GRADED")).toBe(false); + for (const status of Object.keys(EVERY_FINAL_STATUS)) { + if (status === "NOT_GRADED") continue; + expect(isGraded(status)).toBe(true); + } + // A null status is "no row here", not "ran but unscored". + expect(isGraded(null)).toBe(true); + }); +}); + +describe("statusSortRank", () => { + it("sorts failures first, ungraded in the middle, passes last", () => { + expect(statusSortRank("FAILURE")).toBeLessThan( + statusSortRank("NOT_GRADED"), + ); + expect(statusSortRank("NOT_GRADED")).toBeLessThan( + statusSortRank("SUCCESS"), + ); + }); +}); diff --git a/evalboard/lib/runs.ts b/evalboard/lib/runs.ts index ad31a97d..9da79c88 100644 --- a/evalboard/lib/runs.ts +++ b/evalboard/lib/runs.ts @@ -60,6 +60,7 @@ export interface RunSummary { tasksSucceeded: number; tasksFailed: number; tasksError: number; + tasksNotGraded: number; totalCostUsd: number | null; componentShas: ComponentSha[]; // What actually produced this run, for the run header. `harness` is the @@ -463,6 +464,9 @@ interface RawRunJson { tasks_succeeded?: number; tasks_failed?: number; tasks_error?: number; + // Rows that ran but were never scored (`coder-eval execute`). Optional: a + // run.json written before the field existed simply has none. + tasks_not_graded?: number; task_results?: RawTaskResult[]; // Values are scalars except `tool_plugins`, a {plugin: version} map of // the installed @uipath/*-tool packages (recorded since coder_eval #366). @@ -831,6 +835,7 @@ export async function readRunSummary( tasksSucceeded: data.tasks_succeeded ?? 0, tasksFailed: data.tasks_failed ?? 0, tasksError: data.tasks_error ?? 0, + tasksNotGraded: data.tasks_not_graded ?? 0, totalCostUsd: taskResults.length ? totalCost : null, componentShas: extractComponentShas(data.environment_info), harness: extractRunConfig(data).harness, diff --git a/evalboard/lib/status.ts b/evalboard/lib/status.ts index 17796f4c..7d261aa9 100644 --- a/evalboard/lib/status.ts +++ b/evalboard/lib/status.ts @@ -2,13 +2,16 @@ // Mirrors coder_eval `FinalStatus.category` (src/coder_eval/models/enums.py): // SUCCESS -> passed // ERROR / BUILD_FAILED -> error (BUILD_FAILED is an environment/setup failure) -// NOT_GRADED -> unknown (`coder-eval execute`: ran, deliberately unscored) +// NOT_GRADED -> ungraded (`coder-eval execute`: ran, deliberately unscored) // anything else (FAILURE, TIMEOUT, MAX_TURNS_EXHAUSTED, …) -> failed // -// NOT_GRADED maps to "unknown" rather than gaining a category of its own: every -// consumer already handles "unknown" (a null status) as "no verdict here", which -// is exactly what an ungraded row is. It is therefore not a pass, not a failure, -// and sorts in the middle — the same treatment a missing status gets. +// "ungraded" is its OWN member rather than being folded into "unknown". Folding +// it there looks safe — an ungraded row genuinely has no verdict — but every +// rate helper in this app is written as `if passed … else if error … else +// failed++`, so anything that is not a pass or an error is counted as a failure +// AND kept in the denominator. A clean `execute` run then renders as 0% pass, N +// failed. A distinct member makes that a type error at each site instead, so a +// consumer has to decide what to do with it. // // Note: this only categorizes coder_eval task statuses. UI status display // (e.g. StatusPill) also handles flow execution statuses like "Completed" @@ -16,16 +19,23 @@ import { taskVariantKey } from "./variants"; -export type StatusCategory = "passed" | "failed" | "error" | "unknown"; +export type StatusCategory = "passed" | "failed" | "error" | "ungraded" | "unknown"; export function statusCategory(status: string | null): StatusCategory { if (!status) return "unknown"; if (status === "SUCCESS") return "passed"; if (status === "ERROR" || status === "BUILD_FAILED") return "error"; - if (status === "NOT_GRADED") return "unknown"; + if (status === "NOT_GRADED") return "ungraded"; return "failed"; } +// Whether a row was measured at all. An ungraded row must leave BOTH sides of +// every rate — it is not a pass and not a failure, so counting it either way +// (or keeping it in a denominator) misreports a run that was never scored. +export function isGraded(status: string | null): boolean { + return statusCategory(status) !== "ungraded"; +} + // Whether a status is a pass (SUCCESS). The single predicate behind the // "a task passes if any replicate passed" rule. export function isPassStatus(status: string | null): boolean { diff --git a/evalboard/lib/trends.ts b/evalboard/lib/trends.ts index 24128a8e..2480fffc 100644 --- a/evalboard/lib/trends.ts +++ b/evalboard/lib/trends.ts @@ -12,6 +12,7 @@ import { } from "./overview"; import { DEFAULT_HARNESS } from "./harness"; import { DEFAULT_SOURCE, type Source } from "./sources"; +import { isGraded } from "./status"; import { taskCarriesRepoTag } from "./tags"; import type { ComponentSha } from "./runs"; @@ -151,7 +152,10 @@ export function aggregate(perRun: PerRun[]): TrendsData { } if (!b.skill && t.skill) b.skill = t.skill; for (const tg of t.tags) b.tagSet.add(tg); - b.totalCount += 1; + // An ungraded row (`coder-eval execute`) was never scored, so it + // enters neither side of the pass rate. Counting it in totalCount + // alone would drag a task's trend down as if it had failed. + if (isGraded(t.status)) b.totalCount += 1; if (t.matureSkipped) b.matureSkips += 1; b.statuses.push({ runId: id, diff --git a/evalboard/lib/watchlist.ts b/evalboard/lib/watchlist.ts index ae215881..79da9ebf 100644 --- a/evalboard/lib/watchlist.ts +++ b/evalboard/lib/watchlist.ts @@ -11,6 +11,7 @@ import type { PerRun } from "./overview"; import type { RunOverviewTask } from "./runs"; +import { isGraded } from "./status"; import { timeRatio } from "./timing"; import { turnRatio } from "./turns"; @@ -116,7 +117,9 @@ function stdev(xs: number[]): number { function skillPassSeq(runs: LoadedRun[], skill: string): number[] { const seq: number[] = []; for (const run of runs) { - const ts = run.tasks.filter((t) => t.skill === skill); + const ts = run.tasks.filter( + (t) => t.skill === skill && isGraded(t.status), + ); if (ts.length === 0) continue; seq.push(ts.filter((t) => isPass(t.status)).length / ts.length); } @@ -137,6 +140,8 @@ export function leaderboard(runs: LoadedRun[]): LeaderboardRow[] { for (const run of runs) { for (const t of run.tasks) { if (!t.skill) continue; + // Ungraded rows leave both sides of the rate — see isGraded. + if (!isGraded(t.status)) continue; total.set(t.skill, (total.get(t.skill) ?? 0) + 1); if (isPass(t.status)) passed.set(t.skill, (passed.get(t.skill) ?? 0) + 1); @@ -202,6 +207,8 @@ export function attention(runs: LoadedRun[]): AttentionRow[] { const ts = run.tasks.filter((t) => t.skill === skill); if (ts.length > 0) appeared++; for (const t of ts) { + // Ungraded rows leave both sides of the rate — see isGraded. + if (!isGraded(t.status)) continue; outcomes++; taskIds.add(t.taskId); if (isPass(t.status)) passes++; diff --git a/src/coder_eval/cli/evaluate_command.py b/src/coder_eval/cli/evaluate_command.py index 7de52fc3..8c61afc9 100644 --- a/src/coder_eval/cli/evaluate_command.py +++ b/src/coder_eval/cli/evaluate_command.py @@ -26,6 +26,7 @@ back_up_pre_grade_record, default_workspace, load_prior_result, + regrade_in_place, task_from_prior, verify_reference_unchanged, ) @@ -115,7 +116,7 @@ def _resolve_run_dir_or_work_dir(target: EvaluateTarget, workspace: Path | None) task.agent = parse_agent_config(**{**task.agent.model_dump(exclude_unset=True), "type": AgentKind.CLAUDE_CODE}) if prior is not None: - verify_reference_unchanged(prior, task) + verify_reference_unchanged(prior, task, task_file) return _ResolvedInputs( target=target, @@ -127,6 +128,18 @@ def _resolve_run_dir_or_work_dir(target: EvaluateTarget, workspace: Path | None) ) +def _replicate_index_of(run_dir: Path) -> int: + """Recover the replicate index a run directory encodes in its leaf name. + + Preservation lays runs out as ``///``. Hardcoding 0 + would relabel every replicate but the first as replicate 0. + """ + try: + return int(run_dir.name) + except ValueError: + return 0 + + def evaluate_command( task_or_run_dir: Path = typer.Argument( # noqa: B008 ..., @@ -261,6 +274,22 @@ def run_evaluation( sandbox = Sandbox(sandbox_config, task_id=task.task_id, task_dir=task_dir) async def _setup_and_run() -> EvaluationResult: + if grade_in_place and prior is not None: + # Delegate to the shared re-grade core. Restating its body here is + # how this path and `run --resume` came to differ (replicate_index, + # error semantics) while CLAUDE.md called regrade.py the single + # implementation — two copies of "how to re-grade" drift into two + # verdicts for the same run. + return await regrade_in_place( + task=task, + prior=prior, + workspace=graded_dir, + run_dir=prepared_run_dir, + task_file=task_file, + source_yaml=source_yaml, + variant_id=prior.variant_id, + replicate_index=_replicate_index_of(target.target), + ) if grade_in_place: await asyncio.to_thread(sandbox.adopt, graded_dir) else: diff --git a/src/coder_eval/cli/run_command.py b/src/coder_eval/cli/run_command.py index 65337c65..5c3de001 100644 --- a/src/coder_eval/cli/run_command.py +++ b/src/coder_eval/cli/run_command.py @@ -604,7 +604,14 @@ async def _run_all_tasks( flush_telemetry() # Exit with non-zero code if any tasks failed, errored, or any suite failed its thresholds. - if summary.tasks_failed > 0 or summary.tasks_error > 0 or failed_suite_gates > 0: + # + # An ungraded row counts too, but only under `run`: `run` was asked for a + # verdict and did not produce one (the grade crashed, or --resume could not + # grade the row), which is a failure of the command even though the row is + # neither `failed` nor `error`. Under `execute` an ungraded row is the + # expected outcome for every task, so it must not fail the command. + ungraded_but_asked_to_grade = grade and summary.tasks_not_graded > 0 + if summary.tasks_failed > 0 or summary.tasks_error > 0 or failed_suite_gates > 0 or ungraded_but_asked_to_grade: raise typer.Exit(1) @@ -686,7 +693,7 @@ async def _grade_resumed_tasks(to_grade: list[ResolvedTask]) -> list[tuple[Resol for rt in to_grade: prior = load_prior_result(rt.run_dir) try: - verify_reference_unchanged(prior, rt.task) + verify_reference_unchanged(prior, rt.task, rt.task_file) workspace = default_workspace(rt.run_dir, prior) # Preserve the ungraded record BEFORE the orchestrator overwrites # task.json in this same directory. @@ -703,7 +710,12 @@ async def _grade_resumed_tasks(to_grade: list[ResolvedTask]) -> list[tuple[Resol ) except (RegradeError, OSError, RuntimeError, ValueError) as e: console.print(f"[yellow]⚠[/] Could not grade {rt.task.task_id}: {e}") + # Stamp the reason onto the row. Without it the failure survives only + # in this console line: the folded-back result keeps the execute + # phase's empty error_message, so run.json, the reports and CI show + # an ungraded row with no explanation of why grading never happened. result = prior + result.error_message = f"Grading failed during --resume: {e}" graded.append( ( rt, diff --git a/src/coder_eval/isolation/docker_runner.py b/src/coder_eval/isolation/docker_runner.py index bb967222..3b46631b 100644 --- a/src/coder_eval/isolation/docker_runner.py +++ b/src/coder_eval/isolation/docker_runner.py @@ -871,8 +871,29 @@ async def _parse_result_or_raise(self, output_dir: Path, returncode: int, log_pa # rather than crashing with an uncaught ValidationError/JSONDecodeError. raise await self._handle_malformed_task_json(task_json, log_path, exc) from exc self._warn_on_version_mismatch(result) + self._assert_grade_honored(result) return result + def _assert_grade_honored(self, result: EvaluationResult) -> None: + """Fail loudly when `execute` came back with a graded verdict. + + ``grade`` crosses the boundary only through ``context.json``. An image + that predates ``execute`` ignores the unknown key and grades anyway, and + the image-version preflight only warns — so ``execute --driver docker`` + against a stale image would silently produce SUCCESS/FAILURE rows that + look like a normal graded run. Version skew must not change what a + command MEANS, so refuse the row rather than publish it. + """ + if self.grade or result.final_status.is_execution_fact: + return + if result.final_status is not FinalStatus.NOT_GRADED: + raise DockerRunError( + "`coder-eval execute` asked the container not to grade, but it returned " + + f"{result.final_status.value} with {len(result.success_criteria_results)} criterion " + + "result(s). The runtime image predates `execute` and ignored the request; " + + "rebuild or pull a matching agent image." + ) + async def _handle_malformed_task_json(self, task_json: Path, log_path: Path, exc: ValueError) -> DockerRunError: """Degrade a present-but-malformed task.json; return the DockerRunError to raise. diff --git a/src/coder_eval/models/enums.py b/src/coder_eval/models/enums.py index 7135de37..f4586b6a 100644 --- a/src/coder_eval/models/enums.py +++ b/src/coder_eval/models/enums.py @@ -33,6 +33,19 @@ def icon(self) -> str: """Single-character icon for reports and CLI output.""" return _STATUS_ICONS[self] + @property + def is_execution_fact(self) -> bool: + """True when this status records HOW THE RUN ENDED, not what grading decided. + + A detached grade (``evaluate `` / ``run --resume``) re-runs the + criteria over a trajectory it did not produce, so it may only move a row + between the three GRADING outcomes — ``NOT_GRADED`` -> ``SUCCESS`` / + ``FAILURE``. It must never launder a run that timed out, crashed, or blew + a budget into a pass: those statuses describe the agent phase, which the + grading pass neither repeated nor observed. + """ + return _EXECUTION_FACT_STATUSES[self] + # Every FinalStatus maps to exactly one reporting category, listed EXPLICITLY (no # catch-all default) so a newly-added status fails the assert below until it is @@ -76,6 +89,25 @@ def icon(self) -> str: assert set(_STATUS_ICONS) == set(FinalStatus), "Missing icon for FinalStatus member" +# Explicit, no catch-all, for the same reason as the two maps above: a new status +# must be classified as "the agent phase ended this way" (True — a detached grade +# preserves it) or "grading decided this" (False — a detached grade replaces it). +# Defaulting either way silently is how an ERROR row becomes a SUCCESS. +_EXECUTION_FACT_STATUSES: dict[FinalStatus, bool] = { + FinalStatus.SUCCESS: False, + FinalStatus.FAILURE: False, + FinalStatus.NOT_GRADED: False, + FinalStatus.ERROR: True, + FinalStatus.BUILD_FAILED: True, + FinalStatus.TIMEOUT: True, + FinalStatus.MAX_TURNS_EXHAUSTED: True, + FinalStatus.TOKEN_BUDGET_EXCEEDED: True, + FinalStatus.COST_BUDGET_EXCEEDED: True, +} + +assert set(_EXECUTION_FACT_STATUSES) == set(FinalStatus), "Unclassified FinalStatus member" + + class ApiBackend(StrEnum): """API backend for LLM calls.""" diff --git a/src/coder_eval/models/experiment.py b/src/coder_eval/models/experiment.py index 4c28982b..c399d7bf 100644 --- a/src/coder_eval/models/experiment.py +++ b/src/coder_eval/models/experiment.py @@ -190,7 +190,11 @@ class VariantResult(BaseModel): # noqa: CE009 -- persisted result model; round- variant_id: str task_id: str - weighted_score: float + # None when nothing was graded (`coder-eval execute`), mirroring + # EvaluationResult.weighted_score. A plain float here would launder the + # ungraded None into 0.000, which renders as — and is picked as a best + # variant against — a real score of zero. + weighted_score: float | None = None final_status: FinalStatus duration_seconds: float total_tokens: int | None = None @@ -257,9 +261,14 @@ def _check_task_count_invariant(self) -> VariantAggregate: raise ValueError(f"Task count invariant violated: {total} != {self.tasks_run}") return self + @computed_field # type: ignore[prop-decorator] @property def tasks_graded(self) -> int: - """Tasks actually measured — ``pass_rate``'s denominator.""" + """Tasks actually measured — ``pass_rate``'s denominator. + + Serialized for the same reason as its RunSummary twin: a consumer that + cannot read the denominator re-derives the rate and drifts. + """ return self.tasks_run - self.tasks_not_graded @computed_field # type: ignore[prop-decorator] diff --git a/src/coder_eval/models/results.py b/src/coder_eval/models/results.py index 170b868b..770a1bb7 100644 --- a/src/coder_eval/models/results.py +++ b/src/coder_eval/models/results.py @@ -887,7 +887,11 @@ class SuiteRollup(BaseModel): rows_passed: int rows_failed: int rows_error: int - pass_rate: float = Field(ge=0.0, le=1.0, description="rows_passed / rows_total") + # The fourth bucket, matching RunSummary.tasks_not_graded and + # VariantAggregate.tasks_not_graded. Defaulted so a suite.json written before + # `execute` existed still parses. + rows_not_graded: int = 0 + pass_rate: float = Field(ge=0.0, le=1.0, description="rows_passed / rows_graded (ungraded rows excluded)") average_weighted_score: float | None = Field( default=None, description="Mean weighted_score across rows that produced one." ) @@ -911,6 +915,20 @@ class SuiteRollup(BaseModel): ), ) + @model_validator(mode="after") + def _check_row_count_invariant(self) -> SuiteRollup: + """The same guard RunSummary carries, which this model was missing. + + Without it a row that lands outside all four buckets — the shape a new + FinalStatus category takes before every counter is updated — silently + drops out of the rollup instead of failing. + """ + buckets = self.rows_passed + self.rows_failed + self.rows_error + self.rows_not_graded + if buckets != self.rows_total: + total = f"{self.rows_passed} + {self.rows_failed} + {self.rows_error} + {self.rows_not_graded}" + raise ValueError(f"Suite row count invariant violated: {total} != {self.rows_total}") + return self + class SkippedTask(BaseModel): """A task YAML that was excluded from the run before reaching the orchestrator. @@ -1030,10 +1048,13 @@ def eval_result_total_cost(result: EvaluationResult) -> float | None: class RunSummary(BaseModel): """Summary of an entire evaluation run across multiple tasks. - ``pass_rate`` is ``tasks_succeeded / tasks_run``: every dispatched task is in the - denominator, errors included as misses. The previous formula excluded errors, - which paid a bonus for erroring. ``error_share`` reports how much of the rate is - errors, so a bad infrastructure night shows instead of being absorbed. + ``pass_rate`` is ``tasks_succeeded / tasks_graded``: every task that was + MEASURED is in the denominator, errors included as misses. An earlier formula + excluded errors, which paid a bonus for erroring. ``error_share`` reports how + much of the rate is errors, so a bad infrastructure night shows instead of + being absorbed. Only ungraded tasks (``coder-eval execute``) leave the + denominator — they were never measured, so a 0/0 run has no rate at all + rather than a 0% one. This is the framework's single denominator: every reporting surface reads ``pass_rate`` rather than re-deriving one. Derived metrics here are computed, @@ -1114,9 +1135,17 @@ def _check_task_count_invariant(self) -> RunSummary: raise ValueError(f"Task count invariant violated: {total} != {self.tasks_run}") return self + @computed_field @property def tasks_graded(self) -> int: - """Tasks that were actually measured — the denominator for every rate below.""" + """Tasks that were actually measured — the denominator for every rate below. + + A ``computed_field`` rather than a plain property so it reaches run.json: + it is the denominator of `pass_rate` and `error_share`, and a consumer + that cannot read it has to re-derive the rate from the raw counts — which + is precisely how a consumer ends up publishing a different number for the + same run. REPORT_SCHEMA.md documents it as serialized. + """ return self.tasks_run - self.tasks_not_graded # Derived run metrics: computed_fields over the stored counts and diff --git a/src/coder_eval/orchestration/experiment.py b/src/coder_eval/orchestration/experiment.py index 3e8c1500..24d0effe 100644 --- a/src/coder_eval/orchestration/experiment.py +++ b/src/coder_eval/orchestration/experiment.py @@ -831,7 +831,7 @@ def _pick_worst_status(statuses: list[FinalStatus]) -> FinalStatus: def _mean_graded_score(vr_list: list[VariantResult]) -> float: """Mean ``weighted_score`` over the graded rows; ``0.0`` when none were graded.""" - graded = [v.weighted_score for v in vr_list if v.final_status.category != "ungraded"] + graded = [v.weighted_score for v in vr_list if v.weighted_score is not None] return sum(graded) / len(graded) if graded else 0.0 @@ -878,11 +878,15 @@ def aggregate_results( # Collect per-replicate scores keyed variant_id → task_id → [scores] for stats rendering. per_replicate_scores: dict[str, dict[str, list[float]]] = {} for (task_id, variant_id), reps in task_variant_reps.items(): - per_replicate_scores.setdefault(variant_id, {})[task_id] = [r.result.weighted_score or 0.0 for r in reps] + per_replicate_scores.setdefault(variant_id, {})[task_id] = [ + r.result.weighted_score for r in reps if r.result.weighted_score is not None + ] task_variants: dict[str, list[VariantResult]] = {} for (task_id, variant_id), reps in task_variant_reps.items(): - scores = [r.result.weighted_score or 0.0 for r in reps] + # Ungraded replicates drop out entirely rather than contributing 0.0 — + # `or 0.0` would average a clean `execute` run down to a real-looking zero. + scores = [r.result.weighted_score for r in reps if r.result.weighted_score is not None] non_errored = [r for r in reps if r.result.final_status.category != "error"] durations = [r.result.duration_seconds for r in non_errored] statuses = [r.result.final_status for r in reps] @@ -895,7 +899,7 @@ def aggregate_results( variant_result = VariantResult( variant_id=variant_id, task_id=task_id, - weighted_score=sum(scores) / len(scores), + weighted_score=sum(scores) / len(scores) if scores else None, final_status=final_status, duration_seconds=sum(durations), total_tokens=sum(token_vals) if token_vals else None, @@ -910,9 +914,12 @@ def aggregate_results( # Build task summaries task_summaries: list[TaskExperimentSummary] = [] for task_id, variants in task_variants.items(): - best = max(variants, key=lambda v: (v.weighted_score, v.variant_id)) - scores = [v.weighted_score for v in variants] - top_count = sum(1 for v in variants if v.weighted_score == best.weighted_score) + # Only graded variants can win or set a spread. Including ungraded ones + # at 0.0 would name an arbitrary "best" among scores that do not exist. + scored = [(v, v.weighted_score) for v in variants if v.weighted_score is not None] + best = max(scored, key=lambda pair: (pair[1], pair[0].variant_id))[0] if scored else variants[0] + scores = [s for _, s in scored] + top_count = sum(1 for _, s in scored if s == best.weighted_score) rep_counts = {v.replicate_count for v in variants} task_summaries.append( TaskExperimentSummary( @@ -920,7 +927,7 @@ def aggregate_results( variant_results=variants, best_variant=best.variant_id, is_tie=top_count > 1, - score_spread=max(scores) - min(scores), + score_spread=(max(scores) - min(scores)) if scores else 0.0, replicate_count=min(rep_counts) if rep_counts else 1, ) ) diff --git a/src/coder_eval/orchestration/regrade.py b/src/coder_eval/orchestration/regrade.py index aa6b15c7..8eba319d 100644 --- a/src/coder_eval/orchestration/regrade.py +++ b/src/coder_eval/orchestration/regrade.py @@ -20,7 +20,7 @@ import logging from pathlib import Path -from coder_eval.models import EvaluationResult, PreservationMode, TaskDefinition +from coder_eval.models import EvaluationResult, PreservationMode, TaskConfigRecord, TaskDefinition from coder_eval.sandbox import Sandbox @@ -61,8 +61,6 @@ def task_from_prior(prior: EvaluationResult, run_dir: Path) -> tuple[TaskDefinit schema change since the run), and says so loudly — a quiet fallback would reintroduce exactly the drift above. """ - from .task_loader import load_task - record = prior.task_config if record is None: raise RegradeError( @@ -70,21 +68,55 @@ def task_from_prior(prior: EvaluationResult, run_dir: Path) -> tuple[TaskDefinit + "rebuilt. Pass the task file explicitly: coder-eval evaluate " ) try: - return TaskDefinition.model_validate(record.resolved), record.source_yaml + task = TaskDefinition.model_validate(record.resolved) except ValueError as e: - if not record.source_file or not Path(record.source_file).is_file(): - raise RegradeError( - f"The resolved task config in {run_dir / TASK_JSON} no longer validates ({e}), and " - + "its source YAML is unavailable. Pass the task file explicitly." - ) from e - logger.warning( - "The recorded resolved config does not validate (%s); falling back to %s. Variant " - + "overrides, -D flags and dataset expansion from the original run are NOT reapplied, " - + "so this grade may not match what ran.", - e, - record.source_file, - ) - return load_task(Path(record.source_file)) + return _fall_back_to_source(record, run_dir, e) + warn_on_embedded_commands(task, run_dir) + return task, record.source_yaml + + +def warn_on_embedded_commands(task: TaskDefinition, run_dir: Path) -> None: + """Name the shell commands a rebuilt config will execute on this host. + + ``task_config.resolved`` is data that travels inside a run directory, and a + run directory is a shareable artifact — the detached-grading flow exists so + one machine can execute and another can grade. Rebuilding the task from it + means the *run dir* decides what ``run_command`` criteria the grader runs, + with the grader's environment. That is the intended behavior (it is how the + grade reproduces the executed config), but it must not be invisible: print + what will run so an unexpected command is noticed before it executes. + """ + commands = [cmd for c in task.success_criteria if isinstance(cmd := getattr(c, "command", None), str)] + commands += [c.command for c in task.pre_run] + [c.command for c in task.post_run] + if not commands: + return + logger.warning( + "Grading %s runs %d shell command(s) taken from that run's own recorded config: %s", + run_dir, + len(commands), + "; ".join(commands), + ) + + +def _fall_back_to_source(record: TaskConfigRecord, run_dir: Path, e: ValueError) -> tuple[TaskDefinition, str]: + """The loud source-YAML fallback for a resolved config that no longer validates.""" + from .task_loader import load_task + + if not record.source_file or not Path(record.source_file).is_file(): + raise RegradeError( + f"The resolved task config in {run_dir / TASK_JSON} no longer validates ({e}), and " + + "its source YAML is unavailable. Pass the task file explicitly." + ) from e + logger.warning( + "The recorded resolved config does not validate (%s); falling back to %s. Variant " + + "overrides, -D flags and dataset expansion from the original run are NOT reapplied, " + + "so this grade may not match what ran.", + e, + record.source_file, + ) + task, source_yaml = load_task(Path(record.source_file)) + warn_on_embedded_commands(task, run_dir) + return task, source_yaml def default_workspace(run_dir: Path, prior: EvaluationResult) -> Path: @@ -112,23 +144,44 @@ def default_workspace(run_dir: Path, prior: EvaluationResult) -> Path: return children[0] if len(children) == 1 else artifacts -def verify_reference_unchanged(prior: EvaluationResult, task: TaskDefinition) -> None: +def verify_reference_unchanged(prior: EvaluationResult, task: TaskDefinition, task_file: Path | None) -> None: """Refuse to grade when the reference tree changed since the run. ``reference_comparison`` and reference-carrying judges score against ``task.reference.directory``. If it moved since the run, the re-grade would silently measure the agent's old work against a new answer key. + + ``task_file`` is what ``reference.directory`` resolves against, so it is + required for any task that declares one — resolving without it raises, which + is why it is threaded through rather than passed as ``None``. """ + if task.reference is None: + return recorded = prior.environment_info.get("reference_digest") - if not isinstance(recorded, str) or task.reference is None: + if not isinstance(recorded, str): + # A run that predates the digest being persisted. Say so: silence here is + # what made this whole guard dead code for its first release. + logger.warning( + "This run recorded no reference_digest, so the answer key cannot be verified. " + + "Grading proceeds; a reference edited since the run would go undetected." + ) return from coder_eval.path_utils import digest_tree from .evaluation import resolve_reference_dir - resolved = resolve_reference_dir(task, None) + try: + resolved = resolve_reference_dir(task, task_file) + except (FileNotFoundError, ValueError) as e: + raise RegradeError( + f"This run's task declares a reference directory that cannot be resolved now ({e}), " + + "so its contents cannot be verified against the executed run." + ) from e if resolved is None or not resolved.is_dir(): - return + raise RegradeError( + f"The reference directory recorded for this run is gone ({resolved}). Grading now " + + "would score against a missing answer key. Restore it, or re-run the task." + ) if digest_tree(resolved) != recorded: raise RegradeError( f"The reference directory {resolved} changed since this run was executed " diff --git a/src/coder_eval/orchestrator.py b/src/coder_eval/orchestrator.py index e9c991ea..1e5c2540 100644 --- a/src/coder_eval/orchestrator.py +++ b/src/coder_eval/orchestrator.py @@ -2,6 +2,7 @@ import asyncio import logging +import os import re import tempfile import time @@ -608,7 +609,21 @@ def _kill_agent_subprocess_sync() -> None: # (like TIMEOUT / BUILD_FAILED / the budget stops on the except # branches below) is a fact about the RUN, not about grading, and # still applies. With grade=True the chain is unchanged. - if success: + # + # A detached grade goes further: the prior run's terminal status + # may itself be an execution fact (TIMEOUT, ERROR, a budget stop) + # that this pass neither repeated nor observed, so grading must + # not overwrite it. Without this, a crashed run re-graded against + # its half-finished workspace reports SUCCESS — with the original + # error_message still attached. + inherited = self.prior_result.final_status if self.prior_result is not None else None + if inherited is not None and inherited.is_execution_fact: + logger.info( + "Preserving the run's terminal status %s: grading cannot overturn an execution fact.", + inherited.value, + ) + self.result.final_status = inherited + elif success: self.result.final_status = FinalStatus.SUCCESS elif self.result.max_turns_exhausted: self.result.final_status = FinalStatus.MAX_TURNS_EXHAUSTED @@ -767,7 +782,22 @@ def _seed_from_prior_result(self) -> None: self.result.max_turns_exhausted = prior.max_turns_exhausted self.result.error_message = prior.error_message self.result.error_details = prior.error_details + self.result.error_log_tail = prior.error_log_tail self.result.sdk_options = prior.sdk_options + self.result.agent_config = prior.agent_config + self.result.expected_commands = prior.expected_commands + self.result.simulation = prior.simulation + + # The hooks belong to the execute phase and are NOT re-run against an + # adopted workspace (see _skip_hooks_for_adopted), so their recorded + # outcomes would otherwise vanish from the graded row. + self.result.pre_run_results = list(prior.pre_run_results) + self.result.post_run_results = list(prior.post_run_results) + + # The artifacts pointer. An adopted sandbox is not deleted by cleanup(), + # so the path stays valid — and a SECOND grade needs it, since without it + # the caller falls back to guessing the workspace. + self.result.sandbox_path = prior.sandbox_path # environment_info: the prior run's capture describes the machine that # RAN the task (installed_tools, api route, coder_eval version). Ours @@ -1325,6 +1355,12 @@ async def _stage_reference(self) -> None: destination = staging / "reference" self._reference_dir = await asyncio.to_thread(stage_reference_dir, source, destination) self._reference_digest = await asyncio.to_thread(digest_tree, self._reference_dir) + # Persist it: a DETACHED grade happens in a different process with no + # access to this instance, and refuses to score old work against a new + # answer key by comparing the tree it stages against this recorded hash + # (orchestration/regrade.py::verify_reference_unchanged). + if self.result is not None: + self.result.environment_info["reference_digest"] = self._reference_digest self._validate_reference_consumers() def _validate_reference_consumers(self) -> None: @@ -1422,7 +1458,7 @@ async def _setup(self) -> None: # against ambient PATH and can disagree with the original verdict. restored_path = self.result.environment_info.get("command_base_path") if isinstance(restored_path, str) and restored_path: - self.sandbox.set_command_base_path(restored_path) + self.sandbox.set_command_base_path(self._sanitize_restored_path(restored_path)) self._resolve_routes() self._record_route_environment_info() @@ -2009,6 +2045,94 @@ def _accumulate_judge_usage( # forward so it isn't dropped from the latest results list. r.token_usage = prior + def _sanitize_restored_path(self, recorded: str) -> str: + """Filter a PATH restored from a run's own ``task.json`` before prepending it. + + The restored value is PREPENDED ahead of the host PATH, and it arrives + from a file inside the directory being graded — a run dir is a shareable + artifact (that is the whole point of the detached-grading flow), and under + ``driver: docker`` it is bind-mounted writable into the container the agent + runs in. Prepending it verbatim lets a run dir decide which binary + ``pytest`` resolves to on the grader's host. + + Two filters, both cheap and both about what PATH parity actually needs: + drop anything that is not an existing directory (a dead entry buys no + parity), and drop any entry inside the workspace being graded (that tree is + agent-writable, so a shim dropped there would shadow a real tool). What + remains is the run's genuine toolchain locations. + """ + workspace = self.sandbox.sandbox_dir.resolve() if self.sandbox and self.sandbox.sandbox_dir else None + kept: list[str] = [] + for entry in recorded.split(os.pathsep): + if not entry: + continue + candidate = Path(entry) + if not candidate.is_dir(): + logger.debug("Dropping recorded PATH entry %s: not a directory here.", entry) + continue + resolved = candidate.resolve() + if workspace is not None and (resolved == workspace or workspace in resolved.parents): + logger.warning( + "Dropping recorded PATH entry %s: it lies inside the workspace being graded, " + + "so a binary there could shadow a real tool on the grader's host.", + entry, + ) + continue + kept.append(str(resolved)) + return os.pathsep.join(kept) + + def _select_gate(self) -> bool: + """Apply the verdict gate to the criteria results already on ``self.result``. + + Gate selection is FIRED-ONLY: the weighted armed gate applies iff the + watcher actually cut the run (``early_stop is not None``) — on a truncated + trajectory the unarmed criteria never had the chance to be satisfied, so + they stay advisory. A run that completed naturally (armed or not, watcher + never fired or disarmed fail-open) has a full trajectory and gates + strict-AND over every gating criterion, exactly like an unarmed run — + arming a criterion (e.g. adding a ``decide_within`` fail-fast timeout) + must never change the verdict of a run it didn't cut. + + BOTH grading paths must call this. A detached grade (``evaluate + `` / ``run --resume``) reaches the verdict through the + evaluate-only branch, where ``early_stop`` arrives via + ``_seed_from_prior_result`` rather than from a live watcher; selecting + the gate there in a second, hand-written place is exactly how the + seeded field came to be carried but never read — re-grading an + early-stopped run under the full-run strict-AND gate flips its verdict. + """ + assert self.result is not None + if self.result.early_stop is not None: + # One gate for every early-stopped run, no per-reason branches: a + # decision-budget stop is just a fail-stop whose deciding criterion + # timed out (the watcher only fires once the weighted ceiling + # proves the armed gate cannot pass). The ceiling is an upper bound + # on the authoritative armed score only because the watcher reduces + # the SAME trajectory the checker scores — it records UNRESOLVED + # tool ends exactly like the agent's EventCollector does (see + # EarlyStopWatcher._on_event_impl) — so the weighted armed gate is + # correct whether the watcher fired on a pass, a fail, or a timeout. + gate_threshold = ( + self.task.run_limits.stop_early_gate_threshold + if self.task.run_limits is not None + else DEFAULT_STOP_EARLY_GATE_THRESHOLD + ) + armed_count = sum(1 for c in self.task.success_criteria if c.is_stop_armed) + logger.info( + "Early-stopped run (%s): gating on %d armed criteria (%d advisory, not gated).", + self.result.early_stop.reason.value, + armed_count, + len(self.task.success_criteria) - armed_count, + ) + return self.result.armed_criteria_passed(self.task.success_criteria, gate_threshold) + + if self._early_stop_watcher is not None: + if self._early_stop_watcher.disarmed: + logger.info("early-stop watcher disarmed fail-open (verdict error): gating on the full set.") + else: + logger.info("early-stop armed but never fired (run completed naturally): gating on the full set.") + return self.result.all_criteria_passed(self.task.success_criteria) + async def _evaluation_loop(self) -> bool: """Run the main evaluation loop. @@ -2058,7 +2182,7 @@ async def _evaluation_loop(self) -> bool: turn_records=self.result.iterations, ) self.result.success_criteria_results = criteria_results - return self.result.all_criteria_passed(self.task.success_criteria) + return self._select_gate() # Working directory context prepended to every prompt (including feedback). # The agent resumes its session between iterations via session_id. @@ -2127,44 +2251,7 @@ async def _evaluation_loop(self) -> bool: pairs = list(zip(criteria_results, self.task.success_criteria, strict=True)) passed_count = sum(1 for r, c in pairs if r.score >= c.pass_threshold) total_count = len(pairs) - # Gate selection is FIRED-ONLY: the weighted armed gate applies iff the - # watcher actually cut the run (early_stop is not None) — on a truncated - # trajectory the unarmed criteria never had the chance to be satisfied, - # so they stay advisory. A run that completed naturally (armed or not, - # watcher never fired or disarmed fail-open) has a full trajectory and - # gates strict-AND over every gating criterion, exactly like an unarmed - # run — arming a criterion (e.g. adding a decide_within fail-fast - # timeout) must never change the verdict of a run it didn't cut. - if self.result.early_stop is not None: - # One gate for every early-stopped run, no per-reason branches: a - # decision-budget stop is just a fail-stop whose deciding criterion - # timed out (the watcher only fires once the weighted ceiling - # proves the armed gate cannot pass). The ceiling is an upper bound - # on the authoritative armed score only because the watcher reduces - # the SAME trajectory the checker scores — it records UNRESOLVED - # tool ends exactly like the agent's EventCollector does (see - # EarlyStopWatcher._on_event_impl) — so the weighted armed gate is - # correct whether the watcher fired on a pass, a fail, or a timeout. - gate_threshold = ( - self.task.run_limits.stop_early_gate_threshold - if self.task.run_limits is not None - else DEFAULT_STOP_EARLY_GATE_THRESHOLD - ) - all_passed = self.result.armed_criteria_passed(self.task.success_criteria, gate_threshold) - armed_count = sum(1 for c in self.task.success_criteria if c.is_stop_armed) - logger.info( - "Early-stopped run (%s): gating on %d armed criteria (%d advisory, not gated).", - self.result.early_stop.reason.value, - armed_count, - total_count - armed_count, - ) - else: - if self._early_stop_watcher is not None: - if self._early_stop_watcher.disarmed: - logger.info("early-stop watcher disarmed fail-open (verdict error): gating on the full set.") - else: - logger.info("early-stop armed but never fired (run completed naturally): gating on the full set.") - all_passed = self.result.all_criteria_passed(self.task.success_criteria) + all_passed = self._select_gate() # Reuse the model method for weighted score (single source of truth) self.result.calculate_weighted_score(self.task.success_criteria) @@ -2835,8 +2922,10 @@ async def _run_pre_run_commands(self) -> None: outer ``except Exception`` handler and lands the run as ``FinalStatus.ERROR``. Post-run commands and cleanup still execute via the ``finally`` block. + + Skipped entirely on an adopted sandbox — see ``_skip_hooks_for_adopted``. """ - if self.result is None: + if self.result is None or self._skip_hooks_for_adopted("pre_run"): return await self._run_command_list(self.task.pre_run, self.result.pre_run_results, "pre_run") @@ -2846,11 +2935,39 @@ async def _run_post_run_commands(self) -> None: See ``_run_command_list``. Post-run commands are informational only — ``fail_on_error`` is not part of ``PostRunCommand``, so failures are warning-logged and never affect the evaluation verdict. + + Skipped entirely on an adopted sandbox — see ``_skip_hooks_for_adopted``. """ - if self.result is None: + if self.result is None or self._skip_hooks_for_adopted("post_run"): return await self._run_command_list(self.task.post_run, self.result.post_run_results, "post_run") + def _skip_hooks_for_adopted(self, phase: str) -> bool: + """True when ``phase``'s commands must not run against an adopted sandbox. + + ``adopt()`` guarantees it materializes nothing into the workspace, but + that guarantee is only as strong as its weakest caller: ``run()`` invokes + the pre/post-run hooks unconditionally, and those commands run with + ``cwd = sandbox_dir``. Several in-tree tasks stage fixtures there + (``cp -a /app/[!.]* "$PWD/"``), so re-running them during a detached + grade would overwrite the agent's deliverables *before* the criteria read + them — silently changing the verdict and destroying preserved artifacts. + + The hooks belong to the EXECUTE phase; the prior run already ran them, + and their recorded results are carried over by ``_seed_from_prior_result``. + """ + if self.sandbox is None or not self.sandbox.was_adopted: + return False + commands = self.task.pre_run if phase == "pre_run" else self.task.post_run + if commands: + logger.info( + "Skipping %d %s command(s): the sandbox was adopted for grading, and re-running them " + + "would mutate the workspace under evaluation.", + len(commands), + phase, + ) + return True + async def _cleanup(self) -> None: """Clean up all resources.""" # Stop agent @@ -2914,8 +3031,15 @@ async def _cleanup(self) -> None: await asyncio.to_thread(self.sandbox.grant_read_access) logger.info(f"Sandbox preserved (in-place): {self.sandbox.sandbox_dir}") elif self.preservation_mode == PreservationMode.NONE and self.result: - # Sandbox will be deleted by cleanup() below; clear stale path. - self.result.sandbox_path = None + if self.sandbox.was_adopted: + # An adopted sandbox belongs to the caller and survives + # cleanup(), so the path is not stale — and clearing it + # would strip the graded row of its artifacts pointer + # (which a second grade needs to find the workspace). + self.result.sandbox_path = str(self.sandbox.sandbox_dir) + else: + # Sandbox will be deleted by cleanup() below; clear stale path. + self.result.sandbox_path = None elif self.result: # Defensive: a future PreservationMode member with no arm here # would otherwise silently fall through. Treat as no-preserve. diff --git a/src/coder_eval/reports.py b/src/coder_eval/reports.py index ea6d5897..c734f3d0 100644 --- a/src/coder_eval/reports.py +++ b/src/coder_eval/reports.py @@ -868,6 +868,10 @@ def _compute_suite_rollup( rows_passed = sum(1 for r in rows if r.result.final_status.category == "succeeded") rows_failed = sum(1 for r in rows if r.result.final_status.category == "failed") rows_error = sum(1 for r in rows if r.result.final_status.category == "error") + # An ungraded row was never measured, so it leaves BOTH sides of the rate — + # the same rule RunSummary.pass_rate and VariantAggregate.pass_rate follow. + rows_not_graded = sum(1 for r in rows if r.result.final_status.category == "ungraded") + rows_graded = rows_total - rows_not_graded scored = [r.result.weighted_score for r in rows if r.result.weighted_score is not None] average_weighted_score = sum(scored) / len(scored) if scored else None @@ -982,7 +986,8 @@ def _compute_suite_rollup( rows_passed=rows_passed, rows_failed=rows_failed, rows_error=rows_error, - pass_rate=rows_passed / rows_total if rows_total else 0.0, + rows_not_graded=rows_not_graded, + pass_rate=rows_passed / rows_graded if rows_graded else 0.0, average_weighted_score=average_weighted_score, criterion_stats=criterion_stats, failed_samples=failed_samples, @@ -998,8 +1003,9 @@ def _render_suite_markdown(rollup: SuiteRollup) -> str: "", f"**Variant**: `{rollup.variant_id}`", ( - f"**Rows**: {rollup.rows_total} total — " - f"{rollup.rows_passed} passed, {rollup.rows_failed} failed, {rollup.rows_error} errored" + f"**Rows**: {rollup.rows_total} total — {rollup.rows_passed} passed, " + + f"{rollup.rows_failed} failed, {rollup.rows_error} errored" + + (f", {rollup.rows_not_graded} not graded" if rollup.rows_not_graded else "") ), f"**Pass rate**: {rollup.pass_rate * 100:.1f}%", ] diff --git a/src/coder_eval/reports_experiment.py b/src/coder_eval/reports_experiment.py index e3cdd95f..ea1aceac 100644 --- a/src/coder_eval/reports_experiment.py +++ b/src/coder_eval/reports_experiment.py @@ -26,6 +26,7 @@ describe_prompt_config, fmt_mean_sd, fmt_p, + format_score, load_variant_eval_results, paired_comparison, stddev, @@ -248,7 +249,8 @@ def generate_task_report(summary: TaskExperimentSummary) -> str: tokens_str = f"{v.total_tokens:,}" if v.total_tokens is not None else "N/A" avg_dur = v.duration_seconds / v.replicate_count lines.append( - f"| {v.variant_id} | {v.weighted_score:.3f} | {v.final_status}" + f" | {avg_dur:.1f}s | {tokens_str} |" + f"| {v.variant_id} | {format_score(v.weighted_score)} | {v.final_status}" + + f" | {avg_dur:.1f}s | {tokens_str} |" ) return "\n".join(lines) @@ -483,7 +485,7 @@ def _win_loss_lines(result: ExperimentResult) -> list[str]: vr = scores_by_variant.get(vid) if vr: status_icon = vr.final_status.icon - cells.append(f"{vr.weighted_score:.3f} ({status_icon})") + cells.append(f"{format_score(vr.weighted_score)} ({status_icon})") else: cells.append("N/A") best_str = f"{'TIE' if ts.is_tie else ts.best_variant}" @@ -639,7 +641,7 @@ def generate_variant_report(variant_id: str, result: ExperimentResult, run_dir: variant_results = [ vr for ts in result.task_summaries for vr in ts.variant_results if vr.variant_id == variant_id ] - scores = [vr.weighted_score for vr in variant_results] + scores = [vr.weighted_score for vr in variant_results if vr.weighted_score is not None] durations = [vr.duration_seconds / vr.replicate_count for vr in variant_results] if scores and len(scores) >= 2: @@ -673,7 +675,8 @@ def generate_variant_report(variant_id: str, result: ExperimentResult, run_dir: for vr in ts.variant_results: if vr.variant_id == variant_id: avg_duration = vr.duration_seconds / vr.replicate_count - row = f"| {ts.task_id} | {vr.weighted_score:.3f} | {vr.final_status} | {avg_duration:.1f}s |" + score_text = format_score(vr.weighted_score) + row = f"| {ts.task_id} | {score_text} | {vr.final_status} | {avg_duration:.1f}s |" if has_reps: row += f" {vr.replicate_count} |" if has_similarity: diff --git a/src/coder_eval/reports_html.py b/src/coder_eval/reports_html.py index 7f5d4468..128c24ed 100644 --- a/src/coder_eval/reports_html.py +++ b/src/coder_eval/reports_html.py @@ -20,6 +20,7 @@ from coder_eval.models import FinalStatus, eval_result_total_cost, sum_costs from .reports import early_stop_gate_note +from .reports_stats import format_score if TYPE_CHECKING: @@ -1124,7 +1125,7 @@ def _variant_stddev_lines(variant_id: str, result: ExperimentResult | None) -> s from .reports_stats import stddev vrs = [vr for ts in result.task_summaries for vr in ts.variant_results if vr.variant_id == variant_id] - scores = [vr.weighted_score for vr in vrs] + scores = [vr.weighted_score for vr in vrs if vr.weighted_score is not None] durations = [vr.duration_seconds for vr in vrs] extras: list[str] = [] if len(scores) >= 2: @@ -1451,7 +1452,7 @@ def _experiment_per_task_comparison(result: ExperimentResult) -> str: if vr is None: cells.append("N/A") else: - cells.append(f"{vr.weighted_score:.3f} ({_esc(vr.final_status.icon)})") + cells.append(f"{format_score(vr.weighted_score)} ({_esc(vr.final_status.icon)})") best = "TIE" if ts.is_tie else ts.best_variant cells.append(f"{_esc(best)}") cells.append(f"{ts.score_spread:.3f}") @@ -1581,6 +1582,15 @@ def generate_variant_html( ) stddev_lines = _variant_stddev_lines(variant_id, result) rich_sections = _variant_rich_sections(variant_id, result, run_dir) + # Only rendered when non-zero, so an ordinary graded run's tile is + # unchanged — but a `coder-eval execute` run says where its tasks went + # instead of showing Succeeded/Failed/Errors all at zero. + ungraded_stat = ( + '
Not Graded
' + + f'
{agg.tasks_not_graded}
' + if agg.tasks_not_graded > 0 + else "" + ) budget_stats = "" if agg.tasks_token_budget_exceeded > 0: budget_stats += ( @@ -1609,6 +1619,7 @@ def generate_variant_html(
Succeeded
{agg.tasks_succeeded}
Failed
{agg.tasks_failed}
Errors
{agg.tasks_error}
+ {ungraded_stat} {budget_stats} {stddev_lines} @@ -1650,7 +1661,7 @@ def generate_experiment_html( {_esc(vid)} {_score_pill(agg.average_score)} - {agg.tasks_succeeded}/{agg.tasks_run} + {agg.tasks_succeeded}/{agg.tasks_graded} """ ) diff --git a/src/coder_eval/reports_stats.py b/src/coder_eval/reports_stats.py index 25b8c1c4..c123764b 100644 --- a/src/coder_eval/reports_stats.py +++ b/src/coder_eval/reports_stats.py @@ -310,6 +310,17 @@ class VariantSeries(NamedTuple): asst_turns: list[float] +# What an ungraded row shows where a score would go. Deliberately not "0.000": +# an ungraded task was never measured, and a zero is indistinguishable from a +# task that was measured and scored nothing. +UNGRADED_SCORE_TEXT = "n/a" + + +def format_score(score: float | None) -> str: + """Render a weighted score for a report table, or ``n/a`` when ungraded.""" + return UNGRADED_SCORE_TEXT if score is None else f"{score:.3f}" + + def collect_variant_series(result: ExperimentResult) -> dict[str, VariantSeries]: """Per-variant (scores, durations, tokens, assistant-turns) series, keyed by variant id. @@ -324,6 +335,14 @@ def collect_variant_series(result: ExperimentResult) -> dict[str, VariantSeries] s = series.get(vr.variant_id) if s is None: # a task result for a variant not in variant_ids continue + if vr.weighted_score is None: + # Ungraded row: no score exists, and appending 0.0 would enter a + # fabricated data point into every statistic below. Skip the row + # WHOLE rather than just its score — paired_comparison pairs the + # series across variants by index, so dropping one field would + # misalign them. `grade` is run-level, so an experiment is either + # entirely graded or entirely ungraded; this never splits a pair. + continue s.scores.append(vr.weighted_score) s.durations.append(vr.duration_seconds / vr.replicate_count) if vr.total_tokens is not None: diff --git a/src/coder_eval/sandbox.py b/src/coder_eval/sandbox.py index 3fbead1b..51734c69 100644 --- a/src/coder_eval/sandbox.py +++ b/src/coder_eval/sandbox.py @@ -163,6 +163,12 @@ def __init__( self.sandbox_dir: Path | None = None self.venv_dir: Path | None = None self._cleanup_on_exit = True + # True once adopt() takes over an existing workspace. Read by the + # Orchestrator to suppress every step that would MUTATE the tree it was + # asked to grade (pre_run/post_run above all — several in-tree tasks + # copy fixtures over the workspace there) and to keep sandbox_path in + # the result, since an adopted directory outlives cleanup(). + self.was_adopted = False self.installed_tool_versions: dict[str, str] = {} self._command_base_path: str | None = None # Cached canonical `node_modules/@uipath`; pins UiPath CLI plugin discovery @@ -318,6 +324,7 @@ def adopt(self, workspace: Path) -> Path: self.sandbox_dir = workspace.resolve() # Never flipped True: an adopted directory belongs to the caller. self._cleanup_on_exit = False + self.was_adopted = True # Only NON-materializing steps below. Deliberately skipped, and why: # _setup_template would overwrite the workspace being graded diff --git a/tests/test_cleanup_preservation_guard.py b/tests/test_cleanup_preservation_guard.py index e02c2f8e..b2519cd8 100644 --- a/tests/test_cleanup_preservation_guard.py +++ b/tests/test_cleanup_preservation_guard.py @@ -201,6 +201,10 @@ async def test_none_without_workspace_dir_discards_path(tmp_path) -> None: orchestrator.workspace_dir = None orchestrator.result.sandbox_path = "/stale/path" # must be cleared mock_sandbox = MagicMock() + # A real Sandbox that was not adopted. Explicit because a bare MagicMock + # attribute is truthy, which would take the adopted arm (that one KEEPS the + # path, since an adopted directory survives cleanup) and hide the discard. + mock_sandbox.was_adopted = False orchestrator.sandbox = mock_sandbox await orchestrator._cleanup() diff --git a/tests/test_cli_telemetry.py b/tests/test_cli_telemetry.py index 1c757524..06f49cad 100644 --- a/tests/test_cli_telemetry.py +++ b/tests/test_cli_telemetry.py @@ -79,7 +79,7 @@ def test_help_never_crashes_when_telemetry_enabled_and_config_unwritable(tmp_pat async def test_run_emits_run_start_and_flushes(tmp_path): - summary = Mock(tasks_failed=0, tasks_error=0) + summary = Mock(tasks_failed=0, tasks_error=0, tasks_not_graded=0) with ( patch("coder_eval.cli.run_command.prepare_run_directory", return_value=tmp_path), @@ -115,7 +115,7 @@ async def test_run_emits_run_start_and_flushes(tmp_path): async def test_run_start_uses_default_fallbacks_for_none_inputs(tmp_path): # agent_type=None / stream_mode=None must surface as the "default"/"none" # fallback property values, not as null. - summary = Mock(tasks_failed=0, tasks_error=0) + summary = Mock(tasks_failed=0, tasks_error=0, tasks_not_graded=0) with ( patch("coder_eval.cli.run_command.prepare_run_directory", return_value=tmp_path), diff --git a/tests/test_execute_evaluate_loop.py b/tests/test_execute_evaluate_loop.py index 9ab17528..96c0b415 100644 --- a/tests/test_execute_evaluate_loop.py +++ b/tests/test_execute_evaluate_loop.py @@ -11,6 +11,7 @@ from __future__ import annotations import json +import shutil from pathlib import Path from typing import Any @@ -204,6 +205,54 @@ def test_execute_resume_treats_an_executed_row_as_done(tmp_path: Path) -> None: assert _row(_task_dir(run_dir))["final_status"] == FinalStatus.NOT_GRADED.value +def test_a_detached_grade_does_not_re_run_pre_run_against_the_workspace(tmp_path: Path) -> None: + """`run()` calls the pre/post-run hooks unconditionally, with cwd = the + sandbox. On an ADOPTED sandbox that sandbox is the agent's own output, and + several in-tree tasks stage fixtures there (`cp -a /app/[!.]* "$PWD/"`), so + re-running them would overwrite the deliverables before the criteria read + them — changing the verdict and destroying preserved artifacts.""" + run_dir = tmp_path / "r" + _invoke(["execute", str(AGENTLESS_TASK), "--run-dir", str(run_dir)]) + task_dir = _task_dir(run_dir) + proof = sorted(run_dir.glob("**/artifacts/**/proof.txt"))[0] + # Mark the agent's file. The fixture's pre_run rewrites proof.txt from + # scratch, so a re-run would wipe this marker. + proof.write_text("coder-eval-ran-without-a-coder AND-THE-AGENT-EDITED-THIS", encoding="utf-8") + + _invoke(["evaluate", str(task_dir)]) + + assert "AND-THE-AGENT-EDITED-THIS" in proof.read_text(encoding="utf-8"), ( + "pre_run re-ran against the adopted workspace and overwrote the agent's work" + ) + # The hooks' recorded outcomes are carried over rather than lost. + assert _row(task_dir)["pre_run_results"], "the execute phase's pre_run results were dropped" + + +def test_run_resume_exits_non_zero_when_it_cannot_grade(tmp_path: Path) -> None: + """`run` was asked for a verdict. If grading fails, reporting exit 0 tells CI + the suite is fine when nothing was actually scored.""" + run_dir = tmp_path / "r" + _invoke(["execute", str(AGENTLESS_TASK), "--run-dir", str(run_dir)]) + # Remove the workspace so the re-grade has nothing to grade against. + shutil.rmtree(run_dir / "default" / "agentless_smoke_test" / "00" / "artifacts", ignore_errors=True) + row = _task_dir(run_dir) / "task.json" + record = json.loads(row.read_text(encoding="utf-8")) + record["sandbox_path"] = str(tmp_path / "gone") + row.write_text(json.dumps(record), encoding="utf-8") + + result = runner.invoke(app, ["run", str(AGENTLESS_TASK), "--run-dir", str(run_dir), "--resume"]) + + assert result.exit_code != 0, "a run that graded nothing must not report success" + + +def test_execute_still_exits_zero_with_every_row_ungraded(tmp_path: Path) -> None: + """The other side of the rule above: under `execute` an ungraded row is the + expected outcome, not a failure of the command.""" + run_dir = tmp_path / "r" + result = runner.invoke(app, ["execute", str(AGENTLESS_TASK), "--run-dir", str(run_dir)]) + assert result.exit_code == 0, result.output + + def test_execute_to_run_resume_emits_no_config_drift_warning(tmp_path: Path) -> None: """`grade` is exempt from the fingerprint diff: this flow is supported, and the warning's "keeps their original-config results" text is wrong for it.""" diff --git a/tests/test_post_run.py b/tests/test_post_run.py index a55f0e14..a5b5687f 100644 --- a/tests/test_post_run.py +++ b/tests/test_post_run.py @@ -119,7 +119,7 @@ def _make_orchestrator(task: TaskDefinition, tmp_path: Path) -> Orchestrator: async def test_post_run_skipped_when_empty(tmp_path): task = _make_task(post_run=[]) orch = _make_orchestrator(task, tmp_path) - orch.sandbox = AsyncMock() + orch.sandbox = AsyncMock(was_adopted=False) orch.sandbox.sandbox_dir = tmp_path await orch._run_post_run_commands() @@ -142,7 +142,7 @@ async def test_post_run_skipped_when_no_sandbox(tmp_path): async def test_post_run_command_success(tmp_path): task = _make_task(post_run=[PostRunCommand(command="echo '{\"ok\": true}'")]) orch = _make_orchestrator(task, tmp_path) - orch.sandbox = AsyncMock() + orch.sandbox = AsyncMock(was_adopted=False) orch.sandbox.sandbox_dir = tmp_path await orch._run_post_run_commands() @@ -161,7 +161,7 @@ async def test_post_run_command_failure_does_not_affect_result(tmp_path): ) orch = _make_orchestrator(task, tmp_path) orch.result.final_status = FinalStatus.SUCCESS - orch.sandbox = AsyncMock() + orch.sandbox = AsyncMock(was_adopted=False) orch.sandbox.sandbox_dir = tmp_path await orch._run_post_run_commands() @@ -178,7 +178,7 @@ async def test_post_run_command_with_pipes(tmp_path): """Shell commands support pipes and redirects.""" task = _make_task(post_run=[PostRunCommand(command="echo hello world | tr a-z A-Z")]) orch = _make_orchestrator(task, tmp_path) - orch.sandbox = AsyncMock() + orch.sandbox = AsyncMock(was_adopted=False) orch.sandbox.sandbox_dir = tmp_path await orch._run_post_run_commands() @@ -192,7 +192,7 @@ async def test_post_run_command_with_pipes(tmp_path): async def test_post_run_command_timeout(tmp_path): task = _make_task(post_run=[PostRunCommand(command="sleep 10", timeout=1)]) orch = _make_orchestrator(task, tmp_path) - orch.sandbox = AsyncMock() + orch.sandbox = AsyncMock(was_adopted=False) orch.sandbox.sandbox_dir = tmp_path await orch._run_post_run_commands() @@ -211,7 +211,7 @@ async def test_post_run_multiple_commands(tmp_path): ] ) orch = _make_orchestrator(task, tmp_path) - orch.sandbox = AsyncMock() + orch.sandbox = AsyncMock(was_adopted=False) orch.sandbox.sandbox_dir = tmp_path await orch._run_post_run_commands() @@ -229,7 +229,7 @@ async def test_post_run_cwd_is_sandbox(tmp_path): task = _make_task(post_run=[PostRunCommand(command='python3 -c "import os; print(os.getcwd())"')]) orch = _make_orchestrator(task, tmp_path) - orch.sandbox = AsyncMock() + orch.sandbox = AsyncMock(was_adopted=False) orch.sandbox.sandbox_dir = sandbox_dir await orch._run_post_run_commands() @@ -243,7 +243,7 @@ async def test_post_run_streams_stdout_to_logger(tmp_path, caplog): """Each line of stdout is forwarded to the orchestrator logger as it is read.""" task = _make_task(post_run=[PostRunCommand(command="python3 -c \"print('line-one'); print('line-two')\"")]) orch = _make_orchestrator(task, tmp_path) - orch.sandbox = AsyncMock() + orch.sandbox = AsyncMock(was_adopted=False) orch.sandbox.sandbox_dir = tmp_path with caplog.at_level(logging.INFO, logger="coder_eval.orchestrator"): @@ -263,7 +263,7 @@ async def test_post_run_streams_stderr_as_warning(tmp_path, caplog): """Stderr lines are forwarded at WARNING level (separate from stdout).""" task = _make_task(post_run=[PostRunCommand(command="python3 -c \"import sys; print('boom', file=sys.stderr)\"")]) orch = _make_orchestrator(task, tmp_path) - orch.sandbox = AsyncMock() + orch.sandbox = AsyncMock(was_adopted=False) orch.sandbox.sandbox_dir = tmp_path with caplog.at_level(logging.WARNING, logger="coder_eval.orchestrator"): @@ -279,7 +279,7 @@ async def test_post_run_output_truncated(tmp_path): # Generate output larger than the limit task = _make_task(post_run=[PostRunCommand(command="python3 -c \"print('x' * 200_000)\"")]) orch = _make_orchestrator(task, tmp_path) - orch.sandbox = AsyncMock() + orch.sandbox = AsyncMock(was_adopted=False) orch.sandbox.sandbox_dir = tmp_path await orch._run_post_run_commands() diff --git a/tests/test_pre_run.py b/tests/test_pre_run.py index 6febaaa7..947039ed 100644 --- a/tests/test_pre_run.py +++ b/tests/test_pre_run.py @@ -150,7 +150,7 @@ def _make_orchestrator(task: TaskDefinition, tmp_path: Path) -> Orchestrator: async def test_pre_run_skipped_when_empty(tmp_path): task = _make_task(pre_run=[]) orch = _make_orchestrator(task, tmp_path) - orch.sandbox = AsyncMock() + orch.sandbox = AsyncMock(was_adopted=False) orch.sandbox.sandbox_dir = tmp_path await orch._run_pre_run_commands() @@ -173,7 +173,7 @@ async def test_pre_run_skipped_when_no_sandbox(tmp_path): async def test_pre_run_command_success(tmp_path): task = _make_task(pre_run=[PreRunCommand(command="echo '{\"ok\": true}'")]) orch = _make_orchestrator(task, tmp_path) - orch.sandbox = AsyncMock() + orch.sandbox = AsyncMock(was_adopted=False) orch.sandbox.sandbox_dir = tmp_path await orch._run_pre_run_commands() @@ -189,7 +189,7 @@ async def test_pre_run_command_success(tmp_path): async def test_pre_run_command_with_pipes(tmp_path): task = _make_task(pre_run=[PreRunCommand(command="echo hello world | tr a-z A-Z")]) orch = _make_orchestrator(task, tmp_path) - orch.sandbox = AsyncMock() + orch.sandbox = AsyncMock(was_adopted=False) orch.sandbox.sandbox_dir = tmp_path await orch._run_pre_run_commands() @@ -208,7 +208,7 @@ async def test_pre_run_multiple_commands(tmp_path): ] ) orch = _make_orchestrator(task, tmp_path) - orch.sandbox = AsyncMock() + orch.sandbox = AsyncMock(was_adopted=False) orch.sandbox.sandbox_dir = tmp_path await orch._run_pre_run_commands() @@ -225,7 +225,7 @@ async def test_pre_run_cwd_is_sandbox(tmp_path): task = _make_task(pre_run=[PreRunCommand(command='python3 -c "import os; print(os.getcwd())"')]) orch = _make_orchestrator(task, tmp_path) - orch.sandbox = AsyncMock() + orch.sandbox = AsyncMock(was_adopted=False) orch.sandbox.sandbox_dir = sandbox_dir await orch._run_pre_run_commands() @@ -238,7 +238,7 @@ async def test_pre_run_cwd_is_sandbox(tmp_path): async def test_pre_run_streams_stdout_to_logger(tmp_path, caplog): task = _make_task(pre_run=[PreRunCommand(command="python3 -c \"print('line-one'); print('line-two')\"")]) orch = _make_orchestrator(task, tmp_path) - orch.sandbox = AsyncMock() + orch.sandbox = AsyncMock(was_adopted=False) orch.sandbox.sandbox_dir = tmp_path with caplog.at_level(logging.INFO, logger="coder_eval.orchestrator"): @@ -263,7 +263,7 @@ async def test_pre_run_streams_stderr_as_warning(tmp_path, caplog): ] ) orch = _make_orchestrator(task, tmp_path) - orch.sandbox = AsyncMock() + orch.sandbox = AsyncMock(was_adopted=False) orch.sandbox.sandbox_dir = tmp_path with caplog.at_level(logging.WARNING, logger="coder_eval.orchestrator"): @@ -277,7 +277,7 @@ async def test_pre_run_streams_stderr_as_warning(tmp_path, caplog): async def test_pre_run_output_truncated(tmp_path): task = _make_task(pre_run=[PreRunCommand(command="python3 -c \"print('x' * 200_000)\"")]) orch = _make_orchestrator(task, tmp_path) - orch.sandbox = AsyncMock() + orch.sandbox = AsyncMock(was_adopted=False) orch.sandbox.sandbox_dir = tmp_path await orch._run_pre_run_commands() @@ -296,7 +296,7 @@ async def test_pre_run_failure_raises_when_fail_on_error_true(tmp_path): pre_run=[PreRunCommand(command='python3 -c "import sys; sys.exit(1)"')], ) orch = _make_orchestrator(task, tmp_path) - orch.sandbox = AsyncMock() + orch.sandbox = AsyncMock(was_adopted=False) orch.sandbox.sandbox_dir = tmp_path with pytest.raises(RuntimeError, match="Pre-run command failed"): @@ -307,7 +307,7 @@ async def test_pre_run_failure_raises_when_fail_on_error_true(tmp_path): async def test_pre_run_timeout_raises_when_fail_on_error_true(tmp_path): task = _make_task(pre_run=[PreRunCommand(command="sleep 10", timeout=1)]) orch = _make_orchestrator(task, tmp_path) - orch.sandbox = AsyncMock() + orch.sandbox = AsyncMock(was_adopted=False) orch.sandbox.sandbox_dir = tmp_path with pytest.raises(RuntimeError, match="timed out after 1s"): @@ -320,7 +320,7 @@ async def test_pre_run_failure_result_captured_before_raise(tmp_path): pre_run=[PreRunCommand(command='python3 -c "import sys; sys.exit(2)"')], ) orch = _make_orchestrator(task, tmp_path) - orch.sandbox = AsyncMock() + orch.sandbox = AsyncMock(was_adopted=False) orch.sandbox.sandbox_dir = tmp_path with pytest.raises(RuntimeError): @@ -339,7 +339,7 @@ async def test_pre_run_subsequent_commands_skipped_after_abort(tmp_path): ], ) orch = _make_orchestrator(task, tmp_path) - orch.sandbox = AsyncMock() + orch.sandbox = AsyncMock(was_adopted=False) orch.sandbox.sandbox_dir = tmp_path with pytest.raises(RuntimeError): @@ -362,7 +362,7 @@ async def test_pre_run_failure_does_not_raise_when_fail_on_error_false(tmp_path) ], ) orch = _make_orchestrator(task, tmp_path) - orch.sandbox = AsyncMock() + orch.sandbox = AsyncMock(was_adopted=False) orch.sandbox.sandbox_dir = tmp_path await orch._run_pre_run_commands() @@ -376,7 +376,7 @@ async def test_pre_run_spawn_exception_raises_when_fail_on_error_true(tmp_path): """Generic exceptions from create_subprocess_shell propagate as RuntimeError when fail_on_error=True.""" task = _make_task(pre_run=[PreRunCommand(command="echo hi")]) orch = _make_orchestrator(task, tmp_path) - orch.sandbox = AsyncMock() + orch.sandbox = AsyncMock(was_adopted=False) orch.sandbox.sandbox_dir = tmp_path with ( @@ -399,7 +399,7 @@ async def test_pre_run_spawn_exception_does_not_raise_when_fail_on_error_false(t ] ) orch = _make_orchestrator(task, tmp_path) - orch.sandbox = AsyncMock() + orch.sandbox = AsyncMock(was_adopted=False) orch.sandbox.sandbox_dir = tmp_path real_shell = __import__("asyncio").create_subprocess_shell @@ -428,7 +428,7 @@ async def test_pre_run_timeout_does_not_raise_when_fail_on_error_false(tmp_path) ], ) orch = _make_orchestrator(task, tmp_path) - orch.sandbox = AsyncMock() + orch.sandbox = AsyncMock(was_adopted=False) orch.sandbox.sandbox_dir = tmp_path await orch._run_pre_run_commands() diff --git a/tests/test_regrade.py b/tests/test_regrade.py new file mode 100644 index 00000000..668046c8 --- /dev/null +++ b/tests/test_regrade.py @@ -0,0 +1,259 @@ +"""``orchestration/regrade.py`` — the refusals, not the happy path. + +The end-to-end loop test covers a successful re-grade of the agentless task. What +it cannot cover is every branch that REFUSES to grade, and those are the ones that +matter: each exists because grading anyway would publish a plausible number that +is wrong. The reference-digest guard in particular shipped as dead code (nothing +wrote the key it read) precisely because the only test that reached it used a +fixture with no reference at all. +""" + +from __future__ import annotations + +import json +import logging +from pathlib import Path + +import pytest + +from coder_eval.models import ( + AgentKind, + EvaluationResult, + FileExistsCriterion, + FinalStatus, + RunCommandCriterion, + TaskConfigRecord, + TaskDefinition, + parse_agent_config, +) +from coder_eval.orchestration.regrade import ( + PRE_GRADE_JSON, + TASK_JSON, + RegradeError, + back_up_pre_grade_record, + default_workspace, + load_prior_result, + task_from_prior, + verify_reference_unchanged, +) +from coder_eval.path_utils import digest_tree + + +def _task(*, reference: dict[str, str] | None = None, command: str | None = None) -> TaskDefinition: + criteria: list[object] = [FileExistsCriterion(path="x.txt", description="x")] + if command is not None: + criteria.append(RunCommandCriterion(command=command, description="run it")) + return TaskDefinition( + task_id="t", + description="d", + initial_prompt="p", + agent=parse_agent_config(type=AgentKind.CLAUDE_CODE), + reference=reference, # type: ignore[arg-type] + success_criteria=criteria, # type: ignore[arg-type] + ) + + +def _result(**kwargs: object) -> EvaluationResult: + from datetime import datetime + + base: dict[str, object] = { + "task_id": "t", + "task_description": "d", + "variant_id": "v", + "agent_type": AgentKind.CLAUDE_CODE, + "started_at": datetime(2020, 1, 1), + "final_status": FinalStatus.NOT_GRADED, + "iteration_count": 1, + } + base.update(kwargs) + return EvaluationResult(**base) # type: ignore[arg-type] + + +# -------------------------------------------------------------------------- +# load_prior_result +# -------------------------------------------------------------------------- + + +def test_missing_task_json_is_a_regrade_error(tmp_path: Path) -> None: + with pytest.raises(RegradeError, match="Cannot read"): + load_prior_result(tmp_path) + + +def test_unparseable_task_json_is_a_regrade_error(tmp_path: Path) -> None: + (tmp_path / TASK_JSON).write_text("{not json", encoding="utf-8") + with pytest.raises(RegradeError, match="not a readable EvaluationResult"): + load_prior_result(tmp_path) + + +# -------------------------------------------------------------------------- +# task_from_prior — which task gets graded +# -------------------------------------------------------------------------- + + +def test_no_task_config_refuses_rather_than_guessing(tmp_path: Path) -> None: + with pytest.raises(RegradeError, match="carries no task_config"): + task_from_prior(_result(), tmp_path) + + +def test_resolved_config_wins_over_the_source_yaml(tmp_path: Path) -> None: + """`resolved` is post-merge, so it carries variant overrides / -D / dataset + expansion. Re-reading the YAML would grade a DIFFERENT task.""" + source = tmp_path / "t.yaml" + source.write_text("task_id: from-yaml\n", encoding="utf-8") + resolved = _task().model_dump(mode="json") + resolved["task_id"] = "from-resolved" + prior = _result(task_config=TaskConfigRecord(resolved=resolved, source_yaml="raw", source_file=str(source))) + + task, _ = task_from_prior(prior, tmp_path) + + assert task.task_id == "from-resolved" + + +def test_unusable_resolved_config_with_no_source_refuses(tmp_path: Path) -> None: + prior = _result(task_config=TaskConfigRecord(resolved={"nonsense": True}, source_yaml="raw", source_file=None)) + with pytest.raises(RegradeError, match="no longer validates"): + task_from_prior(prior, tmp_path) + + +def test_source_fallback_is_loud(tmp_path: Path, caplog: pytest.LogCaptureFixture) -> None: + """A quiet fallback would silently grade a task other than the one that ran.""" + source = tmp_path / "t.yaml" + source.write_text( + "task_id: from-yaml\ndescription: d\ninitial_prompt: p\n" + + "success_criteria:\n - type: file_exists\n path: x.txt\n description: x\n", + encoding="utf-8", + ) + prior = _result( + task_config=TaskConfigRecord(resolved={"nonsense": True}, source_yaml="raw", source_file=str(source)) + ) + + with caplog.at_level(logging.WARNING): + task, _ = task_from_prior(prior, tmp_path) + + assert task.task_id == "from-yaml" + assert "NOT reapplied" in caplog.text + + +def test_shell_commands_from_a_run_dir_config_are_announced(tmp_path: Path, caplog: pytest.LogCaptureFixture) -> None: + """A run dir is a shareable artifact, and rebuilding from it decides what the + grader executes. Intended, but never silent.""" + resolved = _task(command="echo surprising").model_dump(mode="json") + prior = _result(task_config=TaskConfigRecord(resolved=resolved, source_yaml="raw", source_file=None)) + + with caplog.at_level(logging.WARNING): + task_from_prior(prior, tmp_path) + + assert "echo surprising" in caplog.text + + +# -------------------------------------------------------------------------- +# default_workspace +# -------------------------------------------------------------------------- + + +def test_recorded_sandbox_path_wins_when_it_still_exists(tmp_path: Path) -> None: + workspace = tmp_path / "ws" + workspace.mkdir() + assert default_workspace(tmp_path, _result(sandbox_path=str(workspace))) == workspace + + +def test_falls_back_to_the_single_artifacts_child(tmp_path: Path) -> None: + child = tmp_path / "artifacts" / "t" + child.mkdir(parents=True) + assert default_workspace(tmp_path, _result(sandbox_path="/gone")) == child + + +def test_flat_artifacts_dir_is_itself_the_workspace(tmp_path: Path) -> None: + artifacts = tmp_path / "artifacts" + artifacts.mkdir() + (artifacts / "file.txt").write_text("x", encoding="utf-8") + assert default_workspace(tmp_path, _result()) == artifacts + + +def test_no_workspace_at_all_refuses(tmp_path: Path) -> None: + with pytest.raises(RegradeError, match="No workspace to grade"): + default_workspace(tmp_path, _result()) + + +# -------------------------------------------------------------------------- +# verify_reference_unchanged — the anti-cheat guard +# -------------------------------------------------------------------------- + + +def _reference_task(tmp_path: Path) -> tuple[TaskDefinition, Path, Path]: + task_file = tmp_path / "t.yaml" + task_file.write_text("x", encoding="utf-8") + reference = tmp_path / "ref" + reference.mkdir() + (reference / "answer.py").write_text("print('right')\n", encoding="utf-8") + return _task(reference={"directory": "ref"}), task_file, reference + + +def test_an_edited_reference_refuses_the_grade(tmp_path: Path) -> None: + """The headline guarantee. Without it, an answer key edited between execute + and grade scores the agent's old work against a new one.""" + task, task_file, reference = _reference_task(tmp_path) + prior = _result(environment_info={"reference_digest": digest_tree(reference)}) + verify_reference_unchanged(prior, task, task_file) # unchanged: fine + + (reference / "answer.py").write_text("print('different')\n", encoding="utf-8") + + with pytest.raises(RegradeError, match="digest mismatch"): + verify_reference_unchanged(prior, task, task_file) + + +def test_a_vanished_reference_refuses_rather_than_grading_without_one(tmp_path: Path) -> None: + task, task_file, reference = _reference_task(tmp_path) + prior = _result(environment_info={"reference_digest": digest_tree(reference)}) + for p in reference.iterdir(): + p.unlink() + reference.rmdir() + + with pytest.raises(RegradeError): + verify_reference_unchanged(prior, task, task_file) + + +def test_a_run_without_a_recorded_digest_says_so(tmp_path: Path, caplog: pytest.LogCaptureFixture) -> None: + """Silence here is what let the guard ship as dead code for a release.""" + task, task_file, _ = _reference_task(tmp_path) + + with caplog.at_level(logging.WARNING): + verify_reference_unchanged(_result(), task, task_file) + + assert "cannot be verified" in caplog.text + + +def test_a_task_with_no_reference_is_not_checked(tmp_path: Path) -> None: + verify_reference_unchanged(_result(), _task(), tmp_path / "t.yaml") + + +# -------------------------------------------------------------------------- +# back_up_pre_grade_record +# -------------------------------------------------------------------------- + + +def test_the_pre_grade_record_is_written_once(tmp_path: Path) -> None: + """A second grade must not overwrite the ORIGINAL execute record with an + already-graded one — that is the only evidence the run was ungraded.""" + (tmp_path / TASK_JSON).write_text('{"round": 1}', encoding="utf-8") + back_up_pre_grade_record(tmp_path) + (tmp_path / TASK_JSON).write_text('{"round": 2}', encoding="utf-8") + back_up_pre_grade_record(tmp_path) + + assert json.loads((tmp_path / PRE_GRADE_JSON).read_text(encoding="utf-8")) == {"round": 1} + + +def test_backup_is_a_no_op_with_nothing_to_back_up(tmp_path: Path) -> None: + back_up_pre_grade_record(tmp_path) + assert not (tmp_path / PRE_GRADE_JSON).exists() + + +def test_a_failed_backup_never_fails_the_grade(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + """The audit copy is a convenience; the verdict is the deliverable.""" + (tmp_path / TASK_JSON).write_text("{}", encoding="utf-8") + + def _boom(*_args: object, **_kwargs: object) -> None: + raise OSError("read-only file system") + + monkeypatch.setattr(Path, "write_text", _boom) + back_up_pre_grade_record(tmp_path) # must not raise diff --git a/tests/test_run_command_junit.py b/tests/test_run_command_junit.py index b5ac1de0..7f8e5341 100644 --- a/tests/test_run_command_junit.py +++ b/tests/test_run_command_junit.py @@ -26,7 +26,7 @@ async def _invoke( status: str, failed: bool, ) -> None: - summary = Mock(tasks_failed=1 if failed else 0, tasks_error=0) + summary = Mock(tasks_failed=1 if failed else 0, tasks_error=0, tasks_not_graded=0) async def _fake(*_args, **_kwargs): # Mirror production: run.json is persisted inside _run_with_experiment. diff --git a/tests/test_run_metrics.py b/tests/test_run_metrics.py index 6ac5baff..6866a87f 100644 --- a/tests/test_run_metrics.py +++ b/tests/test_run_metrics.py @@ -5,7 +5,9 @@ **The denominator.** ``pass_rate`` used to be ``succeeded / (run - error)``, which paid a bonus for erroring: the more a run fell over, the smaller its denominator got, up to the degenerate case of a run rendering as a perfect score while passing -a handful of rows. Every surface now divides by ``tasks_run``. +a handful of rows. Every surface now divides by ``tasks_graded`` — every +dispatched task except the ones that were never measured at all (``coder-eval +execute`` leaves rows ``NOT_GRADED``, and those leave BOTH sides of the rate). **The bill.** Cost was summed over whatever rows happened to carry one, so a run whose model was missing from the rate card, or whose turns were killed before the diff --git a/tests/test_seed_from_prior_result.py b/tests/test_seed_from_prior_result.py new file mode 100644 index 00000000..319815d3 --- /dev/null +++ b/tests/test_seed_from_prior_result.py @@ -0,0 +1,272 @@ +"""``Orchestrator._seed_from_prior_result`` — the detached grade's fidelity contract. + +A detached grade (``evaluate `` / ``run --resume``) recomputes the +verdict but must not recompute the RUN. Every field it carries over is a fact the +agent phase established and this pass cannot re-derive; every field it does not +carry is either recomputed from the trajectory or deliberately dropped. + +The end-to-end tests in ``test_execute_evaluate_loop.py`` exercise this through a +fixture where most of these fields hold their defaults, so deleting a carry line +leaves them green. These tests set every field to a distinctive value instead, and +the partition below fails closed when a new field is added to ``EvaluationResult`` +without a decision about it. +""" + +from __future__ import annotations + +from datetime import datetime +from pathlib import Path + +import pytest + +from coder_eval.models import ( + AgentKind, + CommandExecutedCriterion, + CriterionResult, + EarlyStopInfo, + EarlyStopReason, + EvaluationResult, + FileExistsCriterion, + FinalStatus, + PostRunResult, + TaskDefinition, + parse_agent_config, +) +from coder_eval.orchestrator import Orchestrator + + +# Every field on EvaluationResult, partitioned by what a detached grade does with +# it. No catch-all: a new field fails the parity test below until it is listed, +# which is the same fail-closed shape as _STATUS_CATEGORIES in models/enums.py. +CARRIED = { + "started_at", + "iterations", + "iteration_count", + "early_stop", + "max_turns_exhausted", + "error_message", + "error_details", + "error_log_tail", + "sdk_options", + "agent_config", + "expected_commands", + "simulation", + "pre_run_results", + "post_run_results", + "sandbox_path", + "environment_info", +} + +# Recomputed by this pass — carrying them would defeat the point. +RECOMPUTED = { + # The verdict itself: exactly what the grading pass produces. + "final_status", + "weighted_score", + "success_criteria_results", + "post_failure_criteria_results", + # Derived from `iterations`, which IS carried — so seeding the trajectory + # reproduces these exactly without copying them. + "model_used", + "command_stats", + "total_token_usage", + "total_assistant_turns", + "actual_commands", + "commands_efficiency", + # Identity, supplied by the caller from the task being graded. + "task_id", + "task_description", + "variant_id", + "agent_type", + "task_config", + # Timing: `duration_seconds` is restored from the prior result in + # _finalize_result (after its own timing write), and `completed_at` marks + # when the row reached its final state, which the grade genuinely changes. + "duration_seconds", + "completed_at", +} + + +def _prior() -> EvaluationResult: + """A prior result with a distinctive value in every carried field.""" + return EvaluationResult( + task_id="t", + task_description="d", + variant_id="v", + agent_type=AgentKind.CLAUDE_CODE, + started_at=datetime(2020, 1, 1, 0, 0, 0), + final_status=FinalStatus.NOT_GRADED, + iteration_count=7, + max_turns_exhausted=True, + error_message="prior message", + error_details={"where": "prior"}, + error_log_tail="prior tail", + sdk_options={"opt": "prior"}, + agent_config=parse_agent_config(type=AgentKind.CLAUDE_CODE, model="prior-model"), + expected_commands=11, + pre_run_results=[PostRunResult(command="prior-pre", exit_code=0)], + post_run_results=[PostRunResult(command="prior-post", exit_code=0)], + sandbox_path="/prior/workspace", + environment_info={"installed_tools": "prior"}, + early_stop=EarlyStopInfo( + reason=EarlyStopReason.CRITERION_FAILED, + deciding_criterion_type="skill_triggered", + deciding_criterion_description="the armed criterion", + sdk_turn_index=0, + tool_call_index=1, + elapsed_seconds=1.0, + gate_threshold=1.0, + ), + ) + + +def _seeded(tmp_path: Path) -> tuple[Orchestrator, EvaluationResult]: + task = TaskDefinition( + task_id="t", + description="d", + initial_prompt="p", + agent=parse_agent_config(type=AgentKind.CLAUDE_CODE), + success_criteria=[FileExistsCriterion(path="x.txt", description="x")], + ) + prior = _prior() + orch = Orchestrator(task=task, run_dir=tmp_path, variant_id="v", prior_result=prior) + orch.result = EvaluationResult( + task_id="t", + task_description="d", + variant_id="v", + agent_type=AgentKind.CLAUDE_CODE, + started_at=datetime(2030, 1, 1, 0, 0, 0), + final_status=FinalStatus.FAILURE, + iteration_count=0, + environment_info={"installed_tools": "grader"}, + ) + orch._seed_from_prior_result() + assert orch.result is not None + return orch, prior + + +def test_field_partition_covers_every_evaluation_result_field() -> None: + """The sensor. A field added to EvaluationResult must be classified as + carried or recomputed before this passes — otherwise it silently defaults on + every detached grade, which is exactly how `agent_config` was being lost.""" + classified = CARRIED | RECOMPUTED + fields = set(EvaluationResult.model_fields) + assert not fields - classified, ( + f"Unclassified EvaluationResult field(s): {sorted(fields - classified)}. Decide whether " + "_seed_from_prior_result must carry them, then add them to CARRIED or RECOMPUTED." + ) + assert not classified - fields, f"Stale entries: {sorted(classified - fields)}" + + +@pytest.mark.parametrize("field", sorted(CARRIED - {"environment_info"})) +def test_every_carried_field_reaches_the_regrade(field: str, tmp_path: Path) -> None: + orch, prior = _seeded(tmp_path) + assert orch.result is not None + assert getattr(orch.result, field) == getattr(prior, field), ( + f"_seed_from_prior_result dropped `{field}`; the graded row would report its default " + "instead of what the run actually did." + ) + + +def test_early_stop_is_carried_because_it_selects_the_gate(tmp_path: Path) -> None: + """Called out separately because it is the one carried field that changes the + VERDICT: gate selection is FIRED-ONLY, so a dropped early_stop re-grades a + truncated trajectory under the full-run strict-AND gate.""" + orch, _ = _seeded(tmp_path) + assert orch.result is not None and orch.result.early_stop is not None + assert orch.result.early_stop.reason is EarlyStopReason.CRITERION_FAILED + + +def test_grader_environment_is_kept_beside_the_run_s_not_over_it(tmp_path: Path) -> None: + orch, _ = _seeded(tmp_path) + assert orch.result is not None + # The run's own capture wins: a report showing the grader's tool versions as + # the run's is worse than one showing neither. + assert orch.result.environment_info["installed_tools"] == "prior" + assert orch.result.environment_info["graded_by"] == {"installed_tools": "grader"} + + +def test_the_evaluate_only_path_selects_the_same_gate_as_the_agent_path(tmp_path: Path) -> None: + """C1: gate selection is FIRED-ONLY, and a detached grade reaches the verdict + through the evaluate-only branch. That branch used to call + ``all_criteria_passed`` unconditionally, so re-grading an early-stopped run + applied the full-run strict-AND gate to a truncated trajectory and could flip + SUCCESS into FAILURE. Both paths must go through ``_select_gate``.""" + import inspect + + source = inspect.getsource(Orchestrator._evaluation_loop) + assert source.count("_select_gate()") == 2, ( + "Both the evaluate-only branch and the agent branch must select the gate through " + "_select_gate(); a second hand-written selection is how the seeded early_stop " + "came to be carried but never read." + ) + assert "all_criteria_passed" not in source, "gate selection belongs in _select_gate, not inline" + + # A truncated run: the ARMED criterion passed, the unarmed one never had the + # chance to. The armed gate says SUCCESS; strict-AND says FAILURE. That + # difference IS the flipped verdict, and it is decided purely by early_stop. + task = TaskDefinition( + task_id="t", + description="d", + initial_prompt="p", + agent=parse_agent_config(type=AgentKind.CLAUDE_CODE), + success_criteria=[ + CommandExecutedCriterion( + description="armed", + tool_name="Read", + require_success=True, + stop_early={"on_pass": "stop"}, # type: ignore[arg-type] + ), + FileExistsCriterion(path="x.txt", description="unarmed"), + ], + ) + orch = Orchestrator(task=task, run_dir=tmp_path, variant_id="v", prior_result=_prior()) + orch.result = _prior() + orch.result.success_criteria_results = [ + CriterionResult(criterion_type="command_executed", description="armed", score=1.0), + CriterionResult(criterion_type="file_exists", description="unarmed", score=0.0), + ] + + assert orch._select_gate() is True, "an early-stopped run gates on the armed subset" + orch.result.early_stop = None + assert orch._select_gate() is False, "a run that completed naturally gates strict-AND" + + +def test_grading_cannot_overturn_an_execution_fact() -> None: + """H5: a detached grade re-runs the criteria over a trajectory it did not + produce. It may move NOT_GRADED to a verdict; it must not turn a run that + timed out or crashed into a pass.""" + assert not FinalStatus.NOT_GRADED.is_execution_fact + assert not FinalStatus.SUCCESS.is_execution_fact + assert not FinalStatus.FAILURE.is_execution_fact + for status in ( + FinalStatus.ERROR, + FinalStatus.TIMEOUT, + FinalStatus.BUILD_FAILED, + FinalStatus.MAX_TURNS_EXHAUSTED, + FinalStatus.TOKEN_BUDGET_EXCEEDED, + FinalStatus.COST_BUDGET_EXCEEDED, + ): + assert status.is_execution_fact, f"{status} describes the run, so grading must preserve it" + + +def test_seeding_is_a_no_op_without_a_prior_result(tmp_path: Path) -> None: + task = TaskDefinition( + task_id="t", + description="d", + initial_prompt="p", + agent=parse_agent_config(type=AgentKind.CLAUDE_CODE), + success_criteria=[FileExistsCriterion(path="x.txt", description="x")], + ) + orch = Orchestrator(task=task, run_dir=tmp_path, variant_id="v") + orch.result = EvaluationResult( + task_id="t", + task_description="d", + variant_id="v", + agent_type=AgentKind.CLAUDE_CODE, + started_at=datetime(2030, 1, 1), + final_status=FinalStatus.FAILURE, + iteration_count=0, + ) + orch._seed_from_prior_result() + assert orch.result.started_at == datetime(2030, 1, 1) + assert orch.result.iterations == [] From 489383d05d0a31c964021c644f141c5dbe18c07b Mon Sep 17 00:00:00 2001 From: Akshaya Shanbhogue Date: Thu, 3 Sep 2026 15:41:24 -0700 Subject: [PATCH 5/6] fix(eval): address the medium and low findings from the branch review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The 23 medium / 20 low findings from the same review pass. Grouped by what they change rather than by axis. Correctness * `Sandbox.adopt` discovered `/.venv` unconditionally, while `setup` only ever populates `venv_dir` when `config.python` is set. A venv the task never asked for was prepended to PATH and exported as VIRTUAL_ENV for every criterion — the exact divergence the `command_base_path` round trip exists to close, and a way for an agent to shadow binaries from its own workspace. * `default_workspace` inferred the workspace as "the single child of artifacts/". A dataset row's `task_id` is `/`, so that resolves one level too high and every path-relative criterion then fails as a locating artifact rather than as a verdict. It now resolves `artifacts/` exactly, and RAISES when ambiguous instead of guessing the parent. * `_write_back` overwrote the canonical `task.json` with a plain `write_text` while the orchestrator writes the same file via tmp + `os.replace`. A torn write parses as malformed, which `--resume` reads as "not complete" and pays for the agent again. One `write_text_atomic` helper now serves both. * A grading crash wrote `ERROR` over a re-gradeable `NOT_GRADED` row — and `ERROR` is "complete" for both commands, so the row could never be graded again. Both detached paths now keep the ungraded row. * `load_prior_result` sat outside the resume loop's `try`, so one unreadable row aborted the whole resume BEFORE `run_batch` — none of the `to_run` tasks executed either, the opposite of the documented "one bad row never aborts". * `back_up_pre_grade_record` ran after the orchestrator, so with `--run-dir` pointing at the target it captured an already-graded record — destroying the evidence it exists to preserve. It is now taken during input resolution. * `verify_reference_unchanged` moved INSIDE `regrade_in_place`: a guard a caller has to remember is one a third caller will forget. * `completed_at` is carried from the prior run, so a re-graded row's three time fields agree with each other. * `grade` is now coerced at the container boundary rather than annotated — `"false"` is a truthy str. Reporting * `VariantAggregate.average_score` is `float | None`; `_mean_graded_score` returned 0.0 for the case that actually happens (nothing graded), printing `Average Score: 0.000` beside `Pass Rate: n/a`. * `SuiteRollup` gains `rows_not_graded`, the graded denominator, and the row-count invariant its two siblings have and it did not. * `_seed_from_prior_result` nested a whole env capture under `graded_by`; `environment_info` is consumed as a FLAT map (the HTML report `_esc`apes each value into a cell), so it renders as a Python dict repr. Flattened to `graded_by_*` scalars, kept only where they differ, and a second grade no longer clobbers the first grader's stamp. * `command_base_path` is a full PATH string written on every run; it and the provenance keys are excluded from the rendered Environment tables. * The end-of-run hint pointed at `evaluate ` — the shape with NO trajectory, which scores trajectory-reading criteria differently from what `run` would have produced. An empty run also printed no Results line. Two new lint rules, each of which found a live instance the moment it ran * CE047 — an `environment_info` key that is read must be written somewhere in `src/`. This is the durable form of the `reference_digest` fix: the bag is `dict[str, Any]`, so nothing connects a reader to its writer, and a reader with no writer is silently inert. * CE048 — never call a Typer command function in process. It scans `tests/` as well, because that is the only place the defect occurs, and it immediately found six live calls to `plan_command` — whose body already carried an `isinstance(experiment, Path)` guard papering the sentinel over. Split into `run_plan`, matching `run_pipeline` / `run_evaluation`. Also: `TASK_JSON` / `.venv` are single constants in `path_utils` instead of two half-copies plus ten literals; symlink refusal and a containment check on the paths a shared run dir supplies; `evaluate --help`'s usage line no longer renders `[]`; `--resume` and `--preserve` help match the behavior; the resumable-dataset constraint, `task.execute.json` and the suite schema are documented. Tests: `test_ungraded_reporting.py` (JUnit ``, the switched Markdown denominator, the console summary, and the `VariantAggregate` twin of the four `RunSummary` cases), `test_detached_grading_guards.py` (the simulation refusal, `--in-place`/`--copy` selection, the PATH round trip and its filter, the LiteLLM skip), plus grade-idempotence, `--workspace`, the execution-fact refusal, the resume error paths and a `/`-bearing dataset id. make verify green (4688 passed, 92.26%); evalboard 621 tests green. Co-Authored-By: Claude Opus 5 (1M context) --- .claude/shared/run-layout.md | 1 + CLAUDE.md | 2 +- docs/REPORT_SCHEMA.md | 7 +- docs/USER_GUIDE.md | 10 +- plugins/coder-eval/reference/run-layout.md | 1 + src/coder_eval/cli/evaluate_command.py | 60 +++- src/coder_eval/cli/evaluate_target.py | 9 +- src/coder_eval/cli/plan_command.py | 15 +- src/coder_eval/cli/run_command.py | 55 +++- src/coder_eval/cli/run_helpers.py | 13 +- .../cli/run_task_internal_command.py | 10 +- src/coder_eval/models/experiment.py | 5 +- src/coder_eval/orchestration/experiment.py | 15 +- src/coder_eval/orchestration/regrade.py | 97 +++++-- src/coder_eval/orchestrator.py | 41 ++- src/coder_eval/path_utils.py | 27 ++ src/coder_eval/reports_experiment.py | 5 +- src/coder_eval/reports_html.py | 6 +- src/coder_eval/reports_stats.py | 13 + src/coder_eval/sandbox.py | 22 +- .../rules/ce047_env_info_key_round_trip.py | 122 ++++++++ .../ce048_no_in_process_typer_command_call.py | 106 +++++++ tests/lint/runner.py | 4 + tests/test_custom_lint.py | 10 +- tests/test_detached_grading_guards.py | 261 ++++++++++++++++++ tests/test_early_stop.py | 6 +- tests/test_execute_command.py | 8 +- tests/test_execute_evaluate_loop.py | 96 +++++++ tests/test_plan_command.py | 22 +- tests/test_regrade.py | 55 +++- tests/test_seed_from_prior_result.py | 12 +- tests/test_ungraded_reporting.py | 254 +++++++++++++++++ 32 files changed, 1258 insertions(+), 112 deletions(-) create mode 100644 tests/lint/rules/ce047_env_info_key_round_trip.py create mode 100644 tests/lint/rules/ce048_no_in_process_typer_command_call.py create mode 100644 tests/test_detached_grading_guards.py create mode 100644 tests/test_ungraded_reporting.py diff --git a/.claude/shared/run-layout.md b/.claude/shared/run-layout.md index 57edc4a6..89f2bd10 100644 --- a/.claude/shared/run-layout.md +++ b/.claude/shared/run-layout.md @@ -12,6 +12,7 @@ runs/////{task.json, task.log, artifacts/} - `` — zero-padded replicate index (e.g. `00`, `01`). - `task.json` — the persisted per-replicate result (the consumer contract; carries the large `iterations` array — still accepted under its former name `turns` when reading, but not what current runs write). - `task.json.malformed` — present only on the docker degrade path: when an existing `task.json` fails to parse (schema skew from a stale `:latest` image, or a truncated/torn write), the docker runner moves the unparseable original aside to this sidecar and writes a synthetic `final_status=ERROR` `task.json` in its place. Diagnostic-only; `rglob("task.json")` consumers do not match it. +- `task.execute.json` — present only after a DETACHED grade (`coder-eval evaluate ` or `coder-eval run --resume` over a `NOT_GRADED` row). The pre-grade snapshot of `task.json`, written once and never overwritten by a later grade, so "this run was executed separately from grading" stays auditable. Diagnostic-only; `rglob("task.json")` consumers do not match it. - `task.log` — the human-readable task log; `artifacts/` — files the agent produced. **Scope-marker files** (used to detect what a given path represents): diff --git a/CLAUDE.md b/CLAUDE.md index 2b8ac212..71e487f3 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -223,7 +223,7 @@ make plugin-reference # the plugin's bundled criteria reference from the models Editing `src/coder_eval/pricing.py` means editing `evalboard/lib/pricing.ts` too — it is a hand-copied mirror, and `evalboard/lib/__tests__/pricing-parity.test.ts` fails the build on drift in either direction. -Recent additions, each traceable to a shipped defect: **CE037** (no unreferenced module-level private helper in `src/` — a helper whose docstring documents a bug the live code still has is worse than none), **CE038** (in an `@asynccontextmanager`, the acquire must sit INSIDE the `try` whose `finally` releases it — `asyncio.shield` protects the inner task, NOT the await, so a cancel on `__aenter__` skips the unwind while the work completes), **CE039** (a criterion checker must not return a gating `score=0.0` from an `except OSError` over a path the *task author* named — that books an eval-config error as an agent failure; raise `CheckerMisuseError` instead, and `# noqa: CE039` the cases that really are the agent's). +Recent additions, each traceable to a shipped defect: **CE047** (an `environment_info` key that is READ must be WRITTEN somewhere in `src/` — the bag is `dict[str, Any]`, so nothing connects reader to writer, and the `reference_digest` anti-cheat guard shipped as a read with no writer anywhere: `.get()` returned `None`, the guard took its early return, and CLAUDE.md plus the user guide both described it as protection it never provided), **CE048** (never call a Typer command function in process — its parameter defaults are `OptionInfo` sentinels, not values, and the sentinel is TRUTHY, so `in_place=None` silently selected the wrong branch; the fix is the `run_pipeline` / `run_evaluation` / `run_plan` split, and this rule is the one that also scans `tests/`, since that is the only place the defect occurs), **CE037** (no unreferenced module-level private helper in `src/` — a helper whose docstring documents a bug the live code still has is worse than none), **CE038** (in an `@asynccontextmanager`, the acquire must sit INSIDE the `try` whose `finally` releases it — `asyncio.shield` protects the inner task, NOT the await, so a cancel on `__aenter__` skips the unwind while the work completes), **CE039** (a criterion checker must not return a gating `score=0.0` from an `except OSError` over a path the *task author* named — that books an eval-config error as an agent failure; raise `CheckerMisuseError` instead, and `# noqa: CE039` the cases that really are the agent's). When fixing a bug, ask: *could a custom lint rule have prevented this?* If the root cause is a mechanically detectable pattern (e.g., "always import from `coder_eval.models`", "never call blocking IO in async"), add a rule to `tests/lint/rules/` following the CE001+ pattern and wire it up in `tests/lint/runner.py`. This turns a one-time fix into permanent enforcement. See `tests/test_custom_lint.py` for how rules are tested. (Doc-surface / whole-tree rules that reason over Markdown/YAML or the entire `src/` tree rather than one `.py` AST at a time — CE026–CE031, CE033–CE036 — are not `BaseRule`s in the runner; they are wired as dedicated `@pytest.mark.lint` test classes. CE036 enforces the `live_verdict` determinism + monotonicity contract (`criteria/base.py`) that `EarlyStopWatcher`'s latching, deferred fail-stop, and flip-attribution silently depend on: monotonicity over arbitrary Python is undecidable, so instead of a static check it REPLAYS each live criterion against every prefix of recorded trajectories (`tests/lint/live_verdict_contract.py::CASES`) — on the authored ordering AND under seeded shuffles (`permuted_violations`, which catch order-sensitive bugs the authored walk misses) — and asserts the property directly, plus registry-derived coverage — every `LiveSuccessCriterion` in the union must have cases, and every polarity its instances claim via `live_decidable_polarities()` must actually be reached by one (otherwise a single always-`undecided` fixture would "cover" a type while proving nothing). Adding a live criterion therefore means adding `ContractCase`s in the same change. CE035 resolves every `steps..outputs.` / `needs..outputs.` reference in `.github/workflows/**` to a writer that actually produces that key — GitHub expands an unwritten output to the empty string, so a typo degrades a gate silently and actionlint models `steps.*.outputs` as an open string map. CE034 scans `tasks/` and forces an armed, live-*passable* `command_executed` to set `require_success` — a crashed invocation would otherwise latch a live PASS, fire `on_pass: stop`, and let FIRED-ONLY armed gating report SUCCESS without ever consulting the unarmed criteria (negative assertions are fail-only and are exempt). CE033 keeps the plugin's bundled `reference/criteria.md` in parity with the `SuccessCriterion` union that generates it (`make plugin-reference` writes it; the rule re-renders and diffs — never hand-edit the file). CE031 guards against dead config: a behavior-driving field on `SimulationConfig`/`RunLimits`/`Dataset` that no code reads by name. CE026 keeps the GitHub Action's onboarding surfaces honest — `README.md`, `docs/CI_GATE.md`, `docs/tutorials/02-ci-pipeline.md`, and the plugin's `ci` skill, whose emitted workflow users copy into their own repos: a page's *first* Action snippet must show the agent-runtime prerequisite steps (pinned to the `action-dogfood` job that proves them in CI), a zero-install absolute next to such a snippet must name the channel it means, every `github.com/marketplace/actions/` link plus the shields badge label must match `action.yml`'s `name:`, and every `with:` key on a snippet's action step must be a real `action.yml` input (GitHub ignores unknown inputs, so a rename would silently degrade every copied workflow). Renaming an action input or changing its runtime prerequisites therefore means updating the skill too.) diff --git a/docs/REPORT_SCHEMA.md b/docs/REPORT_SCHEMA.md index b4c5dce0..c9405ae6 100644 --- a/docs/REPORT_SCHEMA.md +++ b/docs/REPORT_SCHEMA.md @@ -23,6 +23,7 @@ read). Times are ISO-8601. | --- | --- | --- | | `run.json` / `run.md` | `RunSummary` | Every run (and rebuildable via `coder-eval aggregate`) | | `///task.json` | `EvaluationResult` | One per replicate | +| `///task.execute.json` | `EvaluationResult` | Pre-grade snapshot, written once by a detached grade (`evaluate ` / `run --resume`). Deliberately **not** matched by `rglob("task.json")`, so it never enters an aggregation. | | `//suite.json` / `.md` | `SuiteRollup` | Dataset-backed suites only | | `experiment.json` / `.md` | `ExperimentResult` | Every run (experiment layer) | | `/variant.json` / `.md` | `VariantAggregate` | Per variant | @@ -44,7 +45,7 @@ run-level summary; full per-replicate detail lives in each `task.json`. | `start_time` / `end_time` | `datetime` | Run window. | | `total_duration_seconds` | `float` | Wall-clock. | | `tasks_run` | `int` | Total replicates executed. | -| `tasks_succeeded` / `tasks_failed` / `tasks_error` / `tasks_not_graded` | `int` | Category counts. **Invariant:** the four sum to `tasks_run`. | +| `tasks_succeeded` / `tasks_failed` / `tasks_error` | `int` | Category counts. **Invariant:** these three plus `tasks_not_graded` sum to `tasks_run`. | | `tasks_not_graded` | `int` | Tasks run by `coder-eval execute` — executed, deliberately unscored. Excluded from **both** sides of `pass_rate`. Defaults to `0`, so pre-`execute` `run.json` still parses. | | `tasks_token_budget_exceeded` / `tasks_cost_budget_exceeded` | `int` | Sub-counters of `tasks_failed` (not part of the invariant). | | `skipped_tasks` | `list[{path, reason}]` | Load failures / `skip: true` opt-outs. | @@ -265,8 +266,8 @@ Written for dataset-backed suites; its `passed` flag drives the CI exit code. | Key | Type | Meaning | | --- | --- | --- | | `suite_id` / `variant_id` | `str` | Identity. | -| `rows_total` / `rows_passed` / `rows_failed` / `rows_error` | `int` | Row counts. | -| `pass_rate` | `float` | `rows_passed / rows_total`. | +| `rows_total` / `rows_passed` / `rows_failed` / `rows_error` / `rows_not_graded` | `int` | Row counts. **Invariant:** the four category counts sum to `rows_total`. `rows_not_graded` defaults to `0`. | +| `pass_rate` | `float` | `rows_passed / (rows_total - rows_not_graded)` — ungraded rows leave both sides, matching `RunSummary.pass_rate`. | | `average_weighted_score` | `float \| null` | Mean row score. | | `criterion_stats` | `list[{criterion_type, rows_evaluated, average_score, error_count}]` | Per-criterion summary. | | `failed_samples` | `list[FailedRowSummary]` | Capped at 20 (`{row_id, task_id, final_status, weighted_score, failure_reasons, error_message, task_json_relpath, replicate_index}`). | diff --git a/docs/USER_GUIDE.md b/docs/USER_GUIDE.md index c37fa46f..bbd65286 100644 --- a/docs/USER_GUIDE.md +++ b/docs/USER_GUIDE.md @@ -80,7 +80,7 @@ you want to iterate on afterwards. Grade the results later with budget breach still reports `ERROR` / `TIMEOUT` / `TOKEN_BUDGET_EXCEEDED` and still exits non-zero, exactly as under `run`. -Every `run` flag is available except three things, each refused rather than quietly +Every `run` flag is available except two things, each refused rather than quietly degraded: | Not supported | Why | @@ -127,6 +127,14 @@ original-config results, so the run genuinely mixes configs. The `grade` flag is exempt from that warning, because `execute` → `run --resume` is a supported flow rather than a config mistake. +**A dataset task must pin its sample to be resumable.** Stratified sampling +(`--sample-per-stratum` / `dataset.sample_per_stratum`) re-draws on every +invocation, and each row is its own task (`/`) with its own run +directory. A resume therefore draws a *different* row set, finds no `task.json` +for it, and pays for the agent a second time while the executed rows sit +orphaned in the run dir. Set `dataset.sample_seed`, or use `--sample N` (which is +seeded), before splitting a dataset run across `execute` and `run --resume`. + ### `coder-eval plan` — validate tasks ```bash diff --git a/plugins/coder-eval/reference/run-layout.md b/plugins/coder-eval/reference/run-layout.md index a080dedb..2d4a5e65 100644 --- a/plugins/coder-eval/reference/run-layout.md +++ b/plugins/coder-eval/reference/run-layout.md @@ -11,6 +11,7 @@ runs/////{task.json, task.log, artifacts/} - `` — zero-padded replicate index (e.g. `00`, `01`). - `task.json` — the persisted per-replicate result (the consumer contract; carries the large `iterations` array — still accepted under its former name `turns` when reading, but not what current runs write). - `task.json.malformed` — present only on the docker degrade path: when an existing `task.json` fails to parse (schema skew from a stale `:latest` image, or a truncated/torn write), the docker runner moves the unparseable original aside to this sidecar and writes a synthetic `final_status=ERROR` `task.json` in its place. Diagnostic-only; `rglob("task.json")` consumers do not match it. +- `task.execute.json` — present only after a DETACHED grade (`coder-eval evaluate ` or `coder-eval run --resume` over a `NOT_GRADED` row). The pre-grade snapshot of `task.json`, written once and never overwritten by a later grade, so "this run was executed separately from grading" stays auditable. Diagnostic-only; `rglob("task.json")` consumers do not match it. - `task.log` — the human-readable task log; `artifacts/` — files the agent produced. **Scope-marker files** (used to detect what a given path represents): diff --git a/src/coder_eval/cli/evaluate_command.py b/src/coder_eval/cli/evaluate_command.py index 8c61afc9..d962b050 100644 --- a/src/coder_eval/cli/evaluate_command.py +++ b/src/coder_eval/cli/evaluate_command.py @@ -20,11 +20,10 @@ parse_agent_config, ) from ..orchestration.regrade import ( - PRE_GRADE_JSON, - TASK_JSON, RegradeError, back_up_pre_grade_record, default_workspace, + grading_sandbox_config, load_prior_result, regrade_in_place, task_from_prior, @@ -32,6 +31,7 @@ ) from ..orchestration.task_loader import load_task from ..orchestrator import Orchestrator +from ..path_utils import PRE_GRADE_JSON_FILENAME, TASK_JSON_FILENAME, write_text_atomic from ..sandbox import Sandbox from .console import console from .evaluate_target import EvaluateMode, EvaluateTarget, EvaluateTargetError, resolve_evaluate_target @@ -117,6 +117,11 @@ def _resolve_run_dir_or_work_dir(target: EvaluateTarget, workspace: Path | None) if prior is not None: verify_reference_unchanged(prior, task, task_file) + # Snapshot the ungraded record BEFORE anything grades. Taking it inside + # _write_back instead would capture an ALREADY-GRADED record whenever + # --run-dir points at the target run dir (the orchestrator writes there + # first), destroying the very evidence the copy exists to preserve. + back_up_pre_grade_record(target.target) return _ResolvedInputs( target=target, @@ -143,13 +148,18 @@ def _replicate_index_of(run_dir: Path) -> int: def evaluate_command( task_or_run_dir: Path = typer.Argument( # noqa: B008 ..., - metavar="[TASK_FILE] TARGET", + # One metavar per positional, so the usage line reads as Click renders + # it. A composite metavar on the first ("[TASK_FILE] TARGET") plus an + # empty one on the second produced `[TASK_FILE] TARGET []`. + metavar="TASK_FILE_OR_RUN_DIR", help="Task YAML file, or (when it is the only argument) a finished run directory.", exists=True, ), work_dir: Path | None = typer.Argument( # noqa: B008 None, - metavar="", + # No metavar="" here: an empty one leaks a bare `[]` into both the usage + # line and the arguments table. The first positional's metavar already + # spells out the two shapes. help="Directory containing the code to evaluate. Omit when TASK_FILE is a run directory.", ), workspace: Path | None = typer.Option( # noqa: B008 @@ -179,7 +189,11 @@ def evaluate_command( True, "--preserve/--no-preserve", "-p/-P", - help="Move sandbox artifacts to run directory (default: preserve). The temp sandbox is always removed.", + help=( + "Move sandbox artifacts to run directory (default: preserve). The temp sandbox is " + "always removed. Ignored when grading in place (the default for a run directory) — " + "an adopted directory is never moved or deleted." + ), ), run_dir: Path | None = typer.Option( # noqa: B008 None, @@ -260,15 +274,11 @@ def run_evaluation( console.print(f"[red]✗ Failed to prepare run directory:[/red] {e}") raise typer.Exit(1) from e - sandbox_config = task.sandbox.model_copy(deep=True) + sandbox_config = grading_sandbox_config(task) if not grade_in_place: # Copy path: preload the sandbox with the work dir as a template source. template_source = TemplateDirSource(path=str(graded_dir.resolve())) sandbox_config.template_sources = [template_source, *(sandbox_config.template_sources or [])] - # Grading never runs a container: the docker driver dispatches through - # DockerRunner, which needs an agent. Force tempdir so a task whose YAML says - # `driver: docker` is still gradeable on the host. - sandbox_config = sandbox_config.model_copy(update={"driver": "tempdir"}) task_dir = task_file.parent.resolve() if task_file is not None else None sandbox = Sandbox(sandbox_config, task_id=task.task_id, task_dir=task_dir) @@ -361,7 +371,19 @@ async def _setup_and_run() -> EvaluationResult: f"[dim]Re-graded {prior.final_status.value} → {result.final_status.value} " + f"over {len(result.iterations)} recorded turn(s).[/dim]" ) - _write_back(target.target, result) + if result.final_status is FinalStatus.ERROR: + # A grading-time crash (a failing checker, an unreachable judge) is + # not a verdict about the run. Writing it back would replace a + # perfectly re-gradeable NOT_GRADED row with ERROR — which BOTH + # commands treat as permanently complete, so the run could never be + # graded again without hand-restoring task.execute.json. The + # diagnostic row is still in this grade's own run dir. + console.print( + f"[yellow]⚠[/] Grading errored; leaving {target.target / TASK_JSON_FILENAME} " + + "as it was so the run stays re-gradeable." + ) + else: + _write_back(target.target, result) if result.final_status == FinalStatus.ERROR: console.print(f"\n[red]✗ Evaluation error: {result.error_message}[/red]") @@ -385,11 +407,19 @@ def _write_back(run_dir: Path, result: EvaluationResult) -> None: ungraded record is auditable — the write is not a silent overwrite of the only evidence that the run was executed separately. """ - target = run_dir / TASK_JSON - backup = run_dir / PRE_GRADE_JSON - back_up_pre_grade_record(run_dir) + target = run_dir / TASK_JSON_FILENAME + backup = run_dir / PRE_GRADE_JSON_FILENAME + if target.is_symlink(): + # A run directory is a shareable artifact, so its task.json is untrusted + # input. Following a symlink here turns `evaluate ` into an + # arbitrary-file-overwrite primitive on the grader's host. + console.print(f"[yellow]⚠[/] {target} is a symlink; refusing to write through it.") + return try: - target.write_text(result.model_dump_json(indent=2), encoding="utf-8") + # Atomic, matching the orchestrator's own task.json writer: a torn write + # here makes the row parse as malformed, which a later --resume reads as + # "not complete" and re-pays for the agent. + write_text_atomic(target, result.model_dump_json(indent=2)) except OSError as e: # Never fail the grade over the write-back: the verdict was computed and # already printed, and the fresh run dir holds its own task.json. diff --git a/src/coder_eval/cli/evaluate_target.py b/src/coder_eval/cli/evaluate_target.py index d8039a4e..f9b72b04 100644 --- a/src/coder_eval/cli/evaluate_target.py +++ b/src/coder_eval/cli/evaluate_target.py @@ -21,8 +21,7 @@ from enum import StrEnum from pathlib import Path - -TASK_JSON = "task.json" +from ..path_utils import TASK_JSON_FILENAME class EvaluateMode(StrEnum): @@ -53,7 +52,7 @@ class EvaluateTargetError(ValueError): def is_run_dir(path: Path) -> bool: """Whether ``path`` is a finished task run directory (it holds ``task.json``).""" - return (path / TASK_JSON).is_file() + return (path / TASK_JSON_FILENAME).is_file() def resolve_evaluate_target(first: Path, second: Path | None) -> EvaluateTarget: @@ -79,12 +78,12 @@ def resolve_evaluate_target(first: Path, second: Path | None) -> EvaluateTarget: if not first.is_dir(): raise EvaluateTargetError( f"{first} is not a directory. With a single argument, pass a finished run " - + f"directory (one containing {TASK_JSON}). To grade a directory against a " + + f"directory (one containing {TASK_JSON_FILENAME}). To grade a directory against a " + "task, pass both: coder-eval evaluate " ) if not is_run_dir(first): raise EvaluateTargetError( - f"{first} holds no {TASK_JSON}, so it is not a run directory. Pass the task " + f"{first} holds no {TASK_JSON_FILENAME}, so it is not a run directory. Pass the task " + f"file too: coder-eval evaluate {first}" ) return EvaluateTarget(mode=EvaluateMode.RUN_DIR, target=first, task_file=None) diff --git a/src/coder_eval/cli/plan_command.py b/src/coder_eval/cli/plan_command.py index 25318717..61d2da8b 100644 --- a/src/coder_eval/cli/plan_command.py +++ b/src/coder_eval/cli/plan_command.py @@ -47,6 +47,19 @@ def plan_command( coder-eval plan tasks/*.yaml coder-eval plan tasks/*.yaml -e experiments/model-comparison.yaml """ + run_plan(task_files=task_files, experiment=experiment) + + +def run_plan(*, task_files: list[Path] | None = None, experiment: Path | None = None) -> None: + """The body of ``coder-eval plan``, with real Python defaults. + + Split from the Typer signature for the same reason as ``run_pipeline`` / + ``run_evaluation``: calling a Typer command function in process hands every + unspecified option an ``OptionInfo`` sentinel rather than its default, and + the sentinel is truthy. The `isinstance(experiment, Path)` guard this + function used to need was that bug being papered over rather than fixed. + Callers (tests, library use) call this. Enforced by lint rule CE048. + """ # Default to discovering all tasks under tasks/ when none provided resolved_task_files = task_files if task_files else discover_default_tasks() @@ -64,7 +77,7 @@ def plan_command( from ..orchestration.run_limits import validate_run_limits # Always load experiment (defaults to experiments/default.yaml) - exp_path = experiment if isinstance(experiment, Path) else DEFAULT_EXPERIMENT_PATH + exp_path = experiment or DEFAULT_EXPERIMENT_PATH try: exp_def = load_experiment(exp_path) if exp_path == DEFAULT_EXPERIMENT_PATH: diff --git a/src/coder_eval/cli/run_command.py b/src/coder_eval/cli/run_command.py index 5c3de001..be6ee073 100644 --- a/src/coder_eval/cli/run_command.py +++ b/src/coder_eval/cli/run_command.py @@ -17,7 +17,7 @@ from ..config import Settings, settings from ..logging_config import setup_logging -from ..models import PreservationMode, ResolvedTask, RunSummary, TaskResult +from ..models import EvaluationResult, FinalStatus, PreservationMode, ResolvedTask, RunSummary, TaskResult from ..orchestration.config import BatchRunConfig from ..path_utils import create_latest_symlink, format_task_log_id from ..streaming.callbacks import CompositeStreamCallback @@ -192,10 +192,16 @@ def run_command( "Resume an interrupted run: skip tasks already finalized in --run-dir and " "run only the rest, folding prior results into run.json. A task counts as " "finalized once it has ANY final status — including FAILED/ERROR — so resume " - "does NOT retry failures (delete a task's task.json to force a re-run). " + "does NOT retry failures (delete a task's task.json to force a re-run). The " + "one exception is a NOT_GRADED row left by `coder-eval execute`: `run` was " + "asked for a verdict, so those rows are GRADED in place against the " + "trajectory and workspace already on disk, without re-running the agent. " "Requires --run-dir. A config mismatch (model/backend/flags) is warned, not " "refused — the resumed tasks keep their original-config results, so the run " - "mixes configs; use a fresh --run-dir to keep configs separate." + "mixes configs; use a fresh --run-dir to keep configs separate. A dataset " + "task using nondeterministic stratified sampling needs dataset.sample_seed " + "(or --sample) to be resumable — otherwise the resume draws a different row " + "set and pays for the agent twice." ), ), max_parallel: int = typer.Option( @@ -678,7 +684,15 @@ async def _grade_resumed_tasks(to_grade: list[ResolvedTask]) -> list[tuple[Resol A task that cannot be graded is reported and folded back in with its ORIGINAL ungraded result, so one bad row neither aborts the resume nor silently - vanishes from run.json — it stays visible as ``tasks_not_graded``. + vanishes from run.json — it stays visible as ``tasks_not_graded``, with the + reason on its ``error_message``. That covers three shapes: a helper raising, + a row too broken to read at all (skipped entirely — there is nothing to fold + back), and a re-grade that returns ``FinalStatus.ERROR``, which + ``Orchestrator.run()`` produces INSTEAD of raising and which would otherwise + make the row permanently un-regradeable. + + Returns the graded rows; the caller's exit gate fails the command whenever + any row is still ungraded, so a resume that graded nothing never exits 0. """ from ..orchestration.regrade import ( RegradeError, @@ -686,14 +700,20 @@ async def _grade_resumed_tasks(to_grade: list[ResolvedTask]) -> list[tuple[Resol default_workspace, load_prior_result, regrade_in_place, - verify_reference_unchanged, ) graded: list[tuple[ResolvedTask, TaskResult]] = [] + failed_to_load: list[str] = [] for rt in to_grade: - prior = load_prior_result(rt.run_dir) + # Inside the try: an unreadable row must skip like any other grading + # failure. Outside it, one bad task.json propagates out of the loop and + # aborts the whole resume BEFORE run_batch, so none of the `to_run` + # tasks execute either — the opposite of "one bad row never aborts". + prior: EvaluationResult | None = None try: - verify_reference_unchanged(prior, rt.task, rt.task_file) + prior = load_prior_result(rt.run_dir) + # The reference check lives inside regrade_in_place, so a caller + # cannot forget it. workspace = default_workspace(rt.run_dir, prior) # Preserve the ungraded record BEFORE the orchestrator overwrites # task.json in this same directory. @@ -710,12 +730,33 @@ async def _grade_resumed_tasks(to_grade: list[ResolvedTask]) -> list[tuple[Resol ) except (RegradeError, OSError, RuntimeError, ValueError) as e: console.print(f"[yellow]⚠[/] Could not grade {rt.task.task_id}: {e}") + if prior is None: + # The row could not even be read, so there is nothing to fold + # back. Skipping keeps it out of run.json exactly as it already + # is on disk, and the exit gate still fails the command because + # a task the resume owed a grade produced none. + failed_to_load.append(rt.task.task_id) + continue # Stamp the reason onto the row. Without it the failure survives only # in this console line: the folded-back result keeps the execute # phase's empty error_message, so run.json, the reports and CI show # an ungraded row with no explanation of why grading never happened. result = prior result.error_message = f"Grading failed during --resume: {e}" + else: + if result.final_status is FinalStatus.ERROR: + # An orchestrator-level grading crash is not a verdict about the + # run. Orchestrator.run() converts internal failures into a + # populated ERROR result rather than raising, so without this the + # `except` above never sees them and the ERROR row replaces a + # perfectly re-gradeable NOT_GRADED one — and ERROR is "complete" + # for both commands, so the row could never be graded again. + console.print( + f"[yellow]⚠[/] Grading {rt.task.task_id} errored ({result.error_message}); " + + "keeping the ungraded row so it stays re-gradeable." + ) + prior.error_message = f"Grading errored during --resume: {result.error_message}" + result = prior graded.append( ( rt, diff --git a/src/coder_eval/cli/run_helpers.py b/src/coder_eval/cli/run_helpers.py index 90324ca2..3c5b74bc 100644 --- a/src/coder_eval/cli/run_helpers.py +++ b/src/coder_eval/cli/run_helpers.py @@ -137,8 +137,17 @@ def print_execution_summary(run_dir: Path, summary: RunSummary) -> None: # happened instead, and keep the graded line for whatever WAS graded. if summary.tasks_not_graded: console.print(f"[bold]Results:[/bold] {summary.tasks_not_graded}/{summary.tasks_run} executed, not graded") - console.print("[dim]Grade later: uv run coder-eval evaluate [/dim]") - if summary.tasks_graded: + # Point at the run-dir form, not `evaluate `: the + # two-argument shape grades a bare directory with NO trajectory, so + # command_executed / skill_triggered / trajectory-reading judges score + # differently from what `run` would have produced. The run-dir form + # restores the trajectory AND the resolved config. + console.print(f"[dim]Grade later: uv run coder-eval run --run-dir {run_dir} --resume[/dim]") + console.print("[dim] or: uv run coder-eval evaluate ///00[/dim]") + if summary.tasks_graded or not summary.tasks_not_graded: + # The `or not ...` keeps the pre-existing "0/0 succeeded" line for an + # empty run: without it a run with no tasks at all prints no Results + # line whatsoever, since both counters are falsy. console.print(f"[bold]Results:[/bold] {summary.tasks_succeeded}/{summary.tasks_graded} succeeded") console.print(f"[dim]View report: open {run_dir / 'experiment.md'}[/dim]") console.print(f"[dim]View report: uv run coder-eval report {run_dir}[/dim]") diff --git a/src/coder_eval/cli/run_task_internal_command.py b/src/coder_eval/cli/run_task_internal_command.py index 8bc885b5..e3c4ad85 100644 --- a/src/coder_eval/cli/run_task_internal_command.py +++ b/src/coder_eval/cli/run_task_internal_command.py @@ -156,7 +156,15 @@ def _watch_host_heartbeat() -> None: # `coder-eval run` vs `coder-eval execute`, decided host-side. Defaults to # True (grade) so a host that predates `execute` — which never writes the # key — keeps its exact behavior. - grade: bool = context.get("grade", True) + # Coerced, not annotated: every other value crossing this boundary goes + # through a validating constructor, but `grade` was taken raw — so a + # hand-edited or older-format `"grade": "false"` arrives as a truthy str + # typed as bool and silently grades a run that asked not to be graded. + grade_raw = context.get("grade", True) + if not isinstance(grade_raw, bool): + typer.echo(f"FATAL: context.json 'grade' must be a boolean, got {grade_raw!r}", err=True) + raise typer.Exit(2) + grade: bool = grade_raw # Docker WORKDIR alignment: the host resolves the concrete WORKDIR # (config value / "auto" -> `docker inspect` / fallback) and forwards it here. # Absent -> None -> standard run_dir/artifacts workspace. diff --git a/src/coder_eval/models/experiment.py b/src/coder_eval/models/experiment.py index c399d7bf..d6b8745a 100644 --- a/src/coder_eval/models/experiment.py +++ b/src/coder_eval/models/experiment.py @@ -233,7 +233,10 @@ class VariantAggregate(BaseModel): # noqa: CE009 -- persisted result model; rou ge=0, description="Tasks executed without grading (`coder-eval execute`). Excluded from pass_rate entirely.", ) - average_score: float + # None when nothing in this variant was graded (`coder-eval execute`). + # A 0.0 here is indistinguishable from "measured and scored zero" — the same + # reason EvaluationResult.weighted_score is Optional. + average_score: float | None average_duration: float total_tokens: int | None = None replicate_count: int = Field( diff --git a/src/coder_eval/orchestration/experiment.py b/src/coder_eval/orchestration/experiment.py index 24d0effe..58cc3558 100644 --- a/src/coder_eval/orchestration/experiment.py +++ b/src/coder_eval/orchestration/experiment.py @@ -829,10 +829,10 @@ def _pick_worst_status(statuses: list[FinalStatus]) -> FinalStatus: return min(statuses, key=lambda s: priority.get(s.category, -1)) -def _mean_graded_score(vr_list: list[VariantResult]) -> float: - """Mean ``weighted_score`` over the graded rows; ``0.0`` when none were graded.""" +def _mean_graded_score(vr_list: list[VariantResult]) -> float | None: + """Mean ``weighted_score`` over the graded rows; ``None`` when none were graded.""" graded = [v.weighted_score for v in vr_list if v.weighted_score is not None] - return sum(graded) / len(graded) if graded else 0.0 + return sum(graded) / len(graded) if graded else None def _mean_reference_similarity(reps: list[TaskResult]) -> float | None: @@ -943,7 +943,7 @@ def aggregate_results( tasks_succeeded=0, tasks_failed=0, tasks_error=0, - average_score=0.0, + average_score=None, average_duration=0.0, ) continue @@ -959,10 +959,9 @@ def aggregate_results( tasks_not_graded=sum(1 for v in vr_list if v.final_status.category == "ungraded"), tasks_token_budget_exceeded=sum(1 for v in vr_list if v.final_status == FinalStatus.TOKEN_BUDGET_EXCEEDED), tasks_cost_budget_exceeded=sum(1 for v in vr_list if v.final_status == FinalStatus.COST_BUDGET_EXCEEDED), - # Mean over GRADED rows only. An ungraded row has no score (it - # arrives here as 0.0 because VariantResult.weighted_score is a - # plain float), so including it would report a clean execute run as - # average_score 0.0 — a number indistinguishable from "scored zero". + # Mean over GRADED rows only, and None when there are none: a clean + # execute run has no average score, and reporting 0.000 next to + # "Pass Rate: n/a" is a number indistinguishable from "scored zero". average_score=_mean_graded_score(vr_list), average_duration=sum(v.duration_seconds / v.replicate_count for v in vr_list) / len(vr_list), total_tokens=total_tokens, diff --git a/src/coder_eval/orchestration/regrade.py b/src/coder_eval/orchestration/regrade.py index 8eba319d..892cee0e 100644 --- a/src/coder_eval/orchestration/regrade.py +++ b/src/coder_eval/orchestration/regrade.py @@ -20,14 +20,19 @@ import logging from pathlib import Path -from coder_eval.models import EvaluationResult, PreservationMode, TaskConfigRecord, TaskDefinition +from coder_eval.models import ( + EvaluationResult, + PreservationMode, + SandboxConfig, + TaskConfigRecord, + TaskDefinition, +) +from coder_eval.path_utils import PRE_GRADE_JSON_FILENAME, TASK_JSON_FILENAME from coder_eval.sandbox import Sandbox logger = logging.getLogger(__name__) -TASK_JSON = "task.json" -PRE_GRADE_JSON = "task.execute.json" ARTIFACTS_DIRNAME = "artifacts" @@ -37,7 +42,7 @@ class RegradeError(Exception): def load_prior_result(run_dir: Path) -> EvaluationResult: """Read a finished run's ``task.json``.""" - path = run_dir / TASK_JSON + path = run_dir / TASK_JSON_FILENAME try: raw = path.read_text(encoding="utf-8") except OSError as e: @@ -64,7 +69,7 @@ def task_from_prior(prior: EvaluationResult, run_dir: Path) -> tuple[TaskDefinit record = prior.task_config if record is None: raise RegradeError( - f"{run_dir / TASK_JSON} carries no task_config, so the executed task cannot be " + f"{run_dir / TASK_JSON_FILENAME} carries no task_config, so the executed task cannot be " + "rebuilt. Pass the task file explicitly: coder-eval evaluate " ) try: @@ -104,7 +109,7 @@ def _fall_back_to_source(record: TaskConfigRecord, run_dir: Path, e: ValueError) if not record.source_file or not Path(record.source_file).is_file(): raise RegradeError( - f"The resolved task config in {run_dir / TASK_JSON} no longer validates ({e}), and " + f"The resolved task config in {run_dir / TASK_JSON_FILENAME} no longer validates ({e}), and " + "its source YAML is unavailable. Pass the task file explicitly." ) from e logger.warning( @@ -123,12 +128,26 @@ def default_workspace(run_dir: Path, prior: EvaluationResult) -> Path: """Locate the workspace a finished run left behind. ``sandbox_path`` is authoritative when it still exists — it is where the run - actually worked. Otherwise fall back to the preserved artifacts tree, whose - single child is named for the task. + actually worked. Otherwise fall back to the preserved artifacts tree, where + preservation nests the workspace under the task id. + + Raises rather than guessing when neither is conclusive. Guessing is worse + than failing here: grading the WRONG directory makes every path-relative + criterion fail as a locating artifact rather than as a verdict, and it + reports that as an ordinary score. """ if prior.sandbox_path: recorded = Path(prior.sandbox_path) if recorded.is_dir(): + if not _is_within(recorded, run_dir): + # An absolute path out of the run's own task.json, which is + # untrusted input for a shared run dir. Criteria execute with + # cwd there and may mutate it, so an out-of-tree location has to + # be the operator's explicit choice. + raise RegradeError( + f"The recorded sandbox_path ({recorded}) is outside the run directory " + + f"({run_dir}). Pass --workspace explicitly to grade it." + ) return recorded artifacts = run_dir / ARTIFACTS_DIRNAME @@ -138,10 +157,33 @@ def default_workspace(run_dir: Path, prior: EvaluationResult) -> Path: + f"({prior.sandbox_path or 'unset'}) is gone. The run was probably made with " + "--preservation-mode NONE." ) + # The exact path, not a heuristic. `task_id` may contain "/" (dataset rows + # are "/"), so "the single child of artifacts/" resolves one + # level too high for every row task. + by_task_id = artifacts / prior.task_id + if by_task_id.is_dir(): + return by_task_id + children = [p for p in sorted(artifacts.iterdir()) if p.is_dir()] - # Preservation nests the workspace under the task id; a flat artifacts dir - # (no subdirectory) means the workspace IS artifacts/. - return children[0] if len(children) == 1 else artifacts + if not children: + # A flat artifacts dir (no subdirectory) means the workspace IS artifacts/. + return artifacts + if len(children) == 1: + return children[0] + raise RegradeError( + f"Cannot tell which directory under {artifacts} is the workspace: no {prior.task_id!r} " + + f"child, and {len(children)} candidates ({', '.join(p.name for p in children)}). " + + "Pass --workspace explicitly." + ) + + +def _is_within(candidate: Path, root: Path) -> bool: + """True when ``candidate`` resolves inside ``root``.""" + try: + candidate.resolve().relative_to(root.resolve()) + except ValueError: + return False + return True def verify_reference_unchanged(prior: EvaluationResult, task: TaskDefinition, task_file: Path | None) -> None: @@ -198,9 +240,14 @@ def back_up_pre_grade_record(run_dir: Path) -> None: a second grade must not overwrite the ORIGINAL execute record with an already-graded one. """ - source, backup = run_dir / TASK_JSON, run_dir / PRE_GRADE_JSON + source, backup = run_dir / TASK_JSON_FILENAME, run_dir / PRE_GRADE_JSON_FILENAME if backup.exists() or not source.is_file(): return + if source.is_symlink() or backup.is_symlink(): + # Untrusted run dir: writing through a symlink would let a shared + # artifact clobber an arbitrary file the grading user can write. + logger.warning("Not preserving the pre-grade record: %s or %s is a symlink.", source, backup) + return try: backup.write_text(source.read_text(encoding="utf-8"), encoding="utf-8") except OSError as e: @@ -208,6 +255,21 @@ def back_up_pre_grade_record(run_dir: Path) -> None: logger.warning("Could not preserve the pre-grade record at %s: %s", backup, e) +def grading_sandbox_config(task: TaskDefinition) -> SandboxConfig: + """The sandbox config a grading pass runs under. + + Grading never runs a container: the docker driver dispatches through + DockerRunner, which needs an agent. Forcing ``tempdir`` keeps a task whose + YAML says ``driver: docker`` gradeable on the host. + + Re-validated rather than ``model_copy(update=...)``: ``update`` skips both + pydantic validation and pyright, so a typo would produce a SandboxConfig + violating its own ``Literal`` and surface much later at an unrelated + ``if driver == "docker"`` branch. + """ + return SandboxConfig.model_validate({**task.sandbox.model_dump(), "driver": "tempdir"}) + + async def regrade_in_place( *, task: TaskDefinition, @@ -232,12 +294,13 @@ async def regrade_in_place( """ from coder_eval.orchestrator import Orchestrator - # Grading never runs a container: the docker driver dispatches through - # DockerRunner, which needs an agent. Force tempdir so a task whose YAML says - # `driver: docker` is still gradeable on the host. - sandbox_config = task.sandbox.model_copy(deep=True).model_copy(update={"driver": "tempdir"}) + # Inside the shared entry point, not at each caller: a guard a caller has to + # remember is one a third caller will forget, and this one is the difference + # between a verdict and a verdict against the wrong answer key. + verify_reference_unchanged(prior, task, task_file) + sandbox = Sandbox( - sandbox_config, + grading_sandbox_config(task), task_id=task.task_id, task_dir=task_file.parent.resolve() if task_file is not None else None, ) diff --git a/src/coder_eval/orchestrator.py b/src/coder_eval/orchestrator.py index 1e5c2540..306f3155 100644 --- a/src/coder_eval/orchestrator.py +++ b/src/coder_eval/orchestrator.py @@ -67,7 +67,13 @@ from .orchestration.early_stop import EarlyStopWatcher, early_stop_active, validate_early_stop from .orchestration.evaluation import resolve_reference_dir, stage_reference_dir from .orchestration.run_limits import validate_run_limits -from .path_utils import digest_tree, format_task_log_id, rmtree_restrictive, task_log_path +from .path_utils import ( + digest_tree, + format_task_log_id, + rmtree_restrictive, + task_log_path, + write_text_atomic, +) from .sandbox import Sandbox from .simulation import DialogStopReason, SimulatorResult, UserSimulator, evaluate_stop from .streaming.callbacks import CompositeStreamCallback, StreamCallback, TaskScopedCallback, safe_emit @@ -763,6 +769,11 @@ def _seed_from_prior_result(self) -> None: # quietly wrong. _finalize_result restores the duration after its own # timing write; the grading pass's cost is recorded separately there. self.result.started_at = prior.started_at + # completed_at too, so the row's three time fields stay consistent with + # each other: leaving it at grading wall-clock produces a triple where + # completed_at - started_at != duration_seconds, which misleads anyone + # deriving a duration from the timestamps. + self.result.completed_at = prior.completed_at # The trajectory itself. Every derived figure in _finalize_result — # token totals, cost, command_stats, model_used, assistant turns — @@ -802,11 +813,25 @@ def _seed_from_prior_result(self) -> None: # environment_info: the prior run's capture describes the machine that # RAN the task (installed_tools, api route, coder_eval version). Ours # describes the machine grading it. Prior wins on conflict, and ours is - # preserved wholesale under `graded_by` rather than being interleaved — + # preserved as flat `graded_by_*` scalars rather than being interleaved — # a report that shows the grader's tool versions as the run's is worse # than one that shows neither. - graded_by = dict(self.result.environment_info) - self.result.environment_info = {**graded_by, **prior.environment_info, "graded_by": graded_by} + # Flattened to scalars rather than nested wholesale: environment_info is + # a flat map everywhere it is consumed (the HTML report `_esc`apes each + # value into a table cell; the evalboard types it as + # Record>), so a + # whole nested env capture renders as a Python dict repr. Only the three + # facts that identify the grading HOST are kept, and only when they + # differ from the run's. + grader = self.result.environment_info + provenance = { + f"graded_by_{key}": grader[key] + for key in ("coder_eval", "git_commit", "cli_version") + if key in grader and grader.get(key) != prior.environment_info.get(key) + } + # A second grade must not lose the first grader's stamp — merging prior + # over ours would otherwise clobber it and collapse the chain silently. + self.result.environment_info = {**grader, **prior.environment_info, **provenance} async def _run_evaluation_with_failure_evidence( self, @@ -1111,10 +1136,8 @@ def _finalize_result(self, start_time: float) -> None: # docker-driver host-heartbeat watchdog firing) would otherwise leave # a truncated task.json that the host parses as malformed-JSON rather # than as "no result", conflating two distinct failure modes. - import os as _os - - report_tmp = self.report_path.with_suffix(self.report_path.suffix + ".tmp") - report_tmp.write_text( # noqa: CE002 — small JSON write at end of run + write_text_atomic( # noqa: CE002 — small JSON write at end of run + self.report_path, self.result.model_dump_json( indent=2, # Strip inline transcripts: they live in sibling YAML files @@ -1123,9 +1146,7 @@ def _finalize_result(self, start_time: float) -> None: # in the row record without losing any data. exclude=TASK_JSON_TRANSCRIPT_EXCLUDE, ), - encoding="utf-8", ) - _os.replace(report_tmp, self.report_path) # Also emit an HTML trace/report alongside task.json. HTML failure must # never mask the underlying run outcome — write_task_html logs and diff --git a/src/coder_eval/path_utils.py b/src/coder_eval/path_utils.py index 44e47d91..c0a3b39f 100644 --- a/src/coder_eval/path_utils.py +++ b/src/coder_eval/path_utils.py @@ -15,6 +15,17 @@ TASK_LOG_FILENAME = "task.log" +# The per-task result record, and the pre-grade snapshot a detached grade keeps +# beside it. Module-level because ~12 sites name them — including three that +# `rglob` for the first — and two half-copies of the same string in different +# packages is how a rename becomes a silent no-op on the sites it missed. +TASK_JSON_FILENAME = "task.json" +PRE_GRADE_JSON_FILENAME = "task.execute.json" + +# The virtualenv directory `setup` creates and `adopt` discovers. Named because +# whether it is on PATH decides which binaries a criterion resolves. +VENV_DIRNAME = ".venv" + # Ignore list for every copy of a reference solution tree. A module-level # constant, not an inline literal at each call site: the host-side docker mount # (`DockerRunner._prepare_reference_mount`) and the per-run staged copy @@ -25,6 +36,22 @@ REFERENCE_COPY_IGNORE = [".git"] +def write_text_atomic(path: Path, text: str) -> None: + """Write ``text`` to ``path`` via a temp file + ``os.replace``. + + A plain ``write_text`` truncates first, so a SIGKILL or a full disk mid-write + leaves a half-file. For ``task.json`` that is worse than no file: a truncated + record parses as *malformed*, which the recovery paths treat as "not + complete" — so a later ``--resume`` re-executes the task and pays for the + agent again, and the row vanishes from ``run.json``. One writer, so the + orchestrator and the detached grade's write-back cannot have different crash + semantics for the same file. + """ + tmp = path.with_suffix(path.suffix + ".tmp") + tmp.write_text(text, encoding="utf-8") + os.replace(tmp, path) + + def digest_tree(root: Path) -> str: """Content hash of every file under ``root``, stable across runs. diff --git a/src/coder_eval/reports_experiment.py b/src/coder_eval/reports_experiment.py index ea1aceac..a10fb5f7 100644 --- a/src/coder_eval/reports_experiment.py +++ b/src/coder_eval/reports_experiment.py @@ -27,6 +27,7 @@ fmt_mean_sd, fmt_p, format_score, + is_env_table_key, load_variant_eval_results, paired_comparison, stddev, @@ -632,7 +633,7 @@ def generate_variant_report(variant_id: str, result: ExperimentResult, run_dir: # Denominator is the GRADED count, matching VariantAggregate.pass_rate — # an ungraded task was never measured and belongs on neither side. f"- **Pass Rate**: {pass_rate_str} ({agg.tasks_succeeded}/{agg.tasks_graded})", - f"- **Average Score**: {agg.average_score:.3f}", + f"- **Average Score**: {format_score(agg.average_score)}", f"- **Average Duration**: {agg.average_duration:.1f}s", f"- **Total Tokens**: {tokens_str}", ] @@ -723,7 +724,7 @@ def generate_variant_report(variant_id: str, result: ExperimentResult, run_dir: # Environment (from first result with data) for er in eval_results: if er.environment_info: - env = {k: v for k, v in er.environment_info.items() if k != "installed_tools"} + env = {k: v for k, v in er.environment_info.items() if is_env_table_key(k)} if env: lines.extend(["", "## Environment", ""]) for key, value in env.items(): diff --git a/src/coder_eval/reports_html.py b/src/coder_eval/reports_html.py index 128c24ed..39e7a600 100644 --- a/src/coder_eval/reports_html.py +++ b/src/coder_eval/reports_html.py @@ -20,7 +20,7 @@ from coder_eval.models import FinalStatus, eval_result_total_cost, sum_costs from .reports import early_stop_gate_note -from .reports_stats import format_score +from .reports_stats import format_score, is_env_table_key if TYPE_CHECKING: @@ -1074,8 +1074,8 @@ def _render_simulation(result: EvaluationResult) -> str: def _render_environment(result: EvaluationResult) -> str: - """Render Environment section (excluding installed_tools, which has its own).""" - env = {k: v for k, v in (result.environment_info or {}).items() if k != "installed_tools"} + """Render Environment section (excluding the keys with their own treatment).""" + env = {k: v for k, v in (result.environment_info or {}).items() if is_env_table_key(k)} if not env: return "" rows = "".join(f"{_esc(k)}{_esc(v)}" for k, v in env.items()) diff --git a/src/coder_eval/reports_stats.py b/src/coder_eval/reports_stats.py index c123764b..78306b5c 100644 --- a/src/coder_eval/reports_stats.py +++ b/src/coder_eval/reports_stats.py @@ -310,6 +310,19 @@ class VariantSeries(NamedTuple): asst_turns: list[float] +# environment_info keys the Environment table must NOT render as ordinary rows. +# `installed_tools` has its own dedicated section; the rest are harness +# bookkeeping the reader did not ask for — `command_base_path` is a full PATH +# string on every row, and the graded_by_* provenance keys only appear on a +# re-graded row where they would read as facts about the run itself. +ENV_TABLE_EXCLUDE = frozenset({"installed_tools", "command_base_path", "reference_digest"}) + + +def is_env_table_key(key: str) -> bool: + """Whether ``key`` belongs in a rendered Environment table.""" + return key not in ENV_TABLE_EXCLUDE and not key.startswith("graded_by_") + + # What an ungraded row shows where a score would go. Deliberately not "0.000": # an ungraded task was never measured, and a zero is indistinguishable from a # task that was measured and scored nothing. diff --git a/src/coder_eval/sandbox.py b/src/coder_eval/sandbox.py index 51734c69..b77bdb60 100644 --- a/src/coder_eval/sandbox.py +++ b/src/coder_eval/sandbox.py @@ -23,6 +23,7 @@ StarterFilesSource, TemplateDirSource, ) +from .path_utils import VENV_DIRNAME from .resources import get_ignore_patterns, should_ignore_path @@ -298,6 +299,11 @@ def adopt(self, workspace: Path) -> Path: copy path they see the copy's paths, not the ones the agent worked at. * Copying a real workspace costs minutes. + "Materializing nothing" means it writes no FILES. It does still chmod + ``+x`` over the task's declared mock-PATH directories inside the tree — + a mode change the criteria need in order to resolve the same shimmed + binaries the agent did. + The caller keeps ownership: ``_cleanup_on_exit`` stays False, so ``cleanup()`` never deletes an adopted directory. Criteria CAN still mutate it (a ``run_command`` that writes), which is why the copy path @@ -341,9 +347,17 @@ def adopt(self, workspace: Path) -> Path: # Discover an existing venv instead of creating one, so `run_command` # criteria get the same VIRTUAL_ENV/PATH the agent had. Absent venv -> # None, exactly as for a task with no python config. - candidate = self.sandbox_dir / ".venv" - if candidate.is_dir(): - self.venv_dir = candidate + # + # Gated on `config.python` for the same reason `setup` is: venv_dir + # prepends the venv's bin/ to PATH and exports VIRTUAL_ENV for every + # criterion subprocess, so discovering one a task never asked for grades + # it under a PATH it never ran under — the exact divergence the + # command_base_path round trip exists to close. It would also let an + # agent shadow binaries by writing `.venv/bin/` into its own workspace. + if self.config.python: + candidate = self.sandbox_dir / VENV_DIRNAME + if candidate.is_dir(): + self.venv_dir = candidate self._check_parent_node_modules_contamination() self._refresh_plugin_tools_dir() @@ -800,7 +814,7 @@ def _setup_virtualenv(self) -> None: if not self.sandbox_dir: raise RuntimeError("Sandbox directory not initialized") - self.venv_dir = self.sandbox_dir / ".venv" + self.venv_dir = self.sandbox_dir / VENV_DIRNAME # Use uv to create virtual environment (faster than venv) try: diff --git a/tests/lint/rules/ce047_env_info_key_round_trip.py b/tests/lint/rules/ce047_env_info_key_round_trip.py new file mode 100644 index 00000000..1afe6281 --- /dev/null +++ b/tests/lint/rules/ce047_env_info_key_round_trip.py @@ -0,0 +1,122 @@ +"""CE047: every ``environment_info`` key that is READ must also be WRITTEN. + +``EvaluationResult.environment_info`` is a ``dict[str, Any]`` bag, so nothing — +not pydantic, not pyright — connects the site that writes a key to the site that +reads it back. A reader whose writer was never added (or was later removed) is +silently inert: ``.get("k")`` returns ``None``, the guard takes its early return, +and the feature reports success while doing nothing. + +The motivating case: ``verify_reference_unchanged`` read +``environment_info.get("reference_digest")`` to refuse a re-grade whose answer key +had changed. Nothing anywhere wrote that key — a whole-tree grep found exactly one +occurrence, the read itself. The anti-cheat guard shipped, was documented in +CLAUDE.md and the user guide as protection, and never fired once. Every automated +gate in the repo was green. + +This is deliberately a one-way check. An unread key is ordinary (recorded for a +human or a downstream consumer); an unwritten key is always a bug. + +Use ``# noqa: CE047`` for a key genuinely supplied from outside this repo. +""" + +import ast +import re +from pathlib import Path + +from tests.lint.rules.base import BaseRule + + +_SRC_ROOT = Path("src/coder_eval") + +# Keys written by a consumer outside src/ (the docker container's own capture, +# a plugin) or copied wholesale from another dict. Each needs a reason. +_EXTERNALLY_WRITTEN: dict[str, str] = {} + + +def _written_keys() -> set[str]: + """Every string literal assigned into an ``environment_info`` subscript. + + Text-scanned rather than AST-walked across the tree so a write inside any + module counts regardless of how the dict was reached (``self.result.``, + ``result.``, a local alias). The rule only needs to know a literal is + written SOMEWHERE — attributing it precisely would add false positives + without catching anything more. + """ + written: set[str] = set() + pattern = re.compile(r"""environment_info\[\s*["']([\w.-]+)["']\s*\]\s*=""") + # Also count keys named in a dict literal that becomes environment_info, and + # the f-string-built provenance keys (`f"graded_by_{key}"`), which no literal + # scan can resolve — those are covered by the prefix allowance below. + for path in sorted(_SRC_ROOT.rglob("*.py")): + try: + text = path.read_text(encoding="utf-8") + except OSError: # pragma: no cover - unreadable file in src is not our problem + continue + written.update(pattern.findall(text)) + return written + + +class EnvInfoKeyRoundTrip(BaseRule): + id = "CE047" + + _SRC_PATH = re.compile(r"[/\\]src[/\\]coder_eval[/\\]") + _written: set[str] | None = None + + def __init__(self, filepath: str) -> None: + super().__init__(filepath) + self._in_scope = bool(self._SRC_PATH.search(filepath)) + if self._in_scope and EnvInfoKeyRoundTrip._written is None: + EnvInfoKeyRoundTrip._written = _written_keys() + + def visit_Call(self, node: ast.Call) -> None: + self._check_get(node) + self.generic_visit(node) + + def visit_Subscript(self, node: ast.Subscript) -> None: + self._check_subscript(node) + self.generic_visit(node) + + def _check_get(self, node: ast.Call) -> None: + """``<...>.environment_info.get("key")``.""" + func = node.func + if not isinstance(func, ast.Attribute) or func.attr != "get": + return + if not _is_env_info(func.value) or not node.args: + return + self._require_writer(node, node.args[0]) + + def _check_subscript(self, node: ast.Subscript) -> None: + """``<...>.environment_info["key"]`` in a READ position. + + A write is an ``ast.Store`` context, which is exactly what makes it a + writer — only loads are checked. + """ + if not isinstance(node.ctx, ast.Load) or not _is_env_info(node.value): + return + self._require_writer(node, node.slice) + + def _require_writer(self, node: ast.AST, key_node: ast.AST) -> None: + if not self._in_scope: + return + if not isinstance(key_node, ast.Constant) or not isinstance(key_node.value, str): + return # a computed key; nothing to resolve statically + key = key_node.value + if key in _EXTERNALLY_WRITTEN or key.startswith("graded_by_"): + # graded_by_* keys are built with an f-string from a name list, so no + # literal write exists to find. + return + if key in (EnvInfoKeyRoundTrip._written or set()): + return + self.violation( + node, + f"environment_info key {key!r} is read here but never written anywhere in src/coder_eval. " + + "A reader with no writer is silently inert — it returns None, the guard takes its early " + + "return, and the feature reports success while doing nothing (see CE047's docstring for " + + "the anti-cheat guard that shipped this way). Add the write, or list the key in " + + "_EXTERNALLY_WRITTEN with the out-of-tree producer that supplies it.", + ) + + +def _is_env_info(node: ast.AST) -> bool: + """Whether ``node`` is an ``…​.environment_info`` attribute access.""" + return isinstance(node, ast.Attribute) and node.attr == "environment_info" diff --git a/tests/lint/rules/ce048_no_in_process_typer_command_call.py b/tests/lint/rules/ce048_no_in_process_typer_command_call.py new file mode 100644 index 00000000..37c10a2f --- /dev/null +++ b/tests/lint/rules/ce048_no_in_process_typer_command_call.py @@ -0,0 +1,106 @@ +"""CE048: never call a Typer command function in process. + +Typer builds a command's parser from its signature, so every parameter's default +is an ``OptionInfo`` / ``ArgumentInfo`` sentinel, not the value it stands for. +Click substitutes the real defaults when it *invokes* the command; a direct +Python call does not — every unspecified argument arrives as a truthy sentinel +object. + +The failure is silent, which is what makes it worth a rule. ``evaluate``'s +``in_place: bool | None = typer.Option(None, "--in-place/--copy")`` reads as "no +preference" and selects copy-vs-in-place from the target shape; called +in-process, ``in_place`` was an ``OptionInfo``, which is truthy, so the tests +silently graded in place and the default they meant to cover was never +exercised. Nothing failed — the wrong branch simply ran. + +The fix is the one already applied to ``run`` / ``execute`` / ``evaluate``: keep +the Typer signature as a thin wrapper and put the body in a plain function with +real Python defaults (``run_pipeline``, ``run_evaluation``). Call THAT. + +Use ``# noqa: CE048`` only where the sentinel behavior is itself under test. +""" + +import ast +import re +from pathlib import Path + +from tests.lint.rules.base import BaseRule + + +_CLI_ROOT = Path("src/coder_eval/cli") + + +def _typer_command_names() -> set[str]: + """Functions whose signature is a Typer parser — i.e. whose parameters carry + ``typer.Option`` / ``typer.Argument`` defaults. + + Detected by the defaults rather than by the ``app.command(...)`` registration + site, because registration happens in ``cli/__init__.py`` by reference and a + command that is merely *about* to be registered has the same hazard. + """ + names: set[str] = set() + for path in sorted(_CLI_ROOT.rglob("*.py")): + try: + tree = ast.parse(path.read_text(encoding="utf-8")) + except (OSError, SyntaxError): # pragma: no cover - unparseable file in cli/ fails elsewhere + continue + for node in ast.walk(tree): + if not isinstance(node, ast.FunctionDef | ast.AsyncFunctionDef): + continue + if any(_is_typer_param(default) for default in node.args.defaults): + names.add(node.name) + return names + + +def _is_typer_param(node: ast.expr) -> bool: + """``typer.Option(...)`` / ``typer.Argument(...)`` as a parameter default.""" + if not isinstance(node, ast.Call): + return False + func = node.func + return isinstance(func, ast.Attribute) and func.attr in {"Option", "Argument"} + + +class NoInProcessTyperCommandCall(BaseRule): + id = "CE048" + + # The registration site itself hands these to Typer by reference; and the + # module that defines a command may call its own sibling. + _EXEMPT_FILES = re.compile(r"[/\\]src[/\\]coder_eval[/\\]cli[/\\]__init__\.py$") + _commands: set[str] | None = None + + def __init__(self, filepath: str) -> None: + super().__init__(filepath) + self._in_scope = not self._EXEMPT_FILES.search(filepath) + self._imported_from_cli: set[str] = set() + if NoInProcessTyperCommandCall._commands is None: + NoInProcessTyperCommandCall._commands = _typer_command_names() + + def visit_ImportFrom(self, node: ast.ImportFrom) -> None: + # Only names imported FROM a cli module count. A command name is not + # unique in the tree — `run_command` is both a Typer command and + # `Sandbox.run_command` — so matching on the bare name alone would flag + # every criterion that shells out. + if (node.module and "coder_eval.cli" in node.module) or (node.level and node.module == "cli"): + self._imported_from_cli.update(alias.asname or alias.name for alias in node.names) + self.generic_visit(node) + + def visit_Call(self, node: ast.Call) -> None: + self._check(node) + self.generic_visit(node) + + def _check(self, node: ast.Call) -> None: + if not self._in_scope or not isinstance(node.func, ast.Name): + return + name = node.func.id + if name not in self._imported_from_cli: + return + if name not in (NoInProcessTyperCommandCall._commands or set()): + return + self.violation( + node, + f"'{name}' is a Typer command: its parameter defaults are OptionInfo sentinels, not values, " + + "so calling it in process hands every unspecified argument a truthy placeholder and " + + "silently runs the wrong branch. Call the plain-function body instead (the " + + "run_pipeline / run_evaluation split exists for this), or drive it through " + + "typer.testing.CliRunner.", + ) diff --git a/tests/lint/runner.py b/tests/lint/runner.py index 092e97a6..383c3064 100644 --- a/tests/lint/runner.py +++ b/tests/lint/runner.py @@ -26,6 +26,8 @@ from tests.lint.rules.ce039_config_error_escalates import ConfigErrorEscalates from tests.lint.rules.ce043_no_command_output_truncation import NoCommandOutputTruncation from tests.lint.rules.ce046_env_info_spreads_super import EnvInfoSpreadsSuper +from tests.lint.rules.ce047_env_info_key_round_trip import EnvInfoKeyRoundTrip +from tests.lint.rules.ce048_no_in_process_typer_command_call import NoInProcessTyperCommandCall from tests.lint.rules.no_agent_timing_access import NoAgentTimingAccess from tests.lint.rules.no_blocking_io_in_async import NoBlockingIoInAsync from tests.lint.rules.no_cli_imports_in_core import NoCliImportsInCore @@ -75,6 +77,8 @@ ConfigErrorEscalates, NoCommandOutputTruncation, EnvInfoSpreadsSuper, + EnvInfoKeyRoundTrip, + NoInProcessTyperCommandCall, ] # Anti-shadow invariant (mirrors AgentRegistry / register_pricing): every CE rule diff --git a/tests/test_custom_lint.py b/tests/test_custom_lint.py index 64508fe7..8329d629 100644 --- a/tests/test_custom_lint.py +++ b/tests/test_custom_lint.py @@ -23,13 +23,21 @@ SRC = Path(__file__).parent.parent / "src" +# Rules whose defect class lives in the TEST tree, not in src/. CE048's whole +# subject is an in-process call to a Typer command, and the only place that +# happens is a test — scanning src/ alone would leave the rule permanently green +# while the bug it exists for sat five lines away. +_ALSO_SCAN_TESTS = {"CE048"} + + @pytest.mark.lint @pytest.mark.parametrize("rule_class", ALL_RULES, ids=[r.id for r in ALL_RULES]) def test_no_violations(rule_class: type) -> None: import sys mod_doc = (getattr(sys.modules.get(rule_class.__module__), "__doc__", "") or "").splitlines()[0].strip() - violations = check_paths([SRC], rules=[rule_class]) + paths = [SRC, Path(__file__).parent] if rule_class.id in _ALSO_SCAN_TESTS else [SRC] + violations = check_paths(paths, rules=[rule_class]) assert not violations, ( f"\n{len(violations)} violation(s) for {rule_class.id} ({mod_doc}):\n\n" + "\n".join(f" {v}" for v in violations) diff --git a/tests/test_detached_grading_guards.py b/tests/test_detached_grading_guards.py new file mode 100644 index 00000000..2ff234cf --- /dev/null +++ b/tests/test_detached_grading_guards.py @@ -0,0 +1,261 @@ +"""The guards around detached grading, each tested on the branch that fires. + +Every case here is a refusal, a skip, or a mode selection — the branches that +exist precisely because taking the other one would produce a plausible number +that is wrong. They were all shipped with coverage on the happy path only. +""" + +from __future__ import annotations + +from datetime import datetime +from pathlib import Path +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest +from typer.testing import CliRunner + +from coder_eval.cli import app +from coder_eval.cli.evaluate_command import run_evaluation +from coder_eval.models import ( + AgentKind, + EvaluationResult, + FileExistsCriterion, + FinalStatus, + TaskDefinition, + parse_agent_config, +) +from coder_eval.orchestrator import Orchestrator + + +runner = CliRunner() + + +# An agentless task: `type: none` runs no agent and forbids `initial_prompt` +# (there is nothing to read it), which is what makes it usable with no API key. +_AGENTLESS = """task_id: t +description: d +agent: + type: none +success_criteria: + - type: file_exists + path: proof.txt + description: x +""" + +# A simulation task needs a real agent type — the refusal under `execute` fires +# during resolution, so the agent is never created. +_SIMULATED = """task_id: t +description: d +initial_prompt: p +agent: + type: claude-code +simulation: + enabled: true + persona: a user + goal: get it done +success_criteria: + - type: file_exists + path: proof.txt + description: x +""" + + +def _task(tmp_path: Path, *, simulation: bool = False) -> Path: + path = tmp_path / "t.yaml" + path.write_text(_SIMULATED if simulation else _AGENTLESS, encoding="utf-8") + return path + + +# -------------------------------------------------------------------------- +# `execute` refuses simulation tasks +# -------------------------------------------------------------------------- + + +def test_execute_refuses_a_simulation_task_by_name(tmp_path: Path) -> None: + """The dialog loop reads criteria results to decide whether to keep talking, + so an ungraded dialog would silently change its own stopping behavior. The + refusal must name the task, or a user cannot tell which one to remove.""" + result = runner.invoke(app, ["execute", str(_task(tmp_path, simulation=True)), "--run-dir", str(tmp_path / "r")]) + + assert result.exit_code != 0 + assert "simulation" in result.output.lower() + assert "t" in result.output + + +def test_run_still_accepts_the_same_simulation_task(tmp_path: Path) -> None: + """The control: the refusal is about `execute`, not about the task.""" + task = _task(tmp_path, simulation=True) + with patch("coder_eval.cli.run_command._run_with_experiment", new=AsyncMock(return_value=(MagicMock(), 0))): + result = runner.invoke(app, ["run", str(task), "--run-dir", str(tmp_path / "r")]) + assert "does not support simulation" not in result.output + + +# -------------------------------------------------------------------------- +# The evaluate-only path refuses grade=False +# -------------------------------------------------------------------------- + + +async def test_grading_off_on_the_evaluate_only_path_is_refused(tmp_path: Path) -> None: + """No agent AND no grading is a no-op that would still write a task.json. + Refusing beats producing an empty row that looks like a result.""" + task = TaskDefinition( + task_id="t", + description="d", + initial_prompt="p", + agent=parse_agent_config(type=AgentKind.CLAUDE_CODE), + success_criteria=[FileExistsCriterion(path="x.txt", description="x")], + ) + orch = Orchestrator(task=task, run_dir=tmp_path, variant_id="v", grade=False) + orch.success_checker = MagicMock() + orch.result = EvaluationResult( + task_id="t", + task_description="d", + variant_id="v", + agent_type=AgentKind.CLAUDE_CODE, + started_at=datetime(2026, 1, 1), + final_status=FinalStatus.FAILURE, + iteration_count=0, + ) + + with pytest.raises(ValueError, match="meaningless on the evaluate-only path"): + await orch._evaluation_loop() + + +# -------------------------------------------------------------------------- +# --in-place / --copy +# -------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + ("flag", "expect_adopt"), + [(None, False), ("--in-place", True), ("--copy", False)], + ids=["default-for-a-work-dir-is-copy", "explicit-in-place", "explicit-copy"], +) +def test_the_flag_decides_adopt_versus_setup_on_a_work_dir( + tmp_path: Path, flag: str | None, expect_adopt: bool +) -> None: + """The choice is not cosmetic: the copy path filters node_modules / dist / + build / .venv, so a criterion reading those fails as a copying artifact.""" + work = tmp_path / "work" + work.mkdir() + (work / "proof.txt").write_text("x", encoding="utf-8") + # --run-dir is not incidental: without it the grade lands in a repo-relative + # runs//, which several xdist workers race over. + args = ["evaluate", str(_task(tmp_path)), str(work), "--run-dir", str(tmp_path / "r")] + if flag: + args.append(flag) + + with ( + patch("coder_eval.sandbox.Sandbox.adopt") as adopt, + patch("coder_eval.sandbox.Sandbox.setup") as setup, + ): + runner.invoke(app, args) + + assert adopt.called is expect_adopt + assert setup.called is not expect_adopt + + +def test_run_evaluation_has_real_defaults_not_typer_sentinels(tmp_path: Path) -> None: + """`run_evaluation` exists because calling the Typer command in-process hands + every unspecified option an `OptionInfo` — and `in_place=None` became truthy, + silently flipping the copy default to in-place.""" + import inspect + + sig = inspect.signature(run_evaluation) + for name in ("work_dir", "workspace", "in_place", "run_dir"): + assert sig.parameters[name].default is None, f"{name} must default to a real None" + assert sig.parameters["preserve"].default is True + + +# -------------------------------------------------------------------------- +# The PATH round trip +# -------------------------------------------------------------------------- + + +def test_the_agents_path_is_persisted_so_a_later_grade_can_restore_it(tmp_path: Path) -> None: + """Without the persisted value a detached grade resolves `run_command` + binaries against ambient PATH and can disagree with the run it grades.""" + task = TaskDefinition( + task_id="t", + description="d", + initial_prompt="p", + agent=parse_agent_config(type=AgentKind.CLAUDE_CODE), + success_criteria=[FileExistsCriterion(path="x.txt", description="x")], + ) + orch = Orchestrator(task=task, run_dir=tmp_path, variant_id="v") + orch.result = EvaluationResult( + task_id="t", + task_description="d", + variant_id="v", + agent_type=AgentKind.CLAUDE_CODE, + started_at=datetime(2026, 1, 1), + final_status=FinalStatus.FAILURE, + iteration_count=0, + ) + orch.sandbox = MagicMock() + orch.agent = MagicMock() + orch.agent.get_sdk_options.return_value = {"env": {"PATH": f"{tmp_path}:/usr/bin"}} + + orch._sync_sandbox_command_path_with_agent() + + assert "command_base_path" in orch.result.environment_info + + +def test_a_restored_path_drops_entries_inside_the_graded_workspace(tmp_path: Path) -> None: + """The restored value is PREPENDED ahead of the host PATH and comes out of the + run's own task.json. An entry inside the agent-writable workspace could + shadow a real tool on the grader's host.""" + workspace = tmp_path / "ws" + (workspace / "bin").mkdir(parents=True) + outside = tmp_path / "toolchain" + outside.mkdir() + + task = TaskDefinition( + task_id="t", + description="d", + initial_prompt="p", + agent=parse_agent_config(type=AgentKind.CLAUDE_CODE), + success_criteria=[FileExistsCriterion(path="x.txt", description="x")], + ) + orch = Orchestrator(task=task, run_dir=tmp_path, variant_id="v") + orch.sandbox = MagicMock() + orch.sandbox.sandbox_dir = workspace + + kept = orch._sanitize_restored_path(f"{workspace / 'bin'}:{outside}:{tmp_path / 'gone'}") + + assert str(outside.resolve()) in kept + assert str(workspace) not in kept, "an entry inside the graded tree must be dropped" + assert "gone" not in kept, "a non-existent entry buys no parity" + + +# -------------------------------------------------------------------------- +# The LiteLLM cost join +# -------------------------------------------------------------------------- + + +def test_the_actual_cost_join_is_skipped_on_a_re_grade(tmp_path: Path) -> None: + """The join keys on a per-Orchestrator nonce the prior turns never carried, + so running it on a re-grade would clobber already-correct per-turn costs.""" + task = TaskDefinition( + task_id="t", + description="d", + initial_prompt="p", + agent=parse_agent_config(type=AgentKind.CLAUDE_CODE), + success_criteria=[FileExistsCriterion(path="x.txt", description="x")], + ) + prior = EvaluationResult( + task_id="t", + task_description="d", + variant_id="v", + agent_type=AgentKind.CLAUDE_CODE, + started_at=datetime(2026, 1, 1), + final_status=FinalStatus.NOT_GRADED, + iteration_count=0, + ) + orch = Orchestrator(task=task, run_dir=tmp_path, variant_id="v", prior_result=prior) + orch.result = prior + + with patch("coder_eval.litellm_cost.apply_actual_cost") as apply: + orch._join_litellm_actual_cost() + + apply.assert_not_called() diff --git a/tests/test_early_stop.py b/tests/test_early_stop.py index 5f647392..ff57c63d 100644 --- a/tests/test_early_stop.py +++ b/tests/test_early_stop.py @@ -38,7 +38,7 @@ from coder_eval.agents.claude_code_agent import ClaudeCodeAgent from coder_eval.agents.codex_agent import CodexAgent, _CodexTurnState from coder_eval.agents.registry import AgentRegistry -from coder_eval.cli.plan_command import plan_command +from coder_eval.cli.plan_command import run_plan from coder_eval.config import settings from coder_eval.criteria import CriterionRegistry, init_criteria from coder_eval.criteria.command_executed import CommandExecutedChecker @@ -1041,7 +1041,7 @@ def test_run_surface_variant_inherits_task_threshold_with_stop_early_false(self, assert early_stop_active(by_variant["smoke"]) is True def _run_plan(self, task_file: Path, exp_dir: Path) -> tuple[str, int]: - """Invoke the real plan_command against a minimal single-variant experiment. + """Invoke the real plan body against a minimal single-variant experiment. Returns the concatenated console output and the exit code (0 when plan returned normally). @@ -1058,7 +1058,7 @@ def _run_plan(self, task_file: Path, exp_dir: Path) -> tuple[str, int]: patch("coder_eval.cli.plan_command.console") as mock_console, ): try: - plan_command(task_files=[task_file], experiment=exp_file) + run_plan(task_files=[task_file], experiment=exp_file) except typer.Exit as exc: exit_code = exc.exit_code printed = " ".join(str(call) for call in mock_console.print.call_args_list) diff --git a/tests/test_execute_command.py b/tests/test_execute_command.py index be27c2a8..c0189b54 100644 --- a/tests/test_execute_command.py +++ b/tests/test_execute_command.py @@ -213,7 +213,7 @@ def _option_names(command: str) -> set[str]: } -def test_execute_exposes_run_flags_minus_the_two_refused_ones() -> None: +def test_execute_exposes_run_flags_minus_the_refused_one() -> None: run_opts = _option_names("run") execute_opts = _option_names("execute") @@ -282,10 +282,10 @@ def test_container_defaults_to_grading_when_the_host_sends_no_key() -> None: def test_execute_help_explains_the_refused_flags() -> None: - """The two omissions are documented in the help, not silently absent — a user - who reaches for `--resume` needs to learn why it is refused, not just that it + """The omission is documented in the help, not silently absent — a user who + reaches for `--junit-xml` needs to learn why it is refused, not just that it is unrecognised. (Presence as a real *flag* is covered by the option-set test - above; here we only require the help text to mention them.)""" + above; here we only require the help text to mention it.)""" result = runner.invoke(app, ["execute", "--help"]) assert result.exit_code == 0 for flag in _DELIBERATELY_ABSENT_FROM_EXECUTE: diff --git a/tests/test_execute_evaluate_loop.py b/tests/test_execute_evaluate_loop.py index 96c0b415..cd6f566d 100644 --- a/tests/test_execute_evaluate_loop.py +++ b/tests/test_execute_evaluate_loop.py @@ -14,6 +14,7 @@ import shutil from pathlib import Path from typing import Any +from unittest.mock import AsyncMock, patch import pytest from typer.testing import CliRunner @@ -262,3 +263,98 @@ def test_execute_to_run_resume_emits_no_config_drift_warning(tmp_path: Path) -> result = _invoke(["run", str(AGENTLESS_TASK), "--run-dir", str(run_dir), "--resume"]) assert "run config changed" not in result.output + + +def test_run_resume_keeps_the_row_regradeable_when_grading_crashes(tmp_path: Path) -> None: + """A grading crash is not a verdict about the run. Folding the ORIGINAL + ungraded row back keeps the task re-gradeable — writing ERROR over it would + not, since ERROR is "complete" for both commands.""" + run_dir = tmp_path / "r" + _invoke(["execute", str(AGENTLESS_TASK), "--run-dir", str(run_dir)]) + + with patch( + "coder_eval.orchestration.regrade.regrade_in_place", + new=AsyncMock(side_effect=RuntimeError("checker exploded")), + ): + result = runner.invoke(app, ["run", str(AGENTLESS_TASK), "--run-dir", str(run_dir), "--resume"]) + + assert result.exit_code != 0, "a resume that graded nothing must not report success" + assert _row(_task_dir(run_dir))["final_status"] == FinalStatus.NOT_GRADED.value + # The reason is durable, not console-only. It lands in run.json rather than + # task.json: task.json stays the pristine execute record, which is what keeps + # the row re-gradeable below. + summary = json.loads((run_dir / "run.json").read_text(encoding="utf-8")) + assert "checker exploded" in str(summary["task_results"]) + + # And the row really is still re-gradeable. + _invoke(["run", str(AGENTLESS_TASK), "--run-dir", str(run_dir), "--resume"]) + assert _row(_task_dir(run_dir))["final_status"] == FinalStatus.SUCCESS.value + + +def test_run_resume_reports_a_failing_verdict_and_exits_non_zero(tmp_path: Path) -> None: + """The other resume gate: grading that SUCCEEDS but fails the criteria.""" + run_dir = tmp_path / "r" + _invoke(["execute", str(AGENTLESS_TASK), "--run-dir", str(run_dir)]) + # Remove the file the criteria read, so the grade legitimately fails. + for proof in run_dir.glob("**/artifacts/**/proof.txt"): + proof.unlink() + + result = runner.invoke(app, ["run", str(AGENTLESS_TASK), "--run-dir", str(run_dir), "--resume"]) + + assert result.exit_code != 0 + assert _row(_task_dir(run_dir))["final_status"] == FinalStatus.FAILURE.value + summary = json.loads((run_dir / "run.json").read_text(encoding="utf-8")) + assert summary["tasks_failed"] == 1 + assert summary["tasks_not_graded"] == 0 + + +def test_evaluate_grades_the_directory_named_by_workspace(tmp_path: Path) -> None: + """--workspace exists for a verifier that built its own /app; nothing else + asserted it actually grades that directory rather than the run's artifacts.""" + run_dir = tmp_path / "r" + _invoke(["execute", str(AGENTLESS_TASK), "--run-dir", str(run_dir)]) + elsewhere = tmp_path / "elsewhere" + elsewhere.mkdir() + (elsewhere / "proof.txt").write_text("coder-eval-ran-without-a-coder", encoding="utf-8") + # Make the run's own artifacts FAIL, so a pass can only come from --workspace. + for proof in run_dir.glob("**/artifacts/**/proof.txt"): + proof.unlink() + + _invoke(["evaluate", str(_task_dir(run_dir)), "--workspace", str(elsewhere)]) + + assert _row(_task_dir(run_dir))["final_status"] == FinalStatus.SUCCESS.value + + +def test_evaluate_refuses_to_re_grade_a_run_that_errored(tmp_path: Path) -> None: + """Grading may only move NOT_GRADED to a verdict. An ERROR / TIMEOUT run is + an execution fact this pass neither repeated nor observed — laundering it + into SUCCESS would report a crashed run as a pass.""" + run_dir = tmp_path / "r" + _invoke(["execute", str(AGENTLESS_TASK), "--run-dir", str(run_dir)]) + task_dir = _task_dir(run_dir) + row = _row(task_dir) + row["final_status"] = FinalStatus.TIMEOUT.value + (task_dir / "task.json").write_text(json.dumps(row), encoding="utf-8") + + _invoke(["evaluate", str(task_dir)]) + + assert _row(task_dir)["final_status"] == FinalStatus.TIMEOUT.value + + +def test_grading_the_same_run_twice_reaches_the_same_verdict(tmp_path: Path) -> None: + """Idempotence. A second grade must see the same workspace the first did — + it catches both a pre_run that mutated the tree and a lost sandbox_path.""" + run_dir = tmp_path / "r" + _invoke(["execute", str(AGENTLESS_TASK), "--run-dir", str(run_dir)]) + task_dir = _task_dir(run_dir) + + _invoke(["evaluate", str(task_dir)]) + first = _row(task_dir) + _invoke(["evaluate", str(task_dir)]) + second = _row(task_dir) + + assert second["final_status"] == first["final_status"] + assert second["weighted_score"] == first["weighted_score"] + assert second["sandbox_path"] == first["sandbox_path"], "the artifacts pointer must survive a re-grade" + # The pre-grade record is still the ORIGINAL ungraded one, not the first grade's. + assert _row(task_dir, "task.execute.json")["final_status"] == FinalStatus.NOT_GRADED.value diff --git a/tests/test_plan_command.py b/tests/test_plan_command.py index 16a8d663..f8796fb9 100644 --- a/tests/test_plan_command.py +++ b/tests/test_plan_command.py @@ -6,7 +6,7 @@ import pytest import typer -from coder_eval.cli.plan_command import plan_command +from coder_eval.cli.plan_command import run_plan from coder_eval.models import ( AgentConfig, ExperimentDefinition, @@ -73,7 +73,7 @@ def test_plan_shows_na_when_agent_is_none(self, tmp_path: Path) -> None: patch(f"{_EXP}.resolve_task_for_variant", return_value=(resolved_task, {}, 1)), patch("coder_eval.cli.plan_command.console") as mock_console, ): - plan_command(task_files=[task_file]) + run_plan(task_files=[task_file]) printed = [str(call) for call in mock_console.print.call_args_list] agent_lines = [p for p in printed if "Agent:" in p] @@ -98,7 +98,7 @@ def test_plan_shows_agent_type_when_present(self, tmp_path: Path) -> None: patch(f"{_EXP}.resolve_task_for_variant", return_value=(resolved_task, {}, 1)), patch("coder_eval.cli.plan_command.console") as mock_console, ): - plan_command(task_files=[task_file]) + run_plan(task_files=[task_file]) printed = [str(call) for call in mock_console.print.call_args_list] agent_lines = [p for p in printed if "Agent:" in p] @@ -124,7 +124,7 @@ def test_plan_shows_deferred_when_agent_type_is_none(self, tmp_path: Path) -> No patch(f"{_EXP}.resolve_task_for_variant", return_value=(resolved_task, {}, 1)), patch("coder_eval.cli.plan_command.console") as mock_console, ): - plan_command(task_files=[task_file]) + run_plan(task_files=[task_file]) printed = [str(call) for call in mock_console.print.call_args_list] deferred_lines = [p for p in printed if "deferred" in p] @@ -155,7 +155,7 @@ def test_plan_with_experiment_flag_shows_experiment_info(self, tmp_path: Path) - patch(f"{_EXP}.DEFAULT_EXPERIMENT_PATH", exp_file), patch("coder_eval.cli.plan_command.console") as mock_console, ): - plan_command(task_files=[task_file], experiment=exp_file) + run_plan(task_files=[task_file], experiment=exp_file) printed = " ".join(str(call) for call in mock_console.print.call_args_list) assert "test-exp" in printed @@ -185,7 +185,7 @@ def test_plan_with_experiment_shows_resolved_agent_per_variant(self, tmp_path: P patch(f"{_EXP}.DEFAULT_EXPERIMENT_PATH", exp_file), patch("coder_eval.cli.plan_command.console") as mock_console, ): - plan_command(task_files=[task_file], experiment=exp_file) + run_plan(task_files=[task_file], experiment=exp_file) printed = " ".join(str(call) for call in mock_console.print.call_args_list) assert "sonnet-4" in printed @@ -211,7 +211,7 @@ def test_plan_with_default_experiment(self, tmp_path: Path) -> None: patch(f"{_EXP}.resolve_task_for_variant", return_value=(resolved_task, {}, 1)), patch("coder_eval.cli.plan_command.console") as mock_console, ): - plan_command(task_files=[task_file]) + run_plan(task_files=[task_file]) printed = " ".join(str(call) for call in mock_console.print.call_args_list) assert "test-exp" in printed @@ -231,7 +231,7 @@ def test_plan_warns_when_task_timeout_cannot_extend_single_iteration(self, tmp_p patch(f"{_EXP}.resolve_task_for_variant", return_value=(resolved_task, {}, 1)), patch("coder_eval.cli.plan_command.console") as mock_console, ): - plan_command(task_files=[task_file]) + run_plan(task_files=[task_file]) printed = " ".join(str(call) for call in mock_console.print.call_args_list) assert "A larger task_timeout cannot extend the agent's single iteration" in printed @@ -254,7 +254,7 @@ def test_plan_exits_when_default_experiment_missing(self, tmp_path: Path) -> Non patch("coder_eval.cli.plan_command.console") as mock_console, ): with pytest.raises(typer.Exit) as exc_info: - plan_command(task_files=[task_file]) + run_plan(task_files=[task_file]) assert exc_info.value.exit_code == 1 @@ -280,7 +280,7 @@ def test_plan_reports_invalid_task(self, tmp_path: Path) -> None: patch("coder_eval.cli.plan_command.console") as mock_console, ): with pytest.raises(typer.Exit) as exc_info: - plan_command(task_files=[task_file]) + run_plan(task_files=[task_file]) assert exc_info.value.exit_code == 1 @@ -305,7 +305,7 @@ def test_plan_exits_on_explicit_experiment_load_failure(self, tmp_path: Path) -> patch("coder_eval.cli.plan_command.console") as mock_console, ): with pytest.raises(typer.Exit) as exc_info: - plan_command(task_files=[task_file], experiment=exp_file) + run_plan(task_files=[task_file], experiment=exp_file) assert exc_info.value.exit_code == 1 diff --git a/tests/test_regrade.py b/tests/test_regrade.py index 668046c8..6663089b 100644 --- a/tests/test_regrade.py +++ b/tests/test_regrade.py @@ -27,8 +27,6 @@ parse_agent_config, ) from coder_eval.orchestration.regrade import ( - PRE_GRADE_JSON, - TASK_JSON, RegradeError, back_up_pre_grade_record, default_workspace, @@ -36,7 +34,7 @@ task_from_prior, verify_reference_unchanged, ) -from coder_eval.path_utils import digest_tree +from coder_eval.path_utils import PRE_GRADE_JSON_FILENAME, TASK_JSON_FILENAME, digest_tree def _task(*, reference: dict[str, str] | None = None, command: str | None = None) -> TaskDefinition: @@ -80,7 +78,7 @@ def test_missing_task_json_is_a_regrade_error(tmp_path: Path) -> None: def test_unparseable_task_json_is_a_regrade_error(tmp_path: Path) -> None: - (tmp_path / TASK_JSON).write_text("{not json", encoding="utf-8") + (tmp_path / TASK_JSON_FILENAME).write_text("{not json", encoding="utf-8") with pytest.raises(RegradeError, match="not a readable EvaluationResult"): load_prior_result(tmp_path) @@ -235,25 +233,64 @@ def test_a_task_with_no_reference_is_not_checked(tmp_path: Path) -> None: def test_the_pre_grade_record_is_written_once(tmp_path: Path) -> None: """A second grade must not overwrite the ORIGINAL execute record with an already-graded one — that is the only evidence the run was ungraded.""" - (tmp_path / TASK_JSON).write_text('{"round": 1}', encoding="utf-8") + (tmp_path / TASK_JSON_FILENAME).write_text('{"round": 1}', encoding="utf-8") back_up_pre_grade_record(tmp_path) - (tmp_path / TASK_JSON).write_text('{"round": 2}', encoding="utf-8") + (tmp_path / TASK_JSON_FILENAME).write_text('{"round": 2}', encoding="utf-8") back_up_pre_grade_record(tmp_path) - assert json.loads((tmp_path / PRE_GRADE_JSON).read_text(encoding="utf-8")) == {"round": 1} + assert json.loads((tmp_path / PRE_GRADE_JSON_FILENAME).read_text(encoding="utf-8")) == {"round": 1} def test_backup_is_a_no_op_with_nothing_to_back_up(tmp_path: Path) -> None: back_up_pre_grade_record(tmp_path) - assert not (tmp_path / PRE_GRADE_JSON).exists() + assert not (tmp_path / PRE_GRADE_JSON_FILENAME).exists() def test_a_failed_backup_never_fails_the_grade(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: """The audit copy is a convenience; the verdict is the deliverable.""" - (tmp_path / TASK_JSON).write_text("{}", encoding="utf-8") + (tmp_path / TASK_JSON_FILENAME).write_text("{}", encoding="utf-8") def _boom(*_args: object, **_kwargs: object) -> None: raise OSError("read-only file system") monkeypatch.setattr(Path, "write_text", _boom) back_up_pre_grade_record(tmp_path) # must not raise + + +# -------------------------------------------------------------------------- +# Dataset-row ids contain "/" +# -------------------------------------------------------------------------- + + +def test_a_dataset_row_workspace_resolves_by_task_id_not_by_child_count(tmp_path: Path) -> None: + """Preservation writes artifacts/, and a dataset row's task_id is + "/". "The single child of artifacts/" therefore resolves to + artifacts/ — one level too high — and every path-relative criterion + then fails as a locating artifact rather than as a verdict.""" + workspace = tmp_path / "artifacts" / "suite" / "row-1" + workspace.mkdir(parents=True) + + resolved = default_workspace(tmp_path, _result(task_id="suite/row-1")) + + assert resolved == workspace + + +def test_an_ambiguous_artifacts_dir_refuses_rather_than_guessing(tmp_path: Path) -> None: + artifacts = tmp_path / "artifacts" + (artifacts / "a").mkdir(parents=True) + (artifacts / "b").mkdir() + + with pytest.raises(RegradeError, match="Pass --workspace"): + default_workspace(tmp_path, _result(task_id="neither")) + + +def test_a_sandbox_path_outside_the_run_dir_is_refused(tmp_path: Path) -> None: + """`sandbox_path` is an unvalidated absolute path out of the run's own + task.json, and criteria execute with cwd there and may mutate it.""" + outside = tmp_path / "elsewhere" + outside.mkdir() + run_dir = tmp_path / "run" + run_dir.mkdir() + + with pytest.raises(RegradeError, match="outside the run directory"): + default_workspace(run_dir, _result(sandbox_path=str(outside))) diff --git a/tests/test_seed_from_prior_result.py b/tests/test_seed_from_prior_result.py index 319815d3..10b602cc 100644 --- a/tests/test_seed_from_prior_result.py +++ b/tests/test_seed_from_prior_result.py @@ -106,7 +106,7 @@ def _prior() -> EvaluationResult: pre_run_results=[PostRunResult(command="prior-pre", exit_code=0)], post_run_results=[PostRunResult(command="prior-post", exit_code=0)], sandbox_path="/prior/workspace", - environment_info={"installed_tools": "prior"}, + environment_info={"installed_tools": "prior", "coder_eval": "1.0.0-run"}, early_stop=EarlyStopInfo( reason=EarlyStopReason.CRITERION_FAILED, deciding_criterion_type="skill_triggered", @@ -137,7 +137,7 @@ def _seeded(tmp_path: Path) -> tuple[Orchestrator, EvaluationResult]: started_at=datetime(2030, 1, 1, 0, 0, 0), final_status=FinalStatus.FAILURE, iteration_count=0, - environment_info={"installed_tools": "grader"}, + environment_info={"installed_tools": "grader", "coder_eval": "9.9.9-grader"}, ) orch._seed_from_prior_result() assert orch.result is not None @@ -182,7 +182,13 @@ def test_grader_environment_is_kept_beside_the_run_s_not_over_it(tmp_path: Path) # The run's own capture wins: a report showing the grader's tool versions as # the run's is worse than one showing neither. assert orch.result.environment_info["installed_tools"] == "prior" - assert orch.result.environment_info["graded_by"] == {"installed_tools": "grader"} + # The grader is recorded as FLAT scalars, and only where it differs. Nesting + # a whole env capture here renders as a Python dict repr in the HTML report + # and violates the evalboard's declared value type. + assert orch.result.environment_info["graded_by_coder_eval"] == "9.9.9-grader" + assert all(not isinstance(v, dict) or k == "installed_tools" for k, v in orch.result.environment_info.items()), ( + "environment_info is consumed as a flat map" + ) def test_the_evaluate_only_path_selects_the_same_gate_as_the_agent_path(tmp_path: Path) -> None: diff --git a/tests/test_ungraded_reporting.py b/tests/test_ungraded_reporting.py new file mode 100644 index 00000000..db2b9933 --- /dev/null +++ b/tests/test_ungraded_reporting.py @@ -0,0 +1,254 @@ +"""How an ungraded row renders on every reporting surface. + +`coder-eval execute` leaves rows `NOT_GRADED`, and each generator had to learn a +fourth category. Only the HTML badge got an assertion when that landed, so the +JUnit `` element, the Markdown "Not Graded" bullets, and — most +importantly — the switched pass-rate DENOMINATOR were all shipped untested. The +denominator is the "gate that turns a gap into a score" shape: a printed rate +whose divisor changed, with nothing tripping it. + +`VariantAggregate` is here too, mirroring the four `RunSummary` cases in +`test_execute_command.py`: the two models now carry the same formula, and only +one of them was tested. +""" + +from __future__ import annotations + +from datetime import datetime +from pathlib import Path +from typing import Any + +import pytest + +from coder_eval.models import FinalStatus, RunSummary, VariantAggregate, VariantResult +from coder_eval.orchestration.experiment import _mean_graded_score, _pick_worst_status + + +def _summary(**kwargs: object) -> RunSummary: + base: dict[str, object] = { + "run_id": "r", + "start_time": datetime(2026, 1, 1), + "end_time": datetime(2026, 1, 1, 0, 1), + "total_duration_seconds": 60.0, + "tasks_run": 2, + "tasks_succeeded": 1, + "tasks_failed": 0, + "tasks_error": 0, + "tasks_not_graded": 1, + "task_results": [], + "framework_version": "test", + } + base.update(kwargs) + return RunSummary(**base) # type: ignore[arg-type] + + +def _aggregate(**kwargs: object) -> VariantAggregate: + base: dict[str, object] = { + "variant_id": "v", + "tasks_run": 2, + "tasks_succeeded": 1, + "tasks_failed": 0, + "tasks_error": 0, + "tasks_not_graded": 1, + "average_score": 1.0, + "average_duration": 1.0, + } + base.update(kwargs) + return VariantAggregate(**base) # type: ignore[arg-type] + + +# -------------------------------------------------------------------------- +# VariantAggregate — the twin of the RunSummary cases in test_execute_command +# -------------------------------------------------------------------------- + + +def test_variant_aggregate_counts_the_ungraded_bucket_in_its_invariant() -> None: + with pytest.raises(ValueError, match="Task count invariant violated"): + _aggregate(tasks_run=3) # 1 + 0 + 0 + 1 != 3 + + +def test_variant_aggregate_pass_rate_divides_by_graded_not_run() -> None: + """One pass out of one GRADED task is 100%, even beside an ungraded one.""" + assert _aggregate().tasks_graded == 1 + assert _aggregate().pass_rate == 1.0 + + +def test_variant_aggregate_has_no_pass_rate_when_nothing_was_graded() -> None: + agg = _aggregate(tasks_run=2, tasks_succeeded=0, tasks_not_graded=2) + assert agg.pass_rate is None, "0/0 is unknown, not 0%" + + +def test_variant_aggregate_serializes_its_denominator() -> None: + """A consumer that cannot read `tasks_graded` re-derives the rate and drifts.""" + import json + + assert json.loads(_aggregate().model_dump_json())["tasks_graded"] == 1 + + +def test_mean_graded_score_ignores_ungraded_rows() -> None: + def _vr(score: float | None, status: FinalStatus) -> VariantResult: + return VariantResult( + variant_id="v", task_id="t", weighted_score=score, final_status=status, duration_seconds=1.0 + ) + + rows = [_vr(1.0, FinalStatus.SUCCESS), _vr(None, FinalStatus.NOT_GRADED), _vr(None, FinalStatus.NOT_GRADED)] + assert _mean_graded_score(rows) == 1.0 + # Nothing graded -> no mean at all. 0.0 would read as "measured and scored zero". + assert _mean_graded_score(rows[1:]) is None + + +def test_ungraded_loses_to_every_real_outcome_when_picking_the_worst_status() -> None: + """`_pick_worst_status` reports the replicate set's worst outcome. An ungraded + replicate is not an outcome, so it must never mask a real one.""" + assert _pick_worst_status([FinalStatus.NOT_GRADED, FinalStatus.SUCCESS]) is FinalStatus.SUCCESS + assert _pick_worst_status([FinalStatus.NOT_GRADED, FinalStatus.FAILURE]) is FinalStatus.FAILURE + assert _pick_worst_status([FinalStatus.NOT_GRADED]) is FinalStatus.NOT_GRADED + + +# -------------------------------------------------------------------------- +# Markdown +# -------------------------------------------------------------------------- + + +def test_markdown_pass_rate_uses_the_graded_denominator() -> None: + from coder_eval.reports import _pass_rate_lines + + text = "\n".join(_pass_rate_lines(_summary())) + + assert "1/1" in text, "the denominator is tasks_graded, not tasks_run" + assert "1/2" not in text + + +def test_markdown_reports_no_rate_at_all_for_a_fully_ungraded_run() -> None: + from coder_eval.reports import _pass_rate_lines + + text = "\n".join(_pass_rate_lines(_summary(tasks_succeeded=0, tasks_not_graded=2))) + + assert "n/a" in text + assert "0.0%" not in text, "a run that was never measured has no rate, not a 0% one" + + +def test_markdown_reports_no_rate_change_for_an_ordinary_graded_run() -> None: + """The regression guard: adding the fourth bucket must not alter any surface + of a run that has none.""" + from coder_eval.reports import _pass_rate_lines + + text = "\n".join(_pass_rate_lines(_summary(tasks_run=2, tasks_succeeded=1, tasks_failed=1, tasks_not_graded=0))) + + assert "1/2" in text + assert "n/a" not in text + + +# -------------------------------------------------------------------------- +# JUnit +# -------------------------------------------------------------------------- + + +def _junit_for(status: FinalStatus, tmp_path: Path) -> Any: + import json + + # defusedxml on the test side, matching tests/test_reports_junit.py. + from defusedxml.ElementTree import parse as parse_xml + + from coder_eval.reports_junit import write_junit_xml + + run_dir = tmp_path / "run" + task_dir = run_dir / "default" / "t" / "00" + task_dir.mkdir(parents=True) + row = { + "task_id": "t", + "task_description": "d", + "variant_id": "default", + "agent_type": "claude-code", + "started_at": "2026-01-01T00:00:00", + "final_status": status.value, + "iteration_count": 1, + "duration_seconds": 1.0, + } + (task_dir / "task.json").write_text(json.dumps(row), encoding="utf-8") + (run_dir / "run.json").write_text( + json.dumps( + { + "run_id": "r", + "start_time": "2026-01-01T00:00:00", + "end_time": "2026-01-01T00:01:00", + "total_duration_seconds": 60.0, + "tasks_run": 1, + "tasks_succeeded": 1 if status is FinalStatus.SUCCESS else 0, + "tasks_failed": 0, + "tasks_error": 0, + "tasks_not_graded": 1 if status is FinalStatus.NOT_GRADED else 0, + "task_results": [{"task_id": "t", "variant_id": "default", "status": status.value, "duration": 1.0}], + "framework_version": "test", + } + ), + encoding="utf-8", + ) + written = write_junit_xml(run_dir, tmp_path / "junit.xml") + return parse_xml(written).getroot() + + +def test_junit_marks_an_ungraded_row_skipped_not_failed(tmp_path: Path) -> None: + root = _junit_for(FinalStatus.NOT_GRADED, tmp_path) + + skipped = root.findall(".//testcase/skipped") + assert len(skipped) == 1, "an ungraded row is not a verdict, so it is " + assert "not graded" in (skipped[0].get("message") or "") + assert not root.findall(".//testcase/failure"), "an ungraded row must not read as a failure" + + +def test_junit_counts_are_derived_from_the_children(tmp_path: Path) -> None: + root = _junit_for(FinalStatus.NOT_GRADED, tmp_path) + suite = root.find(".//testsuite") + assert suite is not None + assert suite.get("skipped") == "1" + assert suite.get("failures") == "0" + assert suite.get("errors") == "0" + + +def test_junit_still_passes_an_ordinary_graded_row(tmp_path: Path) -> None: + root = _junit_for(FinalStatus.SUCCESS, tmp_path) + assert not root.findall(".//testcase/skipped") + assert not root.findall(".//testcase/failure") + + +# -------------------------------------------------------------------------- +# The end-of-run console summary +# -------------------------------------------------------------------------- + + +def _summary_output(summary: RunSummary, tmp_path: Path) -> str: + from coder_eval.cli.console import console + from coder_eval.cli.run_helpers import print_execution_summary + + with console.capture() as captured: + print_execution_summary(tmp_path, summary) + return captured.get() + + +def test_summary_reports_an_ungraded_run_as_executed_not_failed(tmp_path: Path) -> None: + text = _summary_output(_summary(tasks_succeeded=0, tasks_not_graded=2), tmp_path) + assert "2/2 executed, not graded" in text + assert "0/0 succeeded" not in text, "a fully ungraded run has no succeeded ratio to report" + + +def test_summary_points_at_the_grading_form_that_keeps_the_trajectory(tmp_path: Path) -> None: + """`evaluate ` grades a bare directory with NO + trajectory, so command_executed / skill_triggered / trajectory judges score + differently from what `run` would have produced.""" + text = _summary_output(_summary(tasks_succeeded=0, tasks_not_graded=2), tmp_path) + assert "--resume" in text or "" in text + assert "evaluate " not in text + + +def test_summary_still_reports_a_ratio_for_an_empty_run(tmp_path: Path) -> None: + """Both counters are falsy for tasks_run == 0; independent `if`s would print + no Results line at all, where it previously printed 0/0.""" + empty = _summary(tasks_run=0, tasks_succeeded=0, tasks_not_graded=0) + assert "0/0 succeeded" in _summary_output(empty, tmp_path) + + +def test_summary_is_unchanged_for_an_ordinary_graded_run(tmp_path: Path) -> None: + text = _summary_output(_summary(tasks_run=2, tasks_succeeded=1, tasks_failed=1, tasks_not_graded=0), tmp_path) + assert "1/2 succeeded" in text + assert "not graded" not in text From e9dfb947ad9aed0ccdd41bb666d2cb9c05643e9d Mon Sep 17 00:00:00 2001 From: Akshaya Shanbhogue Date: Thu, 3 Sep 2026 16:38:32 -0700 Subject: [PATCH 6/6] test: fix two CI-only failures in the detached-grading tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Both are test bugs, not product bugs, and both are the same class: an assertion that passes on the developer's machine and only on the developer's machine. `_sanitize_restored_path` splits on `os.pathsep`; its test built the input with a hardcoded ":". On Windows that parses as ONE non-existent entry, so the sanitizer returns "" and every assertion below it passes vacuously — the test was asserting nothing on the platform it failed on. Rich splits an `--option` token across several style spans (`--junit-xml` renders as `-` + `-junit` + `-xml`, each with its own escape), and it styles whenever it believes it is writing to a terminal — which includes GitHub Actions. So a bare substring check over `result.output` is green locally and red only in CI. Strip ANSI first, following the helper and the comment already in tests/test_cli_type_flag.py. Co-Authored-By: Claude Opus 5 (1M context) --- tests/test_detached_grading_guards.py | 7 ++++++- tests/test_execute_command.py | 17 ++++++++++++++++- 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/tests/test_detached_grading_guards.py b/tests/test_detached_grading_guards.py index 2ff234cf..d0f66b18 100644 --- a/tests/test_detached_grading_guards.py +++ b/tests/test_detached_grading_guards.py @@ -7,6 +7,7 @@ from __future__ import annotations +import os from datetime import datetime from pathlib import Path from unittest.mock import AsyncMock, MagicMock, patch @@ -221,7 +222,11 @@ def test_a_restored_path_drops_entries_inside_the_graded_workspace(tmp_path: Pat orch.sandbox = MagicMock() orch.sandbox.sandbox_dir = workspace - kept = orch._sanitize_restored_path(f"{workspace / 'bin'}:{outside}:{tmp_path / 'gone'}") + # os.pathsep, not a hardcoded ":" — the separator is ";" on Windows, where a + # colon-joined value parses as one (non-existent) entry and every assertion + # below passes vacuously against an empty result. + recorded = os.pathsep.join([str(workspace / "bin"), str(outside), str(tmp_path / "gone")]) + kept = orch._sanitize_restored_path(recorded) assert str(outside.resolve()) in kept assert str(workspace) not in kept, "an entry inside the graded tree must be dropped" diff --git a/tests/test_execute_command.py b/tests/test_execute_command.py index c0189b54..e7686c6b 100644 --- a/tests/test_execute_command.py +++ b/tests/test_execute_command.py @@ -16,6 +16,7 @@ from __future__ import annotations import json +import re from pathlib import Path from typing import Any from unittest.mock import patch @@ -29,6 +30,19 @@ runner = CliRunner() +# Rich styles each `--option` token in help text, and it splits the token across +# several style spans (`--junit-xml` renders as `-` + `-junit` + `-xml`, each with +# its own escape sequence). Styling is ON whenever rich thinks it is writing to a +# terminal — which includes GitHub Actions, so a bare substring check over +# `result.output` passes locally and fails only in CI. Same helper, same reason, +# as tests/test_cli_type_flag.py. +_ANSI_RE = re.compile(r"\x1b\[[0-9;]*m") + + +def _strip_ansi(s: str) -> str: + return _ANSI_RE.sub("", s) + + # The agentless smoke task: no agent, no model call, and a pre_run that writes a # file its criteria read back. Executing it must still write that file (proving # the run really happened) while scoring nothing. @@ -288,5 +302,6 @@ def test_execute_help_explains_the_refused_flags() -> None: above; here we only require the help text to mention it.)""" result = runner.invoke(app, ["execute", "--help"]) assert result.exit_code == 0 + output = _strip_ansi(result.output) for flag in _DELIBERATELY_ABSENT_FROM_EXECUTE: - assert flag in result.output, f"execute's help should explain why {flag} is unavailable" + assert flag in output, f"execute's help should explain why {flag} is unavailable"