feat(cli): coder-eval execute + detached grading via evaluate <run_dir>

.claude/shared/run-layout.md

@@ -12,6 +12,7 @@ runs/<run_id>/<variant_id>/<task_id>/<NN>/{task.json, task.log, artifacts/}
 - `<NN>` — zero-padded replicate index (e.g. `00`, `01`).
 - `task.json` — the persisted per-replicate result (the consumer contract; carries the large `iterations` array — still accepted under its former name `turns` when reading, but not what current runs write).
 - `task.json.malformed` — present only on the docker degrade path: when an existing `task.json` fails to parse (schema skew from a stale `:latest` image, or a truncated/torn write), the docker runner moves the unparseable original aside to this sidecar and writes a synthetic `final_status=ERROR` `task.json` in its place. Diagnostic-only; `rglob("task.json")` consumers do not match it.
+- `task.execute.json` — present only after a DETACHED grade (`coder-eval evaluate <run_dir>` or `coder-eval run --resume` over a `NOT_GRADED` row). The pre-grade snapshot of `task.json`, written once and never overwritten by a later grade, so "this run was executed separately from grading" stays auditable. Diagnostic-only; `rglob("task.json")` consumers do not match it.
 - `task.log` — the human-readable task log; `artifacts/` — files the agent produced.
 
 **Scope-marker files** (used to detect what a given path represents):

.github/workflows/verify-published-action.yml

@@ -541,6 +541,18 @@ jobs:
                     "uploaded run dir before re-running; this is an unattended paid job.")
               sys.exit(1)
 
+          # NOT_GRADED means a task ran but was never scored. This job invokes the
+          # published action, which runs `coder-eval run` (graded), so reaching it is
+          # impossible unless the action started dispatching `coder-eval execute` --
+          # in which case every score gate below silently measures nothing and the job
+          # goes green having verified no verdict at all. Hard-fail, don't tolerate.
+          ungraded = [s for s in statuses if s == "NOT_GRADED"]
+          if ungraded:
+              print("::error::task(s) reported NOT_GRADED -- the published action ran without "
+                    "grading. `coder-eval run` always grades, so the action is dispatching the "
+                    "wrong command and every score gate in this job is measuring nothing.")
+              sys.exit(1)
+
           # Exit-contract check, conditional on the model having actually performed.
           # Ignoring the step's exit code entirely (see the continue-on-error rationale
           # above) would also hide a REGRESSION in the action's own exit logic -- e.g. a

docs/REPORT_SCHEMA.md

@@ -23,6 +23,7 @@ read). Times are ISO-8601.
 | --- | --- | --- |
 | `run.json` / `run.md` | `RunSummary` | Every run (and rebuildable via `coder-eval aggregate`) |
 | `<variant>/<task_id>/<NN>/task.json` | `EvaluationResult` | One per replicate |
+| `<variant>/<task_id>/<NN>/task.execute.json` | `EvaluationResult` | Pre-grade snapshot, written once by a detached grade (`evaluate <run_dir>` / `run --resume`). Deliberately **not** matched by `rglob("task.json")`, so it never enters an aggregation. |
 | `<variant>/<suite_id>/suite.json` / `.md` | `SuiteRollup` | Dataset-backed suites only |
 | `experiment.json` / `.md` | `ExperimentResult` | Every run (experiment layer) |
 | `<variant>/variant.json` / `.md` | `VariantAggregate` | Per variant |
@@ -44,7 +45,8 @@ run-level summary; full per-replicate detail lives in each `task.json`.
 | `start_time` / `end_time` | `datetime` | Run window. |
 | `total_duration_seconds` | `float` | Wall-clock. |
 | `tasks_run` | `int` | Total replicates executed. |
-| `tasks_succeeded` / `tasks_failed` / `tasks_error` | `int` | Category counts. **Invariant:** the three sum to `tasks_run`. |
+| `tasks_succeeded` / `tasks_failed` / `tasks_error` | `int` | Category counts. **Invariant:** these three plus `tasks_not_graded` sum to `tasks_run`. |
+| `tasks_not_graded` | `int` | Tasks run by `coder-eval execute` — executed, deliberately unscored. Excluded from **both** sides of `pass_rate`. Defaults to `0`, so pre-`execute` `run.json` still parses. |
 | `tasks_token_budget_exceeded` / `tasks_cost_budget_exceeded` | `int` | Sub-counters of `tasks_failed` (not part of the invariant). |
 | `skipped_tasks` | `list[{path, reason}]` | Load failures / `skip: true` opt-outs. |
 | `max_parallel` | `int` | Concurrency used. |
@@ -59,8 +61,9 @@ publishing different numbers for the same run.
 
 | Key | Type | Meaning |
 | --- | --- | --- |
-| `pass_rate` | `float \| None` | `tasks_succeeded / tasks_run` — errors are in the denominator, counted as misses. `None` on an empty run (0/0 is unknown, not 0%). |
-| `error_share` | `float \| None` | `tasks_error / tasks_run`. Diagnostic only; never adjusts the rate. |
+| `pass_rate` | `float \| None` | `tasks_succeeded / tasks_graded` — errors are in the denominator, counted as misses; ungraded tasks are in neither. `None` on an empty or fully ungraded run (0/0 is unknown, not 0%). |
+| `error_share` | `float \| None` | `tasks_error / tasks_graded`. Diagnostic only; never adjusts the rate. |
+| `tasks_graded` | `int` | `tasks_run - tasks_not_graded`. The denominator of both rates above. |
 | `total_cost_usd` | `float \| None` | **The bill**: agent + judge + simulator, summed over the rows. `None` when nothing could be priced. |
 | `agent_cost_usd` | `float \| None` | Subject-agent spend alone. The harness-vs-harness comparison figure — judge spend is a property of the suite's criteria and identical across harnesses, so leaving it in would make two harnesses look closer than they are. |
 | `eval_overhead_cost_usd` | `float \| None` | Judge + simulator spend. The other half of `total_cost_usd`. |
@@ -232,7 +235,8 @@ the same weighted armed gate as a native fail),
 ## `variant.json` — `VariantAggregate`
 
 A single aggregate (not wrapped): `variant_id`, `tasks_run`, `tasks_succeeded`,
-`tasks_failed`, `tasks_error` (same sum-to-`tasks_run` invariant), `average_score`,
+`tasks_failed`, `tasks_error`, `tasks_not_graded` (same sum-to-`tasks_run` invariant), `average_score`
+(the mean over **graded** rows only),
 `average_duration`, `total_tokens`, `replicate_count`, `tasks_token_budget_exceeded`,
 `tasks_cost_budget_exceeded`.
 
@@ -262,8 +266,8 @@ Written for dataset-backed suites; its `passed` flag drives the CI exit code.
 | Key | Type | Meaning |
 | --- | --- | --- |
 | `suite_id` / `variant_id` | `str` | Identity. |
-| `rows_total` / `rows_passed` / `rows_failed` / `rows_error` | `int` | Row counts. |
-| `pass_rate` | `float` | `rows_passed / rows_total`. |
+| `rows_total` / `rows_passed` / `rows_failed` / `rows_error` / `rows_not_graded` | `int` | Row counts. **Invariant:** the four category counts sum to `rows_total`. `rows_not_graded` defaults to `0`. |
+| `pass_rate` | `float` | `rows_passed / (rows_total - rows_not_graded)` — ungraded rows leave both sides, matching `RunSummary.pass_rate`. |
 | `average_weighted_score` | `float \| null` | Mean row score. |
 | `criterion_stats` | `list[{criterion_type, rows_evaluated, average_score, error_count}]` | Per-criterion summary. |
 | `failed_samples` | `list[FailedRowSummary]` | Capped at 20 (`{row_id, task_id, final_status, weighted_score, failure_reasons, error_message, task_json_relpath, replicate_index}`). |
@@ -308,10 +312,20 @@ String enum values and their reporting category:
 | `COST_BUDGET_EXCEEDED` | failed | `$` |
 | `ERROR` | error | `!` |
 | `BUILD_FAILED` | error | `B` |
+| `NOT_GRADED` | ungraded | `?` |
 
 > **Gotcha:** `BUILD_FAILED` (a failed Docker image build) categorizes as **error**,
 > not failed — easy to miscount downstream.
 
+`NOT_GRADED` is produced only by [`coder-eval execute`](USER_GUIDE.md#coder-eval-execute--run-without-grading):
+the task ran and its full trajectory was captured, but no criterion was checked, so
+`weighted_score` is `None` (**not** `0.0` — that would be indistinguishable from a task
+that was graded and scored zero). `ungraded` is a fourth reporting category, not a fold
+into one of the other three: counting it as failed would depress every pass rate, and
+counting it as succeeded would invent a verdict. Execution facts still win over it — a
+crash, timeout, or budget breach under `execute` reports `ERROR` / `TIMEOUT` /
+`TOKEN_BUDGET_EXCEEDED` as usual.
+
 `TOKEN_BUDGET_EXCEEDED` and `COST_BUDGET_EXCEEDED` are produced by the cumulative budget caps under
 `run_limits:` (`max_input_tokens` / `max_output_tokens` / `max_total_tokens`, and `max_usd`
 respectively), checked after each completed agent turn — see

docs/USER_GUIDE.md

@@ -41,7 +41,7 @@ coder-eval run tasks/hello_date.yaml --stream full  # live LLM output
 | `--driver` | Shorthand alias for `-D sandbox.driver=…` (`tempdir` or `docker`) |
 | `--type, -T` | Override agent type for all tasks (`claude-code`, `codex`, `antigravity`, `opencode`, or a plugin kind). |
 | `--repeats` | Run each `(task, variant)` N times (≥1); overrides experiment/variant `repeats:`. See [Replicates](#replicates). |
-| `--resume` | Resume an interrupted run: skip tasks already finalized in `--run-dir` and run the rest, folding prior results into `run.json`. Requires `--run-dir`. A task with *any* final status (incl. FAILED/ERROR) counts as finalized, so resume does **not** retry failures — delete a task's `task.json` to force a re-run. A config mismatch is warned, not refused. |
+| `--resume` | Resume an interrupted run: skip tasks already finalized in `--run-dir` and run the rest, folding prior results into `run.json`. Requires `--run-dir`. See [Resuming a run](#resuming-a-run). |
 | `--sample N` | For dataset-backed tasks, run a fixed-seed random N-row sample (reproducible; cheap smoke test). See [Bring Your Own Dataset](DATASETS.md). |
 | `--sample-per-stratum N` | For dataset-backed tasks, keep up to N rows per stratum (`stratify_field`). Overridden by `--sample`. Nondeterministic unless `dataset.sample_seed` is set — see [Bring Your Own Dataset](DATASETS.md). |
 | `--include-skipped` | Also run tasks marked `skip: true` in their YAML (off by default so CI keeps excluding them). |
@@ -58,6 +58,83 @@ flags of their own. They live under `run_limits:` in the task YAML, or on the co
 `-D run_limits.<field>=<value>` (e.g. `-D run_limits.max_usd=2.50`). The complete field reference is
 in the [Task Definition Guide](TASK_DEFINITION_GUIDE.md#run-limits).
 
+### `coder-eval execute` — run without grading
+
+```bash
+coder-eval execute tasks/hello_date.yaml                    # run, capture, score nothing
+coder-eval execute tasks/*.yaml --run-dir ./my-run -j 3     # every `run` flag but two
+```
+
+Identical to `coder-eval run` except that no success criterion is checked. Each task
+executes normally and its full trajectory lands in `task.json` as usual, but
+`weighted_score` stays `null` and the row finalizes as `NOT_GRADED` — a reporting
+category of its own, excluded from both sides of every pass rate. The two commands
+share one implementation, so they cannot drift apart.
+
+Use it when something *else* owns the verdict — an external harness that builds its own
+container and runs its own tests — or to separate one expensive agent run from grading
+you want to iterate on afterwards. Grade the results later with
+[`coder-eval evaluate`](#coder-eval-evaluate--grade-without-running-an-agent).
+
+**Only the verdict is withheld, never the facts of the run.** A crash, timeout, or
+budget breach still reports `ERROR` / `TIMEOUT` / `TOKEN_BUDGET_EXCEEDED` and still
+exits non-zero, exactly as under `run`.
+
+Every `run` flag is available except two things, each refused rather than quietly
+degraded:
+
+| Not supported | Why |
+| --- | --- |
+| `--junit-xml` | A JUnit report reports verdicts, and there are none. |
+| Simulation tasks | The dialog loop reads criteria results to decide whether to keep talking, so an ungraded dialog would silently change its own stopping behavior. Rejected by name at startup. |
+
+`stop_early:` blocks are also inert here: early stop exists to cut a run once the
+criteria decide the outcome, and under `execute` the full trajectory is the deliverable.
+
+`--resume` **is** supported, and `run --resume` pairs with it (see below).
+
+### Resuming a run
+
+`--resume` continues an interrupted run without re-paying for finished work. It
+requires `--run-dir` (an auto-generated directory is always fresh).
+
+What it owes each task depends on what it finds in that task's `task.json`:
+
+| On disk | `run --resume` | `execute --resume` |
+| --- | --- | --- |
+| No `task.json`, unreadable, or no `final_status` | re-run | re-run |
+| `NOT_GRADED` | **grade in place** | already complete |
+| Any other status, **including `FAILURE` / `ERROR`** | already complete | already complete |
+
+**"Finished" is relative to the resuming command.** A `NOT_GRADED` row owes
+`execute` nothing — it finished executing — but owes `run` a grade. So
+`run --resume` runs the criteria against the trajectory and workspace already on
+disk instead of re-running the agent, which is the whole reason to split the two
+commands:
+
+```bash
+coder-eval execute tasks/*.yaml --run-dir ./r    # expensive half
+coder-eval run     tasks/*.yaml --run-dir ./r --resume   # grades what execute left
+```
+
+**Resume never retries failures.** `FAILURE` and `ERROR` count as complete under
+both commands — delete a task's `task.json` to force a re-run. A task about to
+re-run has its stale `artifacts/<task_id>` cleared first, so leftover files from
+a killed container cannot satisfy a file-based criterion.
+
+A run-config mismatch is **warned, not refused**: resumed tasks keep their
+original-config results, so the run genuinely mixes configs. The `grade` flag is
+exempt from that warning, because `execute` → `run --resume` is a supported flow
+rather than a config mistake.
+
+**A dataset task must pin its sample to be resumable.** Stratified sampling
+(`--sample-per-stratum` / `dataset.sample_per_stratum`) re-draws on every
+invocation, and each row is its own task (`<task_id>/<row_id>`) with its own run
+directory. A resume therefore draws a *different* row set, finds no `task.json`
+for it, and pays for the agent a second time while the executed rows sit
+orphaned in the run dir. Set `dataset.sample_seed`, or use `--sample N` (which is
+seeded), before splitting a dataset run across `execute` and `run --resume`.
+
 ### `coder-eval plan` — validate tasks
 
 ```bash
@@ -71,23 +148,58 @@ Checks task syntax, required CLI tools, API keys, and schema validity without ex
 | --- | --- |
 | `--experiment, -e` | Experiment definition YAML to resolve variants against (default: `experiments/default.yaml`). |
 
-### `coder-eval evaluate` — test criteria without an agent
+### `coder-eval evaluate` — grade without running an agent
+
+Two shapes, told apart by whether the target holds a `task.json`:
 
 ```bash
-coder-eval evaluate tasks/hello_date.yaml ./my_solution            # evaluate a directory
-coder-eval evaluate tasks/hello_date.yaml ./my_solution --preserve # keep the sandbox
+# 1. Grade a directory against a task
+coder-eval evaluate tasks/hello_date.yaml ./my_solution
+
+# 2. Re-grade a finished run — including one left NOT_GRADED by `execute`
+coder-eval execute  tasks/hello_date.yaml --run-dir ./r
+coder-eval evaluate ./r/default/hello_date/00
+coder-eval aggregate ./r                      # run.json now reports the verdict
+```
+
+**Run-directory mode** rebuilds the task from the run's own recorded
+`task_config.resolved`, not by re-reading the YAML. That is what makes the grade
+describe the run that happened: variant overrides, `-D` flags and dataset row
+expansion are already baked into `resolved`, so re-loading the source would
+silently grade a *different* task. The run's trajectory is restored too, so
+criteria that read the agent's tool calls (`command_executed`, `skill_triggered`,
+judges with trajectory) score exactly as they would have during the run.
+
+It writes the verdict back into the run's `task.json` and keeps the pre-grade
+record beside it as `task.execute.json`. Writing back in place is what makes
+`aggregate` free — no new flag, no second copy of the results.
+
+Passing a task file **over** a run directory re-grades it with different
+criteria, reusing the trajectory and workspace of a run you already paid for:
+
+```bash
+coder-eval evaluate tasks/hello_date.edited.yaml ./r/default/hello_date/00
 ```
 
-Runs a task's success criteria against a directory without an agent — useful for
-testing criterion definitions, validating task configs, or scoring code that was
-already written.
+**In-place vs. copy.** The two-argument form copies your directory into a fresh
+sandbox (criteria can mutate the target, and it is your own tree). Run-directory
+mode grades **in place**, because copying filters build output — `node_modules`,
+`dist`, `build`, `.venv`, `.git` are all on the default ignore list, so a
+criterion like `test -f dist/bundle.js` would fail as a *copying artifact*
+rather than as a verdict. Override either default with `--in-place` / `--copy`.
 
 | Flag | Description |
 | --- | --- |
-| `--preserve / --no-preserve` | Preserve sandbox after evaluation (default: preserve) |
-| `--run-dir` | Custom run directory (default: auto-generated timestamped dir in `runs/`). |
+| `--workspace` | Grade this directory instead of the run's own artifacts (run-directory mode only). |
+| `--in-place / --copy` | Grade where the files are, or copy first. Default: in-place for a run directory, copy for a plain work directory. |
+| `--preserve / --no-preserve` | Preserve sandbox after evaluation (default: preserve). Ignored when grading in place — an adopted directory is never moved or deleted. |
+| `--run-dir` | Where the graded `task.json` lands (default: auto-generated timestamped dir in `runs/`). |
 | `--verbose, -v` | DEBUG-level logging |
 
+A re-grade refuses to run if the task's `reference:` directory changed since the
+run (digest mismatch) — grading then would score the agent's old work against a
+new answer key.
+
 ### `coder-eval report` — view results
 
 ```bash

evalboard/app/runs/[id]/run-view.tsx

@@ -38,6 +38,9 @@ export interface RunMetrics {
     passed: number;
     failed: number;
     errored: number;
+    // Rows that ran but were never scored (`coder-eval execute`). Excluded from
+    // both sides of `pct`, so a fully ungraded run reports 0 of 0, not 0%.
+    ungraded: number;
     failedTotal: number;
     pct: number;
     // Per-task view of pass rate for repeated runs: distinct task_ids, and how
@@ -72,6 +75,7 @@ export function computeRunMetrics(tasks: TaskResultSummary[]): RunMetrics {
     let passed = 0;
     let failed = 0;
     let errored = 0;
+    let ungraded = 0;
     let cost = 0;
     let durationSum = 0;
     const costSamples: number[] = [];
@@ -80,6 +84,11 @@ export function computeRunMetrics(tasks: TaskResultSummary[]): RunMetrics {
         const cat = statusCategory(t.status);
         if (cat === "passed") passed++;
         else if (cat === "error") errored++;
+        // An ungraded row (`coder-eval execute`) was never scored. It leaves
+        // BOTH sides of the rate — the `else failed++` below would otherwise
+        // count it as a failure AND keep it in the denominator, rendering a
+        // clean execute run as 0% pass, N failed.
+        else if (cat === "ungraded") ungraded++;
         else failed++;
         if (t.matureSkipped) continue;
         if (t.totalCostUsd != null) {
@@ -91,13 +100,15 @@ export function computeRunMetrics(tasks: TaskResultSummary[]): RunMetrics {
             durSamples.push(t.durationSeconds);
         }
     }
+    const graded = total - ungraded;
     return {
         total,
         passed,
         failed,
         errored,
+        ungraded,
         failedTotal: failed + errored,
-        pct: total ? (passed / total) * 100 : 0,
+        pct: graded ? (passed / graded) * 100 : 0,
         ...(() => {
             // Per-task rollup (any replicate passed → task passed) via the shared
             // helper, so the run tile and the grid badge apply the same rule.