feat(evalboard): render multi-variant runs, one row per (task, arm)

evalboard/README.md

@@ -57,7 +57,53 @@ show up in the index — empty shells and the `latest` symlink are filtered out.
 
 `<task-id>` is the same string the eval framework writes to
 `task_results[].task_id` (e.g., `skill-flow-calculator`) and equals the
-subdir name under `<run-id>/default/`.
+subdir name under `<run-id>/<variant-id>/`.
+
+## Variants (A/B runs)
+
+A coder_eval experiment that declares `variants:` runs every task once per arm
+and writes each arm to its own subtree (`<run-id>/<variant-id>/<task-id>/<NN>/`),
+stamping `variant_id` on every `run.json` row. An experiment with no `variants:`
+still writes one arm, named `default` — which is why a single-arm run and a
+multi-arm run have the same shape on disk.
+
+Evalboard reads that directly:
+
+- The run page's **Pass rate tile reports one entry per arm** (side by side, so
+  the tile keeps the height of the single-arm version and the tiles beside it are
+  not stretched), and reports no pooled rate at all. A blended rate would average configurations that were
+  deliberately made to differ (and would move when the arms are merely
+  reordered), so on a variant run it is not a number anyone wants. The tile
+  states the observed `spread` between arms and nothing more.
+- **Total cost and Time keep their pooled totals** and carry the per-arm split on
+  their sub-line, in place of p50/p90. A run's spend is a real operational
+  number however many arms produced it, in a way a run's pass rate is not.
+- The task grid gains a **Variant** column and keeps one row per (task, arm).
+  Replicates of one arm still collapse into a single row; **arms never collapse
+  into each other** — that difference is the measurement. The default ordering
+  ranks a *task* by its worst arm rather than ranking each row on its own, so
+  failures still sort to the top while a task's arms stay adjacent. Ranking rows
+  independently splits exactly the pairs worth reading: a task whose arms
+  disagree ends up with one row at the top of the grid and the other at the
+  bottom.
+- Task detail is addressed by `?v=<variant-id>` alongside `?r=<replicate>`, so
+  each arm opens its own transcript, log, criteria and artifacts. A link that
+  names no arm resolves to the task's first — `default` on an ordinary run, the
+  leading arm on an experiment run, whose arms are all named and which therefore
+  has no `default` subtree for a bare link to land in. A link naming an arm the
+  run does not have is a 404, not a fallback: the point of addressing an arm is
+  that you get that arm or nothing.
+
+Every one of those is inert on a run without variants: the column is dropped, the
+tiles keep their existing pooled numbers and percentiles, and no link carries
+`?v=`. Cross-run comparison (two separate run ids) is a different feature and is
+not what this does.
+
+Statistics are deliberately not computed here. Whether a gap between arms is
+real is a question about variance, and coder_eval already answers it in the
+experiment report it writes beside `run.json` (`experiment.md` — win rates,
+per-task comparison, most divergent tasks, paired comparison with Welch t-test
+and bootstrap CIs).
 
 ## Sources
 
@@ -104,9 +150,10 @@ Two invariants worth preserving if you add a source:
 
 - `/api/file?run=<id>&path=<relpath>[&src=<source>]` serves `.flow`, `.uipx`,
   etc. with path-traversal guard (`resolveSafePath`).
-- `/api/download?run=<id>[&task=<id>][&src=<source>]` streams a zip of a task
-  folder (with `task`) or the whole run (without). Files are gathered by `collectTaskFiles`
-  / `collectRunFiles`, which reuse the `walkArtifacts` noise filter, and zipped
-  by `lib/zip.ts` (a dependency-free DEFLATE writer).
+- `/api/download?run=<id>[&task=<id>][&v=<variant>][&src=<source>]` streams a zip
+  of a task folder (with `task`, from the named arm) or the whole run (without).
+  Files are gathered by `collectTaskFiles` / `collectRunFiles`, which reuse the
+  `walkArtifacts` noise filter, and zipped by `lib/zip.ts` (a dependency-free
+  DEFLATE writer).
 - Pass rows render green (`bg-green-50 text-green-700`), failures render red
   (`bg-red-50 text-red-700`), on a white background.

evalboard/app/api/download/route.ts

@@ -1,36 +1,46 @@
 import { promises as fs } from "node:fs";
 import { NextResponse } from "next/server";
 import { collectRunFiles, collectTaskFiles } from "@/lib/runs";
+import { DEFAULT_VARIANT_ID, isValidVariantId } from "@/lib/variants";
 import { sourceById } from "@/lib/sources";
 import { createZip, type ZipEntry } from "@/lib/zip";
 
 export const dynamic = "force-dynamic";
 
 // Bundle a task folder, or a whole run, into a zip download.
-//   ?run=<id>&task=<id>  → just that task's folder (default/<taskId>/)
-//   ?run=<id>            → the entire run folder (run.json + every task dir)
+//   ?run=<id>&task=<id>[&v=<variant>]  → that task's folder (<variant>/<taskId>/)
+//   ?run=<id>                          → the entire run folder (run.json + every task dir)
 // minus the usual scaffolding noise, from the container named by ?src (the
 // skills nightly when absent). In blob mode the collect* helpers fetch the
 // needed blobs first, so this mirrors what the page would load.
 export async function GET(req: Request) {
     const url = new URL(req.url);
     const runId = url.searchParams.get("run");
     const taskId = url.searchParams.get("task");
+    // Experiment arm. Absent / malformed → the single arm a non-experiment run
+    // writes, so pre-variant download URLs resolve unchanged.
+    const v = url.searchParams.get("v");
+    const variantId = isValidVariantId(v) ? v : DEFAULT_VARIANT_ID;
     const source = sourceById(url.searchParams.get("src"));
     if (!runId) {
         return new NextResponse("missing run", { status: 400 });
     }
 
     const files = taskId
-        ? await collectTaskFiles(runId, taskId, source)
+        ? await collectTaskFiles(runId, taskId, source, variantId)
         : await collectRunFiles(runId, source);
     if (!files) {
         return new NextResponse("not found", { status: 404 });
     }
 
     // Top-level folder inside the archive: the task id for a task download,
-    // the run id for a whole-run download.
-    const root = taskId ?? runId;
+    // the run id for a whole-run download. A non-default arm is named too, so
+    // two arms of the same task don't produce two identically-named zips.
+    const root = taskId
+        ? variantId === DEFAULT_VARIANT_ID
+            ? taskId
+            : `${taskId}__${variantId}`
+        : runId;
     const entries: ZipEntry[] = [];
     for (const f of files) {
         const data = await fs.readFile(f.abs).catch(() => null);

evalboard/app/runs/[id]/[...task]/page.tsx

@@ -7,7 +7,9 @@ import {
     readTaskDetail,
     readTaskReplicates,
     replicateDirName,
+    resolveVariantId,
 } from "@/lib/runs";
+import { DEFAULT_VARIANT_ID, isValidVariantId } from "@/lib/variants";
 import { readTaskReview } from "@/lib/reviews";
 import { sourceById } from "@/lib/sources";
 import { scalarParam, withSource } from "@/app/_lib/source-param";
@@ -37,10 +39,10 @@ export default async function TaskPage({
     searchParams,
 }: {
     params: Promise<{ id: string; task: string[] }>;
-    searchParams: Promise<{ r?: string; src?: string | string[] }>;
+    searchParams: Promise<{ r?: string; v?: string; src?: string | string[] }>;
 }) {
     const { id, task: taskSegments } = await params;
-    const { r, src } = await searchParams;
+    const { r, v, src } = await searchParams;
     // Which container this run lives in. Unknown/absent coerces to the skills
     // nightly, so every URL that predates the Scribe tab keeps resolving as-is.
     const source = sourceById(scalarParam(src));
@@ -52,28 +54,63 @@ export default async function TaskPage({
     const parsedR = Number(r);
     const replicate =
         r != null && Number.isInteger(parsedR) && parsedR >= 0 ? parsedR : 0;
-    const task = await readTaskDetail(id, taskId, replicate, source);
+    // Experiment arm from ?v=NAME. A run with variants stores each arm's content
+    // under its own <runId>/<variantId>/ subtree and repeats the task id once per
+    // arm, so without this both arms would resolve to the same transcript. An
+    // absent or malformed value names no arm, and resolveVariantId picks the
+    // task's first — `default` on an ordinary run, so every pre-variant URL
+    // resolves exactly as it did, and the leading arm on an experiment run,
+    // whose arms are named and would otherwise leave a bare link with nothing
+    // to match.
+    const variantId = await resolveVariantId(
+        id,
+        taskId,
+        isValidVariantId(v) ? v : null,
+        source,
+    );
+    const task = await readTaskDetail(id, taskId, replicate, source, variantId);
     if (!task) notFound();
 
     // Replicate indices available for this task — drives the run selector below.
     // [0] (or fewer) for a non-repeated task, so the selector self-hides.
-    const replicates = await readTaskReplicates(id, taskId, source);
+    const replicates = await readTaskReplicates(id, taskId, source, variantId);
 
-    // variant is always "default" here; the replicate selects the <NN>/ dir.
     // readTaskReview returns null for older runs that predate the review feature.
     const review = await readTaskReview(
         id,
-        "default",
+        variantId,
         taskId,
         replicateDirName(replicate),
         source,
     );
 
-    const log = await readLogTail(id, taskId, replicate, undefined, source);
+    const log = await readLogTail(
+        id,
+        taskId,
+        replicate,
+        undefined,
+        source,
+        variantId,
+    );
     const conversation = parseConversation(
-        await readConversationLog(id, taskId, replicate, undefined, source),
+        await readConversationLog(
+            id,
+            taskId,
+            replicate,
+            undefined,
+            source,
+            variantId,
+        ),
     );
     const { flowDebug } = task;
+    // Only a multi-variant run names its arm in a URL. Omitting the param on a
+    // default-arm run keeps every link byte-identical to what it was before
+    // variants were addressable.
+    const variantQuery =
+        variantId === DEFAULT_VARIANT_ID
+            ? ""
+            : `&v=${encodeURIComponent(variantId)}`;
+    const showVariant = variantId !== DEFAULT_VARIANT_ID;
 
     return (
         <div className="space-y-6">
@@ -86,6 +123,14 @@ export default async function TaskPage({
                 </Link>
                 <span className="text-gray-300">/</span>
                 <span className="font-mono text-gray-700">{taskId}</span>
+                {showVariant && (
+                    <>
+                        <span className="text-gray-300">/</span>
+                        <span className="font-mono text-gray-700">
+                            {variantId}
+                        </span>
+                    </>
+                )}
                 {replicates.length > 1 && (
                     <>
                         <span className="text-gray-300">/</span>
@@ -112,7 +157,7 @@ export default async function TaskPage({
                                 <Link
                                     key={ri}
                                     href={withSource(
-                                        `/runs/${id}/${taskId}?r=${ri}`,
+                                        `/runs/${id}/${taskId}?r=${ri}${variantQuery}`,
                                         source.id,
                                     )}
                                     // Keep the scroll position when switching runs
@@ -147,7 +192,7 @@ export default async function TaskPage({
                             href={withSource(
                                 `/api/download?run=${encodeURIComponent(
                                     id,
-                                )}&task=${encodeURIComponent(taskId)}`,
+                                )}&task=${encodeURIComponent(taskId)}${variantQuery}`,
                                 source.id,
                             )}
                             className="ml-auto inline-flex items-center gap-1.5 rounded-md border border-gray-300 bg-white px-3 py-1.5 text-sm font-medium text-gray-700 hover:bg-gray-50 hover:text-studio-blue"
@@ -160,6 +205,7 @@ export default async function TaskPage({
                 <div className="text-xs text-gray-500 tabular-nums font-mono flex flex-wrap items-baseline gap-x-1.5 gap-y-1">
                     <span>
                         {taskId} · run {id}
+                        {showVariant && ` · variant ${variantId}`}
                         {replicates.length > 1 && ` · replicate ${replicate}`}
                     </span>
                     {/* Component SHAs point at internal tooling; internal-only.

evalboard/app/runs/[id]/__tests__/run-view.render.test.tsx

@@ -18,6 +18,7 @@ function row(
 ): TaskResultSummary {
     return {
         taskId,
+        variantId: null,
         replicateIndex: null,
         status: "SUCCESS",
         weightedScore: 1.0,
@@ -83,3 +84,55 @@ describe("RunView — Pass-rate / Failed tiles for repeated runs", () => {
         expect(screen.queryByText(/replicate runs/)).toBeNull();
     });
 });
+
+describe("RunView — multi-variant runs replace the pooled pass rate", () => {
+    // Arm A passes everything, arm B fails everything. The pooled rate would be
+    // 50%, which describes neither configuration; the whole point of the tile
+    // change is that this number no longer appears anywhere on the page.
+    const AB = [
+        row("X", { variantId: "A", status: "SUCCESS", totalCostUsd: 0.1, durationSeconds: 1 }),
+        row("Y", { variantId: "A", status: "SUCCESS", totalCostUsd: 0.1, durationSeconds: 1 }),
+        row("X", { variantId: "B", status: "FAILURE", totalCostUsd: 0.3, durationSeconds: 5 }),
+        row("Y", { variantId: "B", status: "FAILURE", totalCostUsd: 0.3, durationSeconds: 5 }),
+    ];
+
+    test("each arm gets its own rate and the blended rate is gone", () => {
+        render(<RunView sourceId="skills" runId="r1" tasks={AB} />);
+
+        expect(screen.getByText("100%")).toBeInTheDocument();
+        expect(screen.getByText("0%")).toBeInTheDocument();
+        // The blended 50% must not be rendered at all.
+        expect(screen.queryByText("50%")).toBeNull();
+        expect(screen.getByText(/2 arms · spread 100 pts/)).toBeInTheDocument();
+    });
+
+    test("spend and elapsed time keep their pooled total, split by arm on the sub-line", () => {
+        render(<RunView sourceId="skills" runId="r1" tasks={AB} />);
+
+        // Pooled totals stay: a run's cost is real however many arms produced it.
+        expect(screen.getByText("$0.80")).toBeInTheDocument();
+        expect(screen.getByText("12s")).toBeInTheDocument();
+        // ...with the per-arm split replacing p50/p90, which would describe a
+        // pooled population that does not exist.
+        expect(screen.getByText("A $0.20 · B $0.60")).toBeInTheDocument();
+        expect(screen.getByText("A 2s · B 10s")).toBeInTheDocument();
+        expect(screen.queryByText(/p50/)).toBeNull();
+    });
+
+    test("a single-arm run is untouched: pooled rate and p50/p90 as before", () => {
+        render(
+            <RunView
+                sourceId="skills"
+                runId="r1"
+                tasks={[
+                    row("X", { variantId: "only", status: "SUCCESS" }),
+                    row("Y", { variantId: "only", status: "FAILURE" }),
+                ]}
+            />,
+        );
+        expect(screen.getByText("50%")).toBeInTheDocument();
+        expect(screen.queryByText(/arms · spread/)).toBeNull();
+        // Both the cost and time tiles carry one, hence getAll.
+        expect(screen.getAllByText(/p50/).length).toBe(2);
+    });
+});