From 7b230bc6a05273f562aeea40385832418d50c89b Mon Sep 17 00:00:00 2001 From: Bai Li Date: Wed, 2 Sep 2026 16:43:07 -0700 Subject: [PATCH 1/6] perf(evalboard): stop re-reading the whole run store on every render MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The front page took 5-8s cold and ~1.25s warm, and the split was exactly the 5-minute per-run cache TTL. Four causes, all redundant IO against the `/home` Azure Files mount: - The ad-hoc section loaded EVERY non-date-shaped candidate before sorting — 165 runs / ~294 MB of run.json — to render ten rows. It now orders candidates by the date in the id and loads only the newest page's worth. run.json `start_time` is still the authoritative sort. - Per-run projections were cached for 5 minutes regardless of age, so every visit after a short pause re-read the whole window. A run.json is written once at the end of a run, so runs older than 24h now cache for a day. /api/refresh evicts by tag, keeping the escape hatch. - readRunSummary and readRunTasks each read and parsed the same multi-MB run.json; the run page did both on every request. Memoized per request with react/cache. - findMatureSourceRuns walked up to 20 earlier run.jsons serially, and only short-circuits once every mature task resolves, which at ~50% maturity usually never happens. Now reads in concurrent batches, consumed in index order so "most recent execution wins" is unchanged. Filtering within a run also round-tripped the server for nothing: the tag/search params on that page are read entirely client-side, but committing them through router.replace re-ran the whole force-dynamic page. Switched to the native History API, which Next syncs into useSearchParams without a fetch. Whole-run download is removed — button, route branch, collectRunFiles and ensureRunDir. A nightly run is ~10k blobs / ~400 MB, and zipping it meant an uncapped blob fan-out, a stat-per-file walk over Azure Files, and the entire archive buffered in memory before the first byte was sent. Per-task download stays and is unchanged; its CRC-32 now comes from zlib instead of a per-byte JS loop (~47x, measured). Co-Authored-By: Claude Opus 5 (1M context) --- evalboard/app/api/download/route.ts | 33 ++-- .../app/api/refresh/__tests__/route.test.ts | 31 ++++ evalboard/app/api/refresh/route.ts | 10 +- evalboard/app/runs/[id]/page.tsx | 29 ++-- evalboard/app/runs/[id]/run-view.tsx | 42 +++-- evalboard/lib/__tests__/collect.test.ts | 22 +-- evalboard/lib/__tests__/overview.test.ts | 50 ++++++ evalboard/lib/__tests__/runs.test.ts | 40 +++++ evalboard/lib/__tests__/zip.test.ts | 25 +++ evalboard/lib/blob.ts | 28 +--- evalboard/lib/overview.ts | 146 +++++++++++++++--- evalboard/lib/runs.ts | 79 +++++----- evalboard/lib/zip.ts | 37 ++--- 13 files changed, 405 insertions(+), 167 deletions(-) diff --git a/evalboard/app/api/download/route.ts b/evalboard/app/api/download/route.ts index 5aa7a509..65175053 100644 --- a/evalboard/app/api/download/route.ts +++ b/evalboard/app/api/download/route.ts @@ -1,18 +1,24 @@ import { promises as fs } from "node:fs"; import { NextResponse } from "next/server"; -import { collectRunFiles, collectTaskFiles } from "@/lib/runs"; +import { collectTaskFiles } from "@/lib/runs"; import { DEFAULT_VARIANT_ID, isValidVariantId } from "@/lib/variants"; import { sourceById } from "@/lib/sources"; import { createZip, type ZipEntry } from "@/lib/zip"; export const dynamic = "force-dynamic"; -// Bundle a task folder, or a whole run, into a zip download. +// Bundle ONE task folder into a zip download. // ?run=&task=[&v=] → that task's folder (//) -// ?run= → the entire run folder (run.json + every task dir) // minus the usual scaffolding noise, from the container named by ?src (the -// skills nightly when absent). In blob mode the collect* helpers fetch the -// needed blobs first, so this mirrors what the page would load. +// skills nightly when absent). In blob mode collectTaskFiles fetches the needed +// blobs first, so this mirrors what the page would load. +// +// `task` is REQUIRED. Omitting it used to mean "zip the whole run", which for a +// nightly meant ~10k blobs / ~400 MB: an uncapped fan-out of blob downloads, a +// stat-per-file walk over Azure Files, and the entire archive buffered in +// memory before any of it was sent. That path is gone, along with the run +// page's button for it — a task folder is a handful of files and returns in +// well under a second. To inspect a whole run, use the blob container directly. export async function GET(req: Request) { const url = new URL(req.url); const runId = url.searchParams.get("run"); @@ -25,22 +31,19 @@ export async function GET(req: Request) { if (!runId) { return new NextResponse("missing run", { status: 400 }); } + if (!taskId) { + return new NextResponse("missing task", { status: 400 }); + } - const files = taskId - ? await collectTaskFiles(runId, taskId, source, variantId) - : await collectRunFiles(runId, source); + const files = await collectTaskFiles(runId, taskId, source, variantId); if (!files) { return new NextResponse("not found", { status: 404 }); } - // Top-level folder inside the archive: the task id for a task download, - // the run id for a whole-run download. A non-default arm is named too, so + // Top-level folder inside the archive. A non-default arm is named too, so // two arms of the same task don't produce two identically-named zips. - const root = taskId - ? variantId === DEFAULT_VARIANT_ID - ? taskId - : `${taskId}__${variantId}` - : runId; + const root = + variantId === DEFAULT_VARIANT_ID ? taskId : `${taskId}__${variantId}`; const entries: ZipEntry[] = []; for (const f of files) { const data = await fs.readFile(f.abs).catch(() => null); diff --git a/evalboard/app/api/refresh/__tests__/route.test.ts b/evalboard/app/api/refresh/__tests__/route.test.ts index 601315e3..70902a49 100644 --- a/evalboard/app/api/refresh/__tests__/route.test.ts +++ b/evalboard/app/api/refresh/__tests__/route.test.ts @@ -9,13 +9,24 @@ import { test, vi, } from "vitest"; +import { runCacheTag } from "@/lib/overview"; import { SCRIBE_SOURCE, runsDirFor } from "@/lib/sources"; +// The route calls revalidateTag to evict the run's memoized front-page +// projection alongside its on-disk copy. The real implementation asserts it is +// inside a Next request/render store, which a direct POST() call is not, so it +// is stubbed here and asserted on instead. +const revalidateTag = vi.fn(); +vi.mock("next/cache", () => ({ + revalidateTag: (tag: string) => revalidateTag(tag), +})); + // RUNS_DIR and LOCAL_RUNS_DIR are module-level consts read from env at import // time, so each scenario sets env, resets the module registry, then imports a // fresh copy of the route. async function loadPost() { vi.resetModules(); + revalidateTag.mockClear(); return (await import("../route")).POST; } @@ -117,6 +128,26 @@ describe("POST /api/refresh", () => { await expect(fs.access(runDir)).rejects.toThrow(); }); + // Evicting only the on-disk copy is not enough: the front page reads a + // memoized projection of run.json that a settled run holds for a day + // (lib/overview.ts::perRunRevalidate), so without this the refresh + // button is a no-op for anything older than 24h. + test("valid run -> the memoized projection is invalidated too", async () => { + const POST = await loadPost(); + await POST(post("2026-06-01_04-04-22")); + + expect(revalidateTag).toHaveBeenCalledWith( + runCacheTag("skills", "2026-06-01_04-04-22"), + ); + }); + + test("a rejected id revalidates nothing", async () => { + const POST = await loadPost(); + await POST(post("..")); + + expect(revalidateTag).not.toHaveBeenCalled(); + }); + // Run ids collide across sources (both suites name runs // YYYY-MM-DD_HH-MM-SS), so the source decides WHICH cached copy is // evicted. Getting this wrong deletes a run nobody asked about and diff --git a/evalboard/app/api/refresh/route.ts b/evalboard/app/api/refresh/route.ts index 4dd1b22e..00425908 100644 --- a/evalboard/app/api/refresh/route.ts +++ b/evalboard/app/api/refresh/route.ts @@ -1,5 +1,7 @@ +import { revalidateTag } from "next/cache"; import { NextResponse } from "next/server"; import { LOCAL_RUNS_DIR } from "@/lib/blob"; +import { runCacheTag } from "@/lib/overview"; import { RUNS_DIR, clearRunCacheDir } from "@/lib/runs"; import { runsDirFor, sourceById } from "@/lib/sources"; @@ -37,8 +39,12 @@ export async function POST(req: Request) { if (!(await clearRunCacheDir(runsDirFor(RUNS_DIR, source), runId))) { return NextResponse.json({ error: "invalid run" }, { status: 400 }); } + // Evicting the on-disk copy is only half the job: the front page reads a + // memoized PROJECTION of it (lib/overview.ts::cachedLoadPerRunFor), and a + // settled run's projection is held for a day. Drop that entry too, or the + // refresh silently does nothing for every run older than 24h. + revalidateTag(runCacheTag(source.id, runId)); // Re-download is lazy on next render. The run page is force-dynamic and - // reads the sidecars straight from disk, so it is fresh immediately; the - // homepage's per-run title list self-heals within its 5-minute revalidate. + // reads the sidecars straight from disk, so it is fresh immediately. return new Response(null, { status: 204 }); } diff --git a/evalboard/app/runs/[id]/page.tsx b/evalboard/app/runs/[id]/page.tsx index 8088ffbe..70356c3a 100644 --- a/evalboard/app/runs/[id]/page.tsx +++ b/evalboard/app/runs/[id]/page.tsx @@ -10,7 +10,7 @@ import { } from "@/lib/runs"; import { readRunReviewIndex, indexByTask, tagCountsForRun } from "@/lib/reviews"; import { sourceById } from "@/lib/sources"; -import { scalarParam, withSource } from "@/app/_lib/source-param"; +import { scalarParam } from "@/app/_lib/source-param"; import { fmtRunTime } from "@/lib/format"; import { AnalysisPanel } from "./analysis-panel"; import { RefreshButton } from "./refresh-button"; @@ -87,22 +87,23 @@ export default async function RunPage({ Ad-hoc )} - {/* Refresh re-pulls the run from blob storage and Download - zips it — both are internal-hosting surfaces (the public - OSS edition has no blob backend). See lib/edition.ts. */} + {/* Refresh re-pulls the run from blob storage — an + internal-hosting surface (the public OSS edition has no + blob backend). See lib/edition.ts. + + There is deliberately no whole-run download here. A + nightly run is ~10k blobs / ~400 MB, and zipping it meant + fetching every one of them with no concurrency cap, + walking the tree with a stat per file over Azure Files, + and buffering the entire archive in memory before the + first byte reached the browser — minutes of apparent + hang, against a process that already peaks at 2.7 GB RSS. + Per-task download (on the task page) is the supported + shape; it bundles a handful of files and returns in + well under a second. */} {isInternal && ( )} diff --git a/evalboard/app/runs/[id]/run-view.tsx b/evalboard/app/runs/[id]/run-view.tsx index a5171fe5..7b0d4249 100644 --- a/evalboard/app/runs/[id]/run-view.tsx +++ b/evalboard/app/runs/[id]/run-view.tsx @@ -1,6 +1,6 @@ "use client"; -import { usePathname, useRouter, useSearchParams } from "next/navigation"; +import { usePathname, useSearchParams } from "next/navigation"; import { useCallback, useMemo, useState } from "react"; import type { ActivationScore, TaskResultSummary } from "@/lib/runs"; import type { ReviewIndexEntry } from "@/lib/reviews-types"; @@ -273,7 +273,6 @@ export function RunView({ // the one call site instead. sourceId: string; }) { - const router = useRouter(); const pathname = usePathname(); const searchParams = useSearchParams(); @@ -295,6 +294,33 @@ export function RunView({ const q = searchParams.get("q") ?? ""; const [showAllTags, setShowAllTags] = useState(false); + // Commit a filter change to the URL WITHOUT a server round-trip. + // + // Every reader of `tags` / `rtags` / `q` on this page is client-side — the + // grid is filtered by `filtered` below, out of rows the server already + // sent. The page's server component reads only `src`. So `router.replace` + // bought nothing and cost a full re-render of a force-dynamic route: two + // separate parses of the same multi-MB run.json (readRunSummary and + // readRunTasks each read it), the activation sub-run, the review index, and + // the mature-source scan's serial walk back through earlier runs — all to + // return markup this component recomputes locally anyway. + // + // The native History API keeps the URL shareable and the back button + // working; Next syncs `useSearchParams` off pushState/replaceState, so this + // component still re-renders. `replaceState` (not `push`) preserves the old + // behavior of not stacking a history entry per chip click. + const setSearchParams = useCallback( + (params: URLSearchParams) => { + const qs = params.toString(); + window.history.replaceState( + null, + "", + qs ? `${pathname}?${qs}` : pathname, + ); + }, + [pathname], + ); + const updateParam = useCallback( (key: string, next: string[]) => { // Read the live URL on commit so a concurrent debounced write @@ -302,12 +328,9 @@ export function RunView({ const params = new URLSearchParams(window.location.search); if (next.length === 0) params.delete(key); else params.set(key, next.join(",")); - const qs = params.toString(); - router.replace(qs ? `${pathname}?${qs}` : pathname, { - scroll: false, - }); + setSearchParams(params); }, - [pathname, router], + [setSearchParams], ); const toggleTag = useCallback( @@ -339,9 +362,8 @@ export function RunView({ params.delete("q"); params.delete("tags"); params.delete("rtags"); - const qs = params.toString(); - router.replace(qs ? `${pathname}?${qs}` : pathname, { scroll: false }); - }, [pathname, router]); + setSearchParams(params); + }, [setSearchParams]); // Skill is the primary group; everything else is a secondary tag. The // secondary "Tags" rail excludes the per-task skill so it doesn't echo diff --git a/evalboard/lib/__tests__/collect.test.ts b/evalboard/lib/__tests__/collect.test.ts index 78fcf823..04da4a5d 100644 --- a/evalboard/lib/__tests__/collect.test.ts +++ b/evalboard/lib/__tests__/collect.test.ts @@ -3,7 +3,7 @@ import os from "node:os"; import path from "node:path"; import { afterEach, beforeEach, describe, expect, test, vi } from "vitest"; -// collectTaskFiles / collectRunFiles read RUNS_DIR, which lib/blob.ts resolves +// collectTaskFiles reads RUNS_DIR, which lib/blob.ts resolves // from EVALBOARD_LOCAL_RUNS_DIR at *import* time. So each test stubs the env to // a throwaway runs dir, then dynamically imports a fresh module copy // (vi.resetModules) so RUNS_DIR picks up that path. @@ -67,23 +67,3 @@ describe("collectTaskFiles", () => { expect(await collectTaskFiles(RUN, "nope")).toBeNull(); }); }); - -describe("collectRunFiles", () => { - test("returns run-level files + every task's files, minus noise", async () => { - const { collectRunFiles } = await loadRuns(); - const files = await collectRunFiles(RUN); - const rels = files?.map((f) => f.relPath).sort(); - expect(rels).toEqual([ - "analysis.md", - `default/${TASK}/00/artifacts/main.py`, - `default/${TASK}/00/task.json`, - `default/${TASK}/00/task.log`, - "run.json", - ]); - }); - - test("returns null for a missing run", async () => { - const { collectRunFiles } = await loadRuns(); - expect(await collectRunFiles("9999-99-99_00-00-00")).toBeNull(); - }); -}); diff --git a/evalboard/lib/__tests__/overview.test.ts b/evalboard/lib/__tests__/overview.test.ts index 52507dca..d012d39d 100644 --- a/evalboard/lib/__tests__/overview.test.ts +++ b/evalboard/lib/__tests__/overview.test.ts @@ -1,5 +1,6 @@ import { describe, expect, test, vi } from "vitest"; import { + adhocRunDate, avgRunSuccessRate, buildAdhocRows, buildTagTaskRows, @@ -1206,3 +1207,52 @@ describe("projectRunRow", () => { }); }); }); + +// The front page used to load EVERY ad-hoc candidate before sorting — 165 runs +// / ~294 MB of run.json — to render ten rows. The load is now bounded by an +// id-derived date, so these pin the extraction that decides what gets read. +describe("adhocRunDate", () => { + test("reads the date out of the ad-hoc id shapes actually in the container", () => { + const at = (id: string) => adhocRunDate(id)?.toISOString() ?? null; + // The dominant shape: `adhoc-` prefix + a full date_time. + expect(at("adhoc-2026-09-02_21-59-13")).toBe( + "2026-09-02T21:59:13.000Z", + ); + // Legacy hand-named runs: leading date, no time -> start of that day. + expect(at("2026-05-28_skills-full-codex-gpt54")).toBe( + "2026-05-28T00:00:00.000Z", + ); + // Date anywhere in the id, not just the head. + expect(at("haiku45-skills-suite-2026-05-28")).toBe( + "2026-05-28T00:00:00.000Z", + ); + }); + + test("null for an id carrying no date", () => { + // These are always loaded rather than ordered out, since there is no + // key to order them by. + expect(adhocRunDate("sdk-live-r2-final")).toBeNull(); + expect(adhocRunDate("adhoc-flow-v2-preview-20260820")).toBeNull(); + expect(adhocRunDate("deploys")).toBeNull(); + }); + + test("orders newest-first the same way run start_time does", () => { + const ids = [ + "2026-05-28_skills-full-codex-gpt54", + "adhoc-2026-09-02_21-59-13", + "adhoc-2026-09-02_11-20-47", + "adhoc-2026-08-27_19-35-31", + ]; + const sorted = [...ids].sort( + (a, b) => + (adhocRunDate(b)?.getTime() ?? 0) - + (adhocRunDate(a)?.getTime() ?? 0), + ); + expect(sorted).toEqual([ + "adhoc-2026-09-02_21-59-13", + "adhoc-2026-09-02_11-20-47", + "adhoc-2026-08-27_19-35-31", + "2026-05-28_skills-full-codex-gpt54", + ]); + }); +}); diff --git a/evalboard/lib/__tests__/runs.test.ts b/evalboard/lib/__tests__/runs.test.ts index 991419bb..e5ec194a 100644 --- a/evalboard/lib/__tests__/runs.test.ts +++ b/evalboard/lib/__tests__/runs.test.ts @@ -553,6 +553,46 @@ describe("findMatureSourceRuns", () => { expect(await findMatureSourceRuns(["a"], "nope", deps)).toEqual({}); }); + // The scan reads in concurrent batches but must still consume them in index + // order — otherwise "the most recent run that executed this task" becomes + // "whichever read resolved first", which is nondeterministic and wrong. + test("within one batch, the newer run still wins", async () => { + const readRun = vi.fn(async (id: string) => { + // Resolve out of index order, so a scan that trusted completion + // order instead of index order would answer r2. + if (id === "r2") return runs.r2; + await Promise.resolve(); + return runs[id] ?? null; + }); + const out = await findMatureSourceRuns(["a"], "r5", { + listIds: async () => ids, + readRun, + }); + expect(out).toEqual({ a: "r3" }); + }); + + test("resolves across a batch boundary", async () => { + // 7 runs, so the first batch of 5 cannot resolve the task and the walk + // has to continue into the second. + const longIds = ["s7", "s6", "s5", "s4", "s3", "s2", "s1"]; + const skipped = { task_results: [{ task_id: "a", mature_skipped: true }] }; + const longRuns: Record = { + s7: skipped, + s6: skipped, + s5: skipped, + s4: skipped, + s3: skipped, + s2: skipped, + s1: { task_results: [{ task_id: "a" }] }, + }; + const out = await findMatureSourceRuns(["a"], "s7", { + listIds: async () => longIds, + readRun: async (id: string) => + (longRuns[id] as (typeof runs)[string]) ?? null, + }); + expect(out).toEqual({ a: "s1" }); + }); + test("a task with no earlier execution is omitted (stays non-clickable)", async () => { const onlyB = { listIds: async () => ["r2", "r1"], diff --git a/evalboard/lib/__tests__/zip.test.ts b/evalboard/lib/__tests__/zip.test.ts index 573632d1..2dcd1fdd 100644 --- a/evalboard/lib/__tests__/zip.test.ts +++ b/evalboard/lib/__tests__/zip.test.ts @@ -43,6 +43,31 @@ describe("createZip", () => { expect(files["task-1/artifacts/main.py"]).toBe("print('hi')\n"); }); + // The per-entry CRC-32 comes from node:zlib rather than a hand-rolled + // per-byte loop. `unzip -t` verifies every entry's checksum against the + // stored one, so a wrong CRC fails here rather than silently producing + // archives that only some readers reject. + test("stores a CRC-32 the system unzip accepts", async () => { + const zip = await createZip([ + { name: "t/empty.txt", data: Buffer.alloc(0) }, + { name: "t/small.txt", data: Buffer.from("hello") }, + // Non-UTF8 bytes, so the checksum is over real binary content. + { name: "t/bin.dat", data: Buffer.from([0x00, 0xff, 0x80, 0x7f]) }, + { name: "t/big.txt", data: Buffer.from("ab".repeat(100_000)) }, + ]); + const dir = mkdtempSync(path.join(tmpdir(), "evalboard-zip-crc-")); + try { + const zipPath = path.join(dir, "a.zip"); + writeFileSync(zipPath, zip); + const out = execFileSync("unzip", ["-t", zipPath], { + encoding: "utf-8", + }); + expect(out).toMatch(/No errors detected/); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + test("compresses large repetitive payloads (DEFLATE, not STORE)", async () => { const big = Buffer.from("a".repeat(100_000)); const zip = await createZip([{ name: "big.txt", data: big }]); diff --git a/evalboard/lib/blob.ts b/evalboard/lib/blob.ts index 041454f5..90394dca 100644 --- a/evalboard/lib/blob.ts +++ b/evalboard/lib/blob.ts @@ -297,29 +297,11 @@ export async function ensureRunReviewIndex( }); } -// Full fetch: every blob under the run prefix. Used by the "download whole -// run" button, which needs all task subdirs at once (the narrow per-task / -// summary fetches only cache what their page reads). `.venv` trees are -// skipped for the same reason as ensureTaskDir — no page reads them and they -// dwarf the real deliverables. -export async function ensureRunDir( - container: string, - runId: string, - destRoot: string, -): Promise { - assertValidId(runId, "runId"); - if (LOCAL_RUNS_DIR) return; - return dedupe(`run:${container}:${runId}`, async () => { - const c = await getContainer(container); - const ops: Promise[] = []; - const prefix = `${runId}/`; - for await (const blob of c.listBlobsFlat({ prefix })) { - if (blob.name.includes("/.venv/")) continue; - ops.push(downloadBlob(container, blob.name, destRoot)); - } - await Promise.all(ops); - }); -} +// NOTE: there is deliberately no whole-run fetch. `ensureRunDir` used to exist +// for the run page's download-as-zip button and pulled EVERY blob under the run +// prefix with no concurrency cap — ~10k blobs / ~400 MB for a nightly, issued as +// one unbounded Promise.all. Both it and the button are gone; per-task download +// (below) is the supported shape. // Narrow fetch: run.json + just one task subdir. Used by the per-task // detail page so opening a deep link to a 50-task run doesn't pull every diff --git a/evalboard/lib/overview.ts b/evalboard/lib/overview.ts index 3a159df0..b9c9affb 100644 --- a/evalboard/lib/overview.ts +++ b/evalboard/lib/overview.ts @@ -324,6 +324,30 @@ async function loadPerRunForId( }; } +// First `YYYY-MM-DD` (optionally `_HH-MM-SS`) anywhere in a run id. Ad-hoc ids +// are not date-SHAPED — `parseRunIdDate` anchors, so it rejects them — but +// almost all of them still CARRY their date: `adhoc-2026-09-02_21-59-13`, +// `2026-05-28_skills-full-codex-gpt54`, `haiku45-skills-suite-2026-05-28`. +const ADHOC_DATE_RE = /(\d{4})-(\d{2})-(\d{2})(?:_(\d{2})-(\d{2})-(\d{2}))?/; + +// Cheap, id-only start date for an ad-hoc run; null when the id carries no date. +// A date-only id resolves to the start of its day, which is right for "which +// runs are newest" and can only lose a same-day tie-break. Exported for testing. +export function adhocRunDate(id: string): Date | null { + const m = ADHOC_DATE_RE.exec(id); + if (!m) return null; + const [, y, mo, d, h, mi, s] = m; + const t = Date.UTC( + Number(y), + Number(mo) - 1, + Number(d), + Number(h ?? 0), + Number(mi ?? 0), + Number(s ?? 0), + ); + return Number.isFinite(t) ? new Date(t) : null; +} + // Cache at per-run granularity, NOT per-window. A whole-window PerRun[] for // the 30d window exceeds unstable_cache's hard 2MB ceiling (≈400 tasks × ~20 // runs), and on overflow Next.js drops the write AND hands back a truncated @@ -333,24 +357,73 @@ async function loadPerRunForId( // windows + the trends page. The cross-run aggregation downstream is cheap // in-memory work, so leaving it uncached costs nothing. // -// One cached loader per source, with `source.id` in the key parts. Run ids are -// only unique WITHIN a container — both suites name runs `YYYY-MM-DD_HH-MM-SS`, -// so a source-blind key would let a Scribe run and a skills run with the same -// id serve each other's projection. +// One cached loader per (source, run), with `source.id` in the key parts. Run +// ids are only unique WITHIN a container — both suites name runs +// `YYYY-MM-DD_HH-MM-SS`, so a source-blind key would let a Scribe run and a +// skills run with the same id serve each other's projection. +// +// Per-RUN rather than one loader per source, because `unstable_cache` fixes its +// revalidate and tags at construction: a shared loader can only hold one TTL +// for every run, and cannot carry a per-run tag for the refresh button to +// invalidate. Both of those matter — see `perRunRevalidate` below. type PerRunLoader = (id: string) => Promise; -const perRunLoaders = new Map(); +// Keyed `:`. Bounded by the run count of every container the +// process has served, so it grows with history rather than with traffic. +const perRunLoaders = new Map Promise>(); + +// A run's run.json is written once, at the end of the run (the upload is the +// last thing the runner does), so a run that finished yesterday is immutable. +// Its SIDECARS are not — meta.json (title/description), reviews and analysis can +// be edited in blob afterwards — which is what `runCacheTag` is for. +const SETTLED_AFTER_MS = 24 * 60 * 60 * 1000; +// A run whose id says it is still recent. Re-read often, because it may be +// today's nightly landing while the page is open. +const FRESH_REVALIDATE_SECONDS = 300; +// Everything older. The read this avoids is a multi-MB run.json off an Azure +// Files mount, and its content cannot change on its own. +const SETTLED_REVALIDATE_SECONDS = 24 * 60 * 60; + +// Cache tag for one run's projection, so POST /api/refresh can evict it. Without +// this, a settled run edited in blob would keep serving a stale projection to +// the front page for a day. +export function runCacheTag(sourceId: string, runId: string): string { + return `evalboard-run:${sourceId}:${runId}`; +} + +// Settled runs are cached for a day, everything else for 5 minutes. An id that +// carries no date at all is treated as fresh: those are hand-uploaded ad-hoc +// runs, re-uploaded far more often than pipeline runs, and there are a handful. +// +// Evaluated once per run per process, when the loader is first built, so a run +// that settles while the process is up keeps the 5-minute TTL until the next +// restart. That is the safe direction (it is today's behavior) and not worth a +// timer to correct. +function perRunRevalidate(id: string): number { + const started = parseRunIdDate(id) ?? adhocRunDate(id); + if (started == null) return FRESH_REVALIDATE_SECONDS; + return Date.now() - started.getTime() > SETTLED_AFTER_MS + ? SETTLED_REVALIDATE_SECONDS + : FRESH_REVALIDATE_SECONDS; +} function cachedLoadPerRunFor(source: Source): PerRunLoader { - const existing = perRunLoaders.get(source.id); - if (existing) return existing; - const loader = unstable_cache( - (id: string) => loadPerRunForId(id, source), - ["evalboard-per-run", source.id], - { revalidate: 300 }, - ); - perRunLoaders.set(source.id, loader); - return loader; + return (id: string) => { + const cacheKey = `${source.id}:${id}`; + let loader = perRunLoaders.get(cacheKey); + if (!loader) { + loader = unstable_cache( + () => loadPerRunForId(id, source), + ["evalboard-per-run", source.id, id], + { + revalidate: perRunRevalidate(id), + tags: [runCacheTag(source.id, id)], + }, + ); + perRunLoaders.set(cacheKey, loader); + } + return loader(); + }; } async function loadWindowDataInner( @@ -1161,13 +1234,28 @@ export function buildAdhocRows( }; } +// Extra candidates loaded beyond `limit`, to cover ids that turn out to have no +// readable overview (aborted uploads, and the `deploys/` prefix, which is not a +// run at all) and so get dropped from the rows. +const ADHOC_LOAD_SLACK = 10; + // The Ad-hoc runs section (front page, below the daily listing). "Ad-hoc" here // means "not a daily-pipeline run" — i.e. the id isn't date-shaped, which is // exactly the set listRunIdsInWindow excludes from the chart and main table. -// Every ad-hoc candidate is loaded before sorting (the date lives in run.json, -// not the id, so we can't window by id and still show the most recent): the -// ad-hoc set is small by construction (manual uploads only) and per-run reads -// are memoized for 5 min, so a warm front page pays no extra IO. +// +// Only the newest `limit + ADHOC_LOAD_SLACK` candidates are loaded, ordered by +// the date in the id. This section used to load EVERY ad-hoc candidate before +// sorting, on the premise that the set was "small by construction (manual +// uploads only)". That premise expired: nothing expires the ad-hoc prefixes in +// the `runs` container (unlike `runs-gha`'s 14-day rule), so the set had grown +// to 165 runs / ~294 MB of run.json read on every cold front-page render, to +// show ten rows. +// +// The authoritative sort key stays run.json's `start_time` — the id key only +// decides what to LOAD, and the two can only disagree for a run whose id date +// contradicts its own start_time. Ids carrying no date at all are always loaded +// (there are a handful, e.g. `sdk-live-r2-final`), so they can never be ordered +// out of the section by a key they don't have. export async function getAdhocRunListing( limit: number | null, source: Source = DEFAULT_SOURCE, @@ -1175,10 +1263,28 @@ export async function getAdhocRunListing( const ids = (await listRunIds(source)).filter( (id) => parseRunIdDate(id) == null, ); + const dated: { id: string; at: number }[] = []; + const undated: string[] = []; + for (const id of ids) { + const at = adhocRunDate(id); + if (at == null) undated.push(id); + else dated.push({ id, at: at.getTime() }); + } + dated.sort((a, b) => b.at - a.at); + // null limit = "load everything", which the page never asks for but the + // signature allows; a finite limit takes the newest slice plus slack. + const budget = limit == null ? dated.length : limit + ADHOC_LOAD_SLACK; + const loadedAll = budget >= dated.length; + const toLoad = [...undated, ...dated.slice(0, budget).map((d) => d.id)]; const perRun = await mapWithConcurrency( - ids, + toLoad, FETCH_CONCURRENCY, cachedLoadPerRunFor(source), ); - return buildAdhocRows(perRun, limit); + const listing = buildAdhocRows(perRun, limit); + // `total` drives the "Show more" affordance, so while the load is truncated + // it has to report the candidate count rather than what we happened to read + // — otherwise the section caps itself at the first page. Once everything is + // loaded it reverts to the exact readable-row count. + return loadedAll ? listing : { ...listing, total: ids.length }; } diff --git a/evalboard/lib/runs.ts b/evalboard/lib/runs.ts index ad31a97d..0b21ce22 100644 --- a/evalboard/lib/runs.ts +++ b/evalboard/lib/runs.ts @@ -1,10 +1,10 @@ import { promises as fs } from "node:fs"; import path from "node:path"; +import { cache } from "react"; import { LOCAL_RUNS_DIR, ensureActivationSummary, ensureRunAnalysis, - ensureRunDir, ensureRunMeta, ensureRunSummary, ensureTaskDir, @@ -693,14 +693,28 @@ async function readJson(p: string): Promise { } } -async function readRunJson( - id: string, - source: Source = DEFAULT_SOURCE, -): Promise { - const dir = runsDirFor(RUNS_DIR, source); - await ensureRunSummary(source.container, id, dir); - return readJson(path.join(dir, id, "run.json")); -} +// Request-scoped memo. A run.json is 1.5–6.5 MB and several readers want the +// same one in a single render: the run page alone calls readRunSummary AND +// readRunTasks on the same id, and findMatureSourceRuns walks back through +// earlier runs that the same page may also be listing. `dedupe` in blob.ts only +// collapses the DOWNLOAD — without this each caller still paid its own read off +// the Azure Files mount plus its own JSON.parse. +// +// react/cache is per-request, so this never serves one request's data to +// another; cross-request caching stays with unstable_cache in lib/overview.ts. +// Keyed on the argument tuple, hence `source` last with a default — callers +// that omit it and callers that pass DEFAULT_SOURCE are different keys, which +// costs an extra read at worst and can never return the wrong container. +const readRunJson = cache( + async ( + id: string, + source: Source = DEFAULT_SOURCE, + ): Promise => { + const dir = runsDirFor(RUNS_DIR, source); + await ensureRunSummary(source.container, id, dir); + return readJson(path.join(dir, id, "run.json")); + }, +); // The activation suite is a nested sub-run: its self-contained run.json (enriched // cases + the per-skill rollup) lives at /activation/run.json, separate from @@ -877,8 +891,8 @@ export async function readRunTasks( // slot (SLOT_COUNT = 5 in the runner's maturity.py), so its most recent real // execution is at most ~5 *canonical* runs back; the headroom absorbs ad-hoc / // smoke runs that listRunIds() interleaves but maturity replay ignores. The scan -// short-circuits as soon as every task is resolved, so this is a safety cap, not -// the typical read count. +// short-circuits once every task is resolved (at batch granularity — see the +// walk below), so this is a safety cap, not the typical read count. const MATURE_SOURCE_LOOKBACK = 20; // For each mature-skipped task in `fromRunId`, find the most recent *earlier* run @@ -909,13 +923,25 @@ export async function findMatureSourceRuns( const unresolved = new Set(matureTaskIds); const limit = Math.min(ids.length, start + 1 + MATURE_SOURCE_LOOKBACK); - for (let i = start + 1; i < limit && unresolved.size > 0; i++) { - const data = await readRun(ids[i]); - for (const t of data?.task_results ?? []) { - const tid = t.task_id; - if (!tid || !unresolved.has(tid) || t.mature_skipped) continue; - out[tid] = ids[i]; - unresolved.delete(tid); + // Read in small concurrent batches, then CONSUME each batch in index order + // so "the most recent run that executed this task" is unchanged — the walk + // is still newest-first, only the IO overlaps. Serially, a run page with + // mature rows paid 20 round-trips to Azure Files back-to-back before it + // could render, and the scan only short-circuits once every mature task + // resolves, which with ~50% of tasks carried forward usually never happens. + // The cost of overshooting is at most BATCH-1 extra reads, and those are + // request-memoized (see readRunJson) for anything else on the page. + const BATCH = 5; + for (let i = start + 1; i < limit && unresolved.size > 0; i += BATCH) { + const batch = ids.slice(i, Math.min(i + BATCH, limit)); + const loaded = await Promise.all(batch.map((id) => readRun(id))); + for (let j = 0; j < batch.length && unresolved.size > 0; j++) { + for (const t of loaded[j]?.task_results ?? []) { + const tid = t.task_id; + if (!tid || !unresolved.has(tid) || t.mature_skipped) continue; + out[tid] = batch[j]; + unresolved.delete(tid); + } } } return out; @@ -2462,23 +2488,6 @@ export async function collectTaskFiles( return refs.map((r) => ({ relPath: r.relPath, abs: path.join(taskDir, r.relPath) })); } -// Collect every file under a whole run (`/`) for the download-as-zip -// button on the run page. Same noise filter / symlink skip as collectTaskFiles, -// applied across all task subdirs plus run-level files (run.json, analysis.md, -// meta.json, …). Returns null for an invalid id or a missing/empty run dir. -export async function collectRunFiles( - runId: string, - source: Source = DEFAULT_SOURCE, -): Promise<{ relPath: string; abs: string }[] | null> { - if (!isValidId(runId)) return null; - const dir = runsDirFor(RUNS_DIR, source); - await ensureRunDir(source.container, runId, dir); - const runDir = path.join(dir, runId); - const refs = await walkArtifacts(runDir); - if (refs.length === 0) return null; - return refs.map((r) => ({ relPath: r.relPath, abs: path.join(runDir, r.relPath) })); -} - export async function resolveSafePath( runId: string, relPath: string, diff --git a/evalboard/lib/zip.ts b/evalboard/lib/zip.ts index 66454546..ae12b475 100644 --- a/evalboard/lib/zip.ts +++ b/evalboard/lib/zip.ts @@ -1,4 +1,4 @@ -import { deflateRaw } from "node:zlib"; +import { crc32, deflateRaw } from "node:zlib"; import { promisify } from "node:util"; // Async DEFLATE so compression runs on libuv's threadpool instead of blocking @@ -20,31 +20,14 @@ export interface ZipEntry { data: Buffer; } -// Standard CRC-32 (IEEE 802.3 polynomial 0xEDB88320), computed via a lazily -// built lookup table. ZIP stores a CRC-32 per entry for integrity. -let CRC_TABLE: Uint32Array | null = null; - -function crcTable(): Uint32Array { - if (CRC_TABLE) return CRC_TABLE; - const table = new Uint32Array(256); - for (let n = 0; n < 256; n++) { - let c = n; - for (let k = 0; k < 8; k++) { - c = c & 1 ? 0xedb88320 ^ (c >>> 1) : c >>> 1; - } - table[n] = c >>> 0; - } - CRC_TABLE = table; - return table; -} - -function crc32(buf: Buffer): number { - const table = crcTable(); - let crc = 0xffffffff; - for (let i = 0; i < buf.length; i++) { - crc = table[(crc ^ buf[i]) & 0xff] ^ (crc >>> 8); - } - return (crc ^ 0xffffffff) >>> 0; +// Standard CRC-32 (IEEE 802.3 polynomial 0xEDB88320). ZIP stores one per entry +// for integrity. `zlib.crc32` is the same checksum computed natively; the +// hand-rolled table-driven loop this replaces walked the UNCOMPRESSED bytes one +// at a time on the event loop, ~47x slower measured (94 ms vs 2 ms per 50 MB), +// blocking every other request the server was serving for the duration. +// `>>> 0` keeps it unsigned, matching what the old loop returned. +function entryCrc(buf: Buffer): number { + return crc32(buf) >>> 0; } // Convert a JS Date to the DOS date/time fields ZIP uses. Seconds have 2 s @@ -86,7 +69,7 @@ export async function createZip( const { method, body } = await compress(entry.data); return { nameBuf: Buffer.from(entry.name, "utf-8"), - crc: crc32(entry.data), + crc: entryCrc(entry.data), method, body, uncompressedSize: entry.data.length, From cd479c1c32aa1e81e7a5e1167a6fbde54c501d26 Mon Sep 17 00:00:00 2001 From: Bai Li Date: Thu, 3 Sep 2026 11:27:04 -0700 Subject: [PATCH 2/6] fix(evalboard): always show every known harness in the filter The harness segments were purely data-driven off listRecentHarnesses, which discovers from the last 12 usable pipeline runs. Delegate runs on a weekly cron, so between firings its last run slides past that window and the segment disappears, which reads as "delegate was removed" rather than "delegate hasn't run lately". The known set now always gets a segment. Selecting one with no runs in scope shows an empty result, which is honest and one click from recoverable; a control that silently loses an option is neither. Discovery still drives the chart series, so no empty lines appear alongside the restored segment. Co-Authored-By: Claude Opus 5 (1M context) --- .../__tests__/harness-selector.test.tsx | 51 +++++++++++++++++++ .../app/_components/harness-selector.tsx | 20 +++++--- 2 files changed, 65 insertions(+), 6 deletions(-) diff --git a/evalboard/app/_components/__tests__/harness-selector.test.tsx b/evalboard/app/_components/__tests__/harness-selector.test.tsx index 95b1057c..6a6d020f 100644 --- a/evalboard/app/_components/__tests__/harness-selector.test.tsx +++ b/evalboard/app/_components/__tests__/harness-selector.test.tsx @@ -62,6 +62,57 @@ describe("HarnessSelector", () => { ).toHaveAttribute("aria-pressed", "true"); }); + test("a known harness absent from the discovery window still shows", () => { + // Delegate runs weekly, so it falls out of the 12-run discovery window + // between firings. The segment must survive that, or the filter reads as + // "delegate was removed" rather than "delegate hasn't run lately". + render( + , + ); + for (const h of KNOWN_HARNESSES) { + expect( + screen.getByRole("button", { name: harnessShortLabel(h) }), + ).toBeInTheDocument(); + } + }); + + test("a discovered newcomer is kept, and sorts after the known set", () => { + render( + , + ); + const names = screen + .getAllByRole("button") + .map((b) => b.getAttribute("aria-label")); + expect(names).toContain(harnessShortLabel("zzz-new-harness")); + // Known harnesses hold display order; the newcomer lands last. + expect(names[names.length - 1]).toBe( + harnessShortLabel("zzz-new-harness"), + ); + }); + + test("no segment is duplicated when discovery and the known set overlap", () => { + render( + , + ); + const labels = screen + .getAllByRole("button") + .map((b) => b.getAttribute("aria-label")) + .filter((n): n is string => n != null); + expect(new Set(labels).size).toBe(labels.length); + }); + test("every segment is named for screen readers, not color-only", () => { render( ` while @@ -41,12 +42,19 @@ export function HarnessSelector({ const qs = p.toString(); router.replace(qs ? `${pathname}?${qs}` : pathname, { scroll: false }); }; - // Always show the active harness, even if it has aged out of the recent - // window (so a deep-linked `?h=` still reads as selected rather than absent). - const opts = - current == null || harnesses.includes(current) - ? harnesses - : [current, ...harnesses]; + // Every known harness always gets a segment, whether or not it turned up in + // the discovery window. A weekly harness (delegate) drops out of that window + // between firings, and a control that quietly loses an option reads as "this + // harness was removed" rather than "it hasn't run lately". Scoping to one + // with no runs in the window shows an empty result, which is honest and one + // click from recoverable; a missing segment is neither. `current` is unioned + // in too, so a deep-linked `?h=` outside the known set still reads as + // selected. orderHarnesses fixes the order, so segments never reshuffle. + const opts = orderHarnesses([ + ...KNOWN_HARNESSES, + ...harnesses, + ...(current == null ? [] : [current]), + ]); // shrink-0 + whitespace-nowrap: a segment must keep its label on one line // and never compress, so on a narrow screen the control scrolls (see the // wrapper) instead of wrapping "Claude Code" onto two lines and stretching From 1c073b7e87b8797caf5dd77745ef5da7a889ad86 Mon Sep 17 00:00:00 2001 From: Bai Li Date: Thu, 3 Sep 2026 11:36:54 -0700 Subject: [PATCH 3/6] perf(evalboard): move repeated per-row utility classes into the stylesheet The run page shipped 7.6 MB of HTML for a 1,296-task run and /trends shipped 5.6 MB. In both, 55% of the markup was `class` attribute text: 56k attributes carrying only 120 distinct values on the run page, 40k carrying 78 on trends. One 130-char chip string accounted for 1.4 MB on its own, repeated 10,918 times. The hot combinations now live in globals.css as component classes, so the browser parses them once and caches them across navigations instead of re-reading them on every row. Chips carry the largest share; the run grid's stat key, the trends numeric cell, and the trends sparkline bar are the other three with five-figure render counts. Chip hover moves from stripping `hover:` utilities out of the composed string at runtime to a `chip-act` marker the interactive branch emits, which the compound selectors key on. The non-interactive span still shows no hover affordance. Run page 7.64 MB -> 6.02 MB (-21%), trends 5.60 MB -> 4.49 MB (-20%). No visual change: every generated rule was checked against the utilities it replaced in the built stylesheet. Co-Authored-By: Claude Opus 5 (1M context) --- evalboard/app/globals.css | 71 +++++++++++++++++++ evalboard/app/runs/[id]/chips.tsx | 33 +++++---- evalboard/app/runs/[id]/task-grid.tsx | 2 +- .../app/trends/__tests__/trends-view.test.tsx | 8 ++- evalboard/app/trends/trends-view.tsx | 12 ++-- 5 files changed, 103 insertions(+), 23 deletions(-) diff --git a/evalboard/app/globals.css b/evalboard/app/globals.css index 94c5c56e..c93dce79 100644 --- a/evalboard/app/globals.css +++ b/evalboard/app/globals.css @@ -40,3 +40,74 @@ details[open] > summary > .msg-chevron { details[open] > summary .group-chevron { transform: rotate(90deg); } + +/* Filter/tag chips (app/runs/[id]/chips.tsx). The composed utility string was + ~130 chars and rendered ~11k times on a nightly run page (and ~6k on + /trends): 1.4 MB of identical class text in one response, from 120 distinct + values across 56k attributes. The utilities live here now so the emitted + attribute is four short tokens. The component still decides WHICH tokens to + emit; the variant/state matrix is carried by compound selectors below. */ +@layer components { + .chip { + @apply rounded border transition-colors; + } + .chip-sm { + @apply text-[10px] leading-none px-1.5 py-0.5; + } + .chip-md { + @apply text-xs px-2 py-0.5; + } + /* Idle. Exactly one of idle/selected is ever emitted, so these never + collide and neither needs to out-specify the other. */ + .chip-skill { + @apply bg-indigo-50 text-indigo-700 border-indigo-200 font-medium; + } + .chip-review { + @apply bg-rose-50 text-rose-700 border-rose-200; + } + .chip-tag { + @apply bg-gray-50 text-gray-500 border-gray-200; + } + /* Hover rides on .chip-act, which only the interactive (button) branch + emits — a non-clickable span must not show an affordance it can't + honor. Replaces the old runtime hover-utility strip. */ + .chip-act.chip-skill:hover { + @apply bg-indigo-100; + } + .chip-act.chip-review:hover { + @apply bg-rose-100; + } + .chip-act.chip-tag:hover { + @apply bg-gray-100; + } + /* Selected. */ + .chip-skill-on { + @apply bg-indigo-600 text-white border-indigo-600 font-medium; + } + .chip-review-on { + @apply bg-rose-600 text-white border-rose-600; + } + .chip-tag-on { + @apply bg-studio-blue/10 text-studio-blue border-studio-blue/30; + } + /* The md gray-tag (filter-rail) selected state is solid studio-blue, not + the softer 10% tint the sm grid chips use. */ + .chip-tag-on-md { + @apply bg-studio-blue text-white border-studio-blue; + } + + /* Per-row stat key in the grid's stacked card layout — 5.2k renders on a + nightly run page at 49 chars each. */ + .stat-k { + @apply text-[10px] uppercase tracking-wide text-gray-400; + } + /* Numeric table cell on /trends — 5.4k renders at 47 chars each. */ + .num-cell { + @apply py-2 px-3 tabular-nums text-right text-gray-700; + } + /* One bar of a /trends sparkline — 9.9k renders. The status color stays a + utility on the element; only the geometry is shared. */ + .spark-bar { + @apply w-[6px] h-full rounded-sm; + } +} diff --git a/evalboard/app/runs/[id]/chips.tsx b/evalboard/app/runs/[id]/chips.tsx index dc131b58..24760178 100644 --- a/evalboard/app/runs/[id]/chips.tsx +++ b/evalboard/app/runs/[id]/chips.tsx @@ -8,6 +8,12 @@ export type ChipVariant = "skill" | "review" | "tag"; export type ChipSize = "sm" | "md"; +// The colour/size utilities behind these tokens live in app/globals.css under +// `@layer components`. A chip renders ~11k times on a nightly run page, so the +// composed utility string (~130 chars each) was the single largest contributor +// to that page's HTML — 1.4 MB of identical class text. Emitting short tokens +// instead moves that text into the stylesheet, where the browser parses it once +// and caches it across navigations. const STYLES: Record< ChipVariant, { @@ -19,22 +25,22 @@ const STYLES: Record< } > = { skill: { - idle: "bg-indigo-50 text-indigo-700 border-indigo-200 hover:bg-indigo-100 font-medium", - active: "bg-indigo-600 text-white border-indigo-600 font-medium", + idle: "chip-skill", + active: "chip-skill-on", countIdle: "text-indigo-400", countActive: "text-indigo-100", title: "skill", }, review: { - idle: "bg-rose-50 text-rose-700 border-rose-200 hover:bg-rose-100", - active: "bg-rose-600 text-white border-rose-600", + idle: "chip-review", + active: "chip-review-on", countIdle: "text-rose-400", countActive: "text-white/80", title: "review tag", }, tag: { - idle: "bg-gray-50 text-gray-500 border-gray-200 hover:bg-gray-100", - active: "bg-studio-blue/10 text-studio-blue border-studio-blue/30", + idle: "chip-tag", + active: "chip-tag-on", countIdle: "text-gray-400", countActive: "text-studio-blue/70", title: "task tag", @@ -43,11 +49,11 @@ const STYLES: Record< // md gray-tag active uses solid studio-blue (the filter-rail chip variant); // sm gray-tag active uses the softer 10% tint baked into STYLES. -const TAG_MD_ACTIVE = "bg-studio-blue text-white border-studio-blue"; +const TAG_MD_ACTIVE = "chip-tag-on-md"; const SIZE_CLS: Record = { - sm: "text-[10px] leading-none px-1.5 py-0.5", - md: "text-xs px-2 py-0.5", + sm: "chip-sm", + md: "chip-md", }; export function ChipButton({ @@ -70,11 +76,12 @@ export function ChipButton({ const s = STYLES[variant]; const activeCls = variant === "tag" && size === "md" ? TAG_MD_ACTIVE : s.active; - // Strip hover utilities when there's no click handler — otherwise the - // non-interactive branch shows a hover affordance it can't honor. - const idleCls = onClick ? s.idle : s.idle.replace(/\s?hover:\S+/g, ""); + // `chip-act` arms the hover rules, and only the interactive branch gets it — + // otherwise the non-interactive shows a hover affordance it can't + // honor. (This replaces stripping `hover:` utilities out of the string.) + const idleCls = onClick ? `${s.idle} chip-act` : s.idle; const stateCls = active ? activeCls : idleCls; - const baseCls = `${SIZE_CLS[size]} rounded border transition-colors ${stateCls}`; + const baseCls = `chip ${SIZE_CLS[size]} ${stateCls}`; const tooltip = title ?? s.title; const inner = ( <> diff --git a/evalboard/app/runs/[id]/task-grid.tsx b/evalboard/app/runs/[id]/task-grid.tsx index de97a5e4..4209feb2 100644 --- a/evalboard/app/runs/[id]/task-grid.tsx +++ b/evalboard/app/runs/[id]/task-grid.tsx @@ -490,7 +490,7 @@ function Stat({ }) { return (
-
+
{label}
{value}
diff --git a/evalboard/app/trends/__tests__/trends-view.test.tsx b/evalboard/app/trends/__tests__/trends-view.test.tsx index 9899b127..188f0120 100644 --- a/evalboard/app/trends/__tests__/trends-view.test.tsx +++ b/evalboard/app/trends/__tests__/trends-view.test.tsx @@ -75,10 +75,12 @@ describe("TrendsView — status strip run-axis alignment", () => { const unknown = screen.getByTitle("r1 · unknown"); // The absent-run slot is the hollow stub; present runs are full bars // (including a present-but-null status, which renders as "unknown"). + // `spark-bar` is the full-height bar geometry (app/globals.css); the + // stub keeps its own short height, so the token is the discriminator. expect(gap.className).toContain("border-gray-300"); - expect(gap.className).not.toContain("h-full"); - expect(success.className).toContain("h-full"); - expect(unknown.className).toContain("h-full"); + expect(gap.className).not.toContain("spark-bar"); + expect(success.className).toContain("spark-bar"); + expect(unknown.className).toContain("spark-bar"); // Strip renders oldest → newest, left to right. const titles = [...gap.parentElement!.children].map((c) => c.getAttribute("title"), diff --git a/evalboard/app/trends/trends-view.tsx b/evalboard/app/trends/trends-view.tsx index 2bbe9d52..611b7025 100644 --- a/evalboard/app/trends/trends-view.tsx +++ b/evalboard/app/trends/trends-view.tsx @@ -195,7 +195,7 @@ function StatusBar({ ); } @@ -203,7 +203,7 @@ function StatusBar({ ); })} @@ -478,7 +478,7 @@ function TaskRow({
)} - + {t.totalRuns} - + {fmtDuration(t.avgDurationSeconds)} - + {fmtUsd(t.avgCostUsd)} - + {fmtCount(t.avgTotalTurns ?? t.avgActualCommands)} From 3aa3f0c0c6ffc9d5ab6208f5a0fab2d2f37fb147 Mon Sep 17 00:00:00 2001 From: Bai Li Date: Thu, 3 Sep 2026 12:00:49 -0700 Subject: [PATCH 4/6] fix(evalboard): stop counting one model as two in the run header The run header claimed "+1 more" on runs that used a single model end to end. tallyModels voted on the raw model_used string, and that string varies by code path for one and the same model: a row that errors before the model resolves keeps the qualified id it was configured with ("eu.anthropic.claude-sonnet-5") while every completed row records the bare one ("claude-sonnet-5"). One errored task was enough to make the header report a mixed-model run. The vote now groups on the normalized id, the same key pricing already resolves on, so normalizeModel is exported rather than duplicated. Only the "how many models" question moves to normalized keys: display stays the most common raw string inside the winning group, so the chip still shows what the run recorded instead of a value the header derived. Co-Authored-By: Claude Opus 5 (1M context) --- evalboard/lib/__tests__/runs.test.ts | 39 ++++++++++++++++++++++++ evalboard/lib/pricing.ts | 5 +++- evalboard/lib/runs.ts | 45 +++++++++++++++++++++++----- 3 files changed, 80 insertions(+), 9 deletions(-) diff --git a/evalboard/lib/__tests__/runs.test.ts b/evalboard/lib/__tests__/runs.test.ts index e5ec194a..b839efb5 100644 --- a/evalboard/lib/__tests__/runs.test.ts +++ b/evalboard/lib/__tests__/runs.test.ts @@ -475,6 +475,45 @@ describe("tallyModels", () => { distinct: 0, }); }); + + // A row that errors before the model resolves keeps the qualified id it was + // configured with, while completed rows record the bare one. Counting raw + // strings made that single row read as a second model, so the header showed + // "+1 more" on a run that used one model from end to end. + test("a qualified id is the same model as its bare form", () => { + const out = tallyModels([ + row("claude-sonnet-5"), + row("claude-sonnet-5"), + row("eu.anthropic.claude-sonnet-5"), + ]); + expect(out).toEqual({ dominant: "claude-sonnet-5", distinct: 1 }); + }); + + test("display keeps the raw string the run recorded, not a derived one", () => { + // Every row is qualified, so there is no bare variant to prefer; the + // chip must not invent one. + const out = tallyModels([ + row("eu.anthropic.claude-sonnet-5"), + row("eu.anthropic.claude-sonnet-5"), + ]); + expect(out).toEqual({ + dominant: "eu.anthropic.claude-sonnet-5", + distinct: 1, + }); + }); + + test("genuinely different models still count separately", () => { + // The normalization must not collapse an A/B run's real spread. + const out = tallyModels([ + row("eu.anthropic.claude-sonnet-5"), + row("us.anthropic.claude-opus-5"), + row("us.anthropic.claude-opus-5"), + ]); + expect(out).toEqual({ + dominant: "us.anthropic.claude-opus-5", + distinct: 2, + }); + }); }); describe("extractRunConfig", () => { diff --git a/evalboard/lib/pricing.ts b/evalboard/lib/pricing.ts index 988828c0..a3ae1797 100644 --- a/evalboard/lib/pricing.ts +++ b/evalboard/lib/pricing.ts @@ -128,7 +128,10 @@ const _ROUTING_PREFIXES = [ "openrouter/", ]; const _REGION_PREFIXES = ["eu.", "us.", "apac.", "global."]; -function normalizeModel(model: string): string { +// Exported so the run header's model tally can group on the same key pricing +// looks up on. Without it a single row that recorded the qualified id counts +// as a second model and the header claims a mixed-model run. +export function normalizeModel(model: string): string { let m = model.trim(); for (const pre of _ROUTING_PREFIXES) { if (m.startsWith(pre)) { diff --git a/evalboard/lib/runs.ts b/evalboard/lib/runs.ts index 0b21ce22..274ddf29 100644 --- a/evalboard/lib/runs.ts +++ b/evalboard/lib/runs.ts @@ -16,7 +16,7 @@ import { import { DEFAULT_VARIANT_ID, isValidVariantId } from "./variants"; import { DEFAULT_SOURCE, runsDirFor, type Source } from "./sources"; import { DELIVERABLE_KINDS, DELIVERABLE_NAMES } from "./artifact-kinds"; -import { messageCostUsd } from "./pricing"; +import { messageCostUsd, normalizeModel } from "./pricing"; // Resolution order: // 1. EVALBOARD_LOCAL_RUNS_DIR — local mode, points at a coder_eval runs dir @@ -857,24 +857,53 @@ export async function readRunSummary( // appeared. Mirrors mostCommonAgentType's "the thing these tasks ran on" vote, // but also reports the spread so the run header can say when there was more than // one instead of silently picking a winner. +// +// The vote groups on the NORMALIZED id (same key pricing resolves on), because +// the recorded string varies by code path for one and the same model: a row that +// errored before the model resolved keeps the qualified id it was configured +// with ("eu.anthropic.claude-sonnet-5") while every completed row records the +// bare one ("claude-sonnet-5"). Counting raw strings made a single errored row +// read as a second model, and the header then claimed a mixed-model run over +// what was actually one model throughout. +// +// Display stays the most common RAW string inside the winning group, so the +// chip still shows exactly what the run recorded rather than a value the header +// derived. Only the "how many models" question is answered on normalized keys. export function tallyModels(rows: RawTaskResult[]): { dominant: string | null; distinct: number; } { - const counts = new Map(); + const groups = new Map>(); for (const r of rows) { const m = r.model_used; - if (typeof m === "string" && m) counts.set(m, (counts.get(m) ?? 0) + 1); + if (typeof m !== "string" || !m) continue; + const key = normalizeModel(m); + let raw = groups.get(key); + if (!raw) { + raw = new Map(); + groups.set(key, raw); + } + raw.set(m, (raw.get(m) ?? 0) + 1); } let dominant: string | null = null; let bestN = 0; - for (const [model, n] of counts) { - if (n > bestN) { - dominant = model; - bestN = n; + for (const raws of groups.values()) { + let groupN = 0; + let groupTop: string | null = null; + let groupTopN = 0; + for (const [model, n] of raws) { + groupN += n; + if (n > groupTopN) { + groupTop = model; + groupTopN = n; + } + } + if (groupN > bestN) { + dominant = groupTop; + bestN = groupN; } } - return { dominant, distinct: counts.size }; + return { dominant, distinct: groups.size }; } export async function readRunTasks( From d12f29e8ba76169c71100d502aa28e198fb4a16f Mon Sep 17 00:00:00 2001 From: Bai Li Date: Thu, 3 Sep 2026 13:21:46 -0700 Subject: [PATCH 5/6] docs(evalboard): trim the comments added by this branch Comment-only. Cuts the narration of what the code used to do and the restatements of what it now does, keeping the reasons that aren't recoverable from reading it: why the per-run cache is keyed per run rather than per source, why the ad-hoc load key and the sort key differ, why hover moved to a marker class, and why the model vote normalizes. Net 57 lines. No behavior change. Co-Authored-By: Claude Opus 5 (1M context) --- .../app/_components/harness-selector.tsx | 15 ++-- evalboard/app/api/download/route.ts | 8 +- .../app/api/refresh/__tests__/route.test.ts | 6 +- evalboard/app/api/refresh/route.ts | 5 +- evalboard/app/globals.css | 24 +++--- evalboard/app/runs/[id]/chips.tsx | 13 ++- evalboard/app/runs/[id]/run-view.tsx | 12 +-- evalboard/lib/blob.ts | 8 +- evalboard/lib/overview.ts | 81 +++++++------------ evalboard/lib/pricing.ts | 5 +- evalboard/lib/runs.ts | 52 +++++------- evalboard/lib/zip.ts | 10 +-- 12 files changed, 91 insertions(+), 148 deletions(-) diff --git a/evalboard/app/_components/harness-selector.tsx b/evalboard/app/_components/harness-selector.tsx index 49e10294..0646d17f 100644 --- a/evalboard/app/_components/harness-selector.tsx +++ b/evalboard/app/_components/harness-selector.tsx @@ -42,14 +42,13 @@ export function HarnessSelector({ const qs = p.toString(); router.replace(qs ? `${pathname}?${qs}` : pathname, { scroll: false }); }; - // Every known harness always gets a segment, whether or not it turned up in - // the discovery window. A weekly harness (delegate) drops out of that window - // between firings, and a control that quietly loses an option reads as "this - // harness was removed" rather than "it hasn't run lately". Scoping to one - // with no runs in the window shows an empty result, which is honest and one - // click from recoverable; a missing segment is neither. `current` is unioned - // in too, so a deep-linked `?h=` outside the known set still reads as - // selected. orderHarnesses fixes the order, so segments never reshuffle. + // Every known harness gets a segment whether or not it turned up in the + // discovery window. A weekly harness (delegate) drops out of that window + // between firings, and a control that quietly loses an option reads as + // "removed" rather than "hasn't run lately"; an empty result is honest and + // one click from recoverable. `current` is unioned in so a deep-linked `?h=` + // outside the known set still reads as selected, and orderHarnesses fixes + // the order so segments never reshuffle. const opts = orderHarnesses([ ...KNOWN_HARNESSES, ...harnesses, diff --git a/evalboard/app/api/download/route.ts b/evalboard/app/api/download/route.ts index 65175053..79212bf6 100644 --- a/evalboard/app/api/download/route.ts +++ b/evalboard/app/api/download/route.ts @@ -14,11 +14,9 @@ export const dynamic = "force-dynamic"; // blobs first, so this mirrors what the page would load. // // `task` is REQUIRED. Omitting it used to mean "zip the whole run", which for a -// nightly meant ~10k blobs / ~400 MB: an uncapped fan-out of blob downloads, a -// stat-per-file walk over Azure Files, and the entire archive buffered in -// memory before any of it was sent. That path is gone, along with the run -// page's button for it — a task folder is a handful of files and returns in -// well under a second. To inspect a whole run, use the blob container directly. +// nightly meant ~10k blobs / ~400 MB fetched uncapped and buffered in memory +// before any of it was sent. That path and its button are gone; to inspect a +// whole run, use the blob container. export async function GET(req: Request) { const url = new URL(req.url); const runId = url.searchParams.get("run"); diff --git a/evalboard/app/api/refresh/__tests__/route.test.ts b/evalboard/app/api/refresh/__tests__/route.test.ts index 70902a49..630b2099 100644 --- a/evalboard/app/api/refresh/__tests__/route.test.ts +++ b/evalboard/app/api/refresh/__tests__/route.test.ts @@ -128,10 +128,8 @@ describe("POST /api/refresh", () => { await expect(fs.access(runDir)).rejects.toThrow(); }); - // Evicting only the on-disk copy is not enough: the front page reads a - // memoized projection of run.json that a settled run holds for a day - // (lib/overview.ts::perRunRevalidate), so without this the refresh - // button is a no-op for anything older than 24h. + // Without this the button is a no-op for a settled run, whose + // projection is held for a day (lib/overview.ts::perRunRevalidate). test("valid run -> the memoized projection is invalidated too", async () => { const POST = await loadPost(); await POST(post("2026-06-01_04-04-22")); diff --git a/evalboard/app/api/refresh/route.ts b/evalboard/app/api/refresh/route.ts index 00425908..cebeedc5 100644 --- a/evalboard/app/api/refresh/route.ts +++ b/evalboard/app/api/refresh/route.ts @@ -40,9 +40,8 @@ export async function POST(req: Request) { return NextResponse.json({ error: "invalid run" }, { status: 400 }); } // Evicting the on-disk copy is only half the job: the front page reads a - // memoized PROJECTION of it (lib/overview.ts::cachedLoadPerRunFor), and a - // settled run's projection is held for a day. Drop that entry too, or the - // refresh silently does nothing for every run older than 24h. + // memoized projection of it (lib/overview.ts::cachedLoadPerRunFor), held for + // a day on a settled run. Without this the refresh would no-op for those. revalidateTag(runCacheTag(source.id, runId)); // Re-download is lazy on next render. The run page is force-dynamic and // reads the sidecars straight from disk, so it is fresh immediately. diff --git a/evalboard/app/globals.css b/evalboard/app/globals.css index c93dce79..72a04073 100644 --- a/evalboard/app/globals.css +++ b/evalboard/app/globals.css @@ -42,11 +42,9 @@ details[open] > summary .group-chevron { } /* Filter/tag chips (app/runs/[id]/chips.tsx). The composed utility string was - ~130 chars and rendered ~11k times on a nightly run page (and ~6k on - /trends): 1.4 MB of identical class text in one response, from 120 distinct - values across 56k attributes. The utilities live here now so the emitted - attribute is four short tokens. The component still decides WHICH tokens to - emit; the variant/state matrix is carried by compound selectors below. */ + ~130 chars and rendered ~11k times on a nightly run page: 1.4 MB of identical + class text in one response. Emitting four short tokens instead moves it here, + where the browser parses it once and caches it across navigations. */ @layer components { .chip { @apply rounded border transition-colors; @@ -57,8 +55,8 @@ details[open] > summary .group-chevron { .chip-md { @apply text-xs px-2 py-0.5; } - /* Idle. Exactly one of idle/selected is ever emitted, so these never - collide and neither needs to out-specify the other. */ + /* Idle. Exactly one of idle/selected is ever emitted, so neither needs to + out-specify the other. */ .chip-skill { @apply bg-indigo-50 text-indigo-700 border-indigo-200 font-medium; } @@ -69,8 +67,7 @@ details[open] > summary .group-chevron { @apply bg-gray-50 text-gray-500 border-gray-200; } /* Hover rides on .chip-act, which only the interactive (button) branch - emits — a non-clickable span must not show an affordance it can't - honor. Replaces the old runtime hover-utility strip. */ + emits, so a non-clickable span shows no affordance it can't honor. */ .chip-act.chip-skill:hover { @apply bg-indigo-100; } @@ -96,17 +93,16 @@ details[open] > summary .group-chevron { @apply bg-studio-blue text-white border-studio-blue; } - /* Per-row stat key in the grid's stacked card layout — 5.2k renders on a - nightly run page at 49 chars each. */ + /* Per-row stat key in the grid's stacked card layout; 5.2k renders. */ .stat-k { @apply text-[10px] uppercase tracking-wide text-gray-400; } - /* Numeric table cell on /trends — 5.4k renders at 47 chars each. */ + /* Numeric table cell on /trends; 5.4k renders. */ .num-cell { @apply py-2 px-3 tabular-nums text-right text-gray-700; } - /* One bar of a /trends sparkline — 9.9k renders. The status color stays a - utility on the element; only the geometry is shared. */ + /* One bar of a /trends sparkline; 9.9k renders. The status color stays a + utility on the element, so only the geometry is shared. */ .spark-bar { @apply w-[6px] h-full rounded-sm; } diff --git a/evalboard/app/runs/[id]/chips.tsx b/evalboard/app/runs/[id]/chips.tsx index 24760178..aa11f683 100644 --- a/evalboard/app/runs/[id]/chips.tsx +++ b/evalboard/app/runs/[id]/chips.tsx @@ -9,11 +9,9 @@ export type ChipVariant = "skill" | "review" | "tag"; export type ChipSize = "sm" | "md"; // The colour/size utilities behind these tokens live in app/globals.css under -// `@layer components`. A chip renders ~11k times on a nightly run page, so the -// composed utility string (~130 chars each) was the single largest contributor -// to that page's HTML — 1.4 MB of identical class text. Emitting short tokens -// instead moves that text into the stylesheet, where the browser parses it once -// and caches it across navigations. +// `@layer components`. A chip renders ~11k times on a nightly run page, and the +// composed utility string (~130 chars each) was 1.4 MB of identical class text +// in one response; short tokens move it into the cached stylesheet. const STYLES: Record< ChipVariant, { @@ -76,9 +74,8 @@ export function ChipButton({ const s = STYLES[variant]; const activeCls = variant === "tag" && size === "md" ? TAG_MD_ACTIVE : s.active; - // `chip-act` arms the hover rules, and only the interactive branch gets it — - // otherwise the non-interactive shows a hover affordance it can't - // honor. (This replaces stripping `hover:` utilities out of the string.) + // `chip-act` arms the hover rules, and only the interactive branch gets it: + // a non-clickable must not show an affordance it can't honor. const idleCls = onClick ? `${s.idle} chip-act` : s.idle; const stateCls = active ? activeCls : idleCls; const baseCls = `chip ${SIZE_CLS[size]} ${stateCls}`; diff --git a/evalboard/app/runs/[id]/run-view.tsx b/evalboard/app/runs/[id]/run-view.tsx index 7b0d4249..ea1626e9 100644 --- a/evalboard/app/runs/[id]/run-view.tsx +++ b/evalboard/app/runs/[id]/run-view.tsx @@ -296,14 +296,10 @@ export function RunView({ // Commit a filter change to the URL WITHOUT a server round-trip. // - // Every reader of `tags` / `rtags` / `q` on this page is client-side — the - // grid is filtered by `filtered` below, out of rows the server already - // sent. The page's server component reads only `src`. So `router.replace` - // bought nothing and cost a full re-render of a force-dynamic route: two - // separate parses of the same multi-MB run.json (readRunSummary and - // readRunTasks each read it), the activation sub-run, the review index, and - // the mature-source scan's serial walk back through earlier runs — all to - // return markup this component recomputes locally anyway. + // Every reader of `tags` / `rtags` / `q` on this page is client-side (see + // `filtered` below); the server component reads only `src`. So + // `router.replace` bought nothing and cost a full re-render of a + // force-dynamic route to return markup this component recomputes anyway. // // The native History API keeps the URL shareable and the back button // working; Next syncs `useSearchParams` off pushState/replaceState, so this diff --git a/evalboard/lib/blob.ts b/evalboard/lib/blob.ts index 90394dca..9e9624fe 100644 --- a/evalboard/lib/blob.ts +++ b/evalboard/lib/blob.ts @@ -297,11 +297,9 @@ export async function ensureRunReviewIndex( }); } -// NOTE: there is deliberately no whole-run fetch. `ensureRunDir` used to exist -// for the run page's download-as-zip button and pulled EVERY blob under the run -// prefix with no concurrency cap — ~10k blobs / ~400 MB for a nightly, issued as -// one unbounded Promise.all. Both it and the button are gone; per-task download -// (below) is the supported shape. +// There is deliberately no whole-run fetch. `ensureRunDir` used to pull EVERY +// blob under the run prefix with no concurrency cap (~10k blobs / ~400 MB for a +// nightly). Per-task download, below, is the supported shape. // Narrow fetch: run.json + just one task subdir. Used by the per-task // detail page so opening a deep link to a 50-task run doesn't pull every diff --git a/evalboard/lib/overview.ts b/evalboard/lib/overview.ts index b9c9affb..b3343b62 100644 --- a/evalboard/lib/overview.ts +++ b/evalboard/lib/overview.ts @@ -325,14 +325,13 @@ async function loadPerRunForId( } // First `YYYY-MM-DD` (optionally `_HH-MM-SS`) anywhere in a run id. Ad-hoc ids -// are not date-SHAPED — `parseRunIdDate` anchors, so it rejects them — but -// almost all of them still CARRY their date: `adhoc-2026-09-02_21-59-13`, -// `2026-05-28_skills-full-codex-gpt54`, `haiku45-skills-suite-2026-05-28`. +// aren't date-SHAPED (parseRunIdDate anchors, so it rejects them) but almost all +// still carry their date: `adhoc-2026-09-02_21-59-13`, `skills-2026-05-28`. const ADHOC_DATE_RE = /(\d{4})-(\d{2})-(\d{2})(?:_(\d{2})-(\d{2})-(\d{2}))?/; -// Cheap, id-only start date for an ad-hoc run; null when the id carries no date. -// A date-only id resolves to the start of its day, which is right for "which -// runs are newest" and can only lose a same-day tie-break. Exported for testing. +// Id-only start date for an ad-hoc run; null when the id carries no date. A +// date-only id resolves to the start of its day, which can only lose a same-day +// tie-break. export function adhocRunDate(id: string): Date | null { const m = ADHOC_DATE_RE.exec(id); if (!m) return null; @@ -362,43 +361,33 @@ export function adhocRunDate(id: string): Date | null { // `YYYY-MM-DD_HH-MM-SS`, so a source-blind key would let a Scribe run and a // skills run with the same id serve each other's projection. // -// Per-RUN rather than one loader per source, because `unstable_cache` fixes its -// revalidate and tags at construction: a shared loader can only hold one TTL -// for every run, and cannot carry a per-run tag for the refresh button to -// invalidate. Both of those matter — see `perRunRevalidate` below. +// Per-run rather than per-source because `unstable_cache` fixes revalidate and +// tags at construction: one shared loader can hold neither a per-run TTL nor a +// per-run tag for the refresh button to evict. type PerRunLoader = (id: string) => Promise; -// Keyed `:`. Bounded by the run count of every container the -// process has served, so it grows with history rather than with traffic. +// Keyed `:`; grows with history, not with traffic. const perRunLoaders = new Map Promise>(); -// A run's run.json is written once, at the end of the run (the upload is the -// last thing the runner does), so a run that finished yesterday is immutable. -// Its SIDECARS are not — meta.json (title/description), reviews and analysis can -// be edited in blob afterwards — which is what `runCacheTag` is for. +// A run.json is written once, at the end of the run, so a finished run is +// immutable. Its sidecars (meta.json, reviews, analysis) are not, which is what +// `runCacheTag` is for. const SETTLED_AFTER_MS = 24 * 60 * 60 * 1000; -// A run whose id says it is still recent. Re-read often, because it may be -// today's nightly landing while the page is open. +// Recent enough that today's nightly may still be landing. const FRESH_REVALIDATE_SECONDS = 300; -// Everything older. The read this avoids is a multi-MB run.json off an Azure -// Files mount, and its content cannot change on its own. +// Older than that: the read this avoids is a multi-MB run.json off Azure Files. const SETTLED_REVALIDATE_SECONDS = 24 * 60 * 60; -// Cache tag for one run's projection, so POST /api/refresh can evict it. Without -// this, a settled run edited in blob would keep serving a stale projection to -// the front page for a day. +// Cache tag for one run's projection, so POST /api/refresh can evict it; +// without it the button would no-op for anything past SETTLED_AFTER_MS. export function runCacheTag(sourceId: string, runId: string): string { return `evalboard-run:${sourceId}:${runId}`; } -// Settled runs are cached for a day, everything else for 5 minutes. An id that -// carries no date at all is treated as fresh: those are hand-uploaded ad-hoc -// runs, re-uploaded far more often than pipeline runs, and there are a handful. -// -// Evaluated once per run per process, when the loader is first built, so a run -// that settles while the process is up keeps the 5-minute TTL until the next -// restart. That is the safe direction (it is today's behavior) and not worth a -// timer to correct. +// Settled runs cache for a day, everything else for 5 minutes. An id carrying no +// date at all is treated as fresh: those are hand-uploaded and re-uploaded far +// more often than pipeline runs. Evaluated once per run per process, so a run +// that settles while the process is up keeps the short TTL until restart. function perRunRevalidate(id: string): number { const started = parseRunIdDate(id) ?? adhocRunDate(id); if (started == null) return FRESH_REVALIDATE_SECONDS; @@ -1234,9 +1223,8 @@ export function buildAdhocRows( }; } -// Extra candidates loaded beyond `limit`, to cover ids that turn out to have no -// readable overview (aborted uploads, and the `deploys/` prefix, which is not a -// run at all) and so get dropped from the rows. +// Extra candidates loaded beyond `limit`, covering ids with no readable overview +// (aborted uploads, the `deploys/` prefix) that then drop out of the rows. const ADHOC_LOAD_SLACK = 10; // The Ad-hoc runs section (front page, below the daily listing). "Ad-hoc" here @@ -1244,18 +1232,10 @@ const ADHOC_LOAD_SLACK = 10; // exactly the set listRunIdsInWindow excludes from the chart and main table. // // Only the newest `limit + ADHOC_LOAD_SLACK` candidates are loaded, ordered by -// the date in the id. This section used to load EVERY ad-hoc candidate before -// sorting, on the premise that the set was "small by construction (manual -// uploads only)". That premise expired: nothing expires the ad-hoc prefixes in -// the `runs` container (unlike `runs-gha`'s 14-day rule), so the set had grown -// to 165 runs / ~294 MB of run.json read on every cold front-page render, to -// show ten rows. -// -// The authoritative sort key stays run.json's `start_time` — the id key only -// decides what to LOAD, and the two can only disagree for a run whose id date -// contradicts its own start_time. Ids carrying no date at all are always loaded -// (there are a handful, e.g. `sdk-live-r2-final`), so they can never be ordered -// out of the section by a key they don't have. +// the date in the id; loading all 165 to show ten rows cost ~294 MB per cold +// render. run.json's `start_time` stays the authoritative sort, so the id only +// decides what to READ. Ids with no date are always loaded, so they can never be +// ordered out by a key they don't have. export async function getAdhocRunListing( limit: number | null, source: Source = DEFAULT_SOURCE, @@ -1271,8 +1251,7 @@ export async function getAdhocRunListing( else dated.push({ id, at: at.getTime() }); } dated.sort((a, b) => b.at - a.at); - // null limit = "load everything", which the page never asks for but the - // signature allows; a finite limit takes the newest slice plus slack. + // null limit = "load everything": allowed by the signature, never used. const budget = limit == null ? dated.length : limit + ADHOC_LOAD_SLACK; const loadedAll = budget >= dated.length; const toLoad = [...undated, ...dated.slice(0, budget).map((d) => d.id)]; @@ -1282,9 +1261,7 @@ export async function getAdhocRunListing( cachedLoadPerRunFor(source), ); const listing = buildAdhocRows(perRun, limit); - // `total` drives the "Show more" affordance, so while the load is truncated - // it has to report the candidate count rather than what we happened to read - // — otherwise the section caps itself at the first page. Once everything is - // loaded it reverts to the exact readable-row count. + // `total` drives "Show more", so a truncated load must report the candidate + // count or the section caps itself at the first page. return loadedAll ? listing : { ...listing, total: ids.length }; } diff --git a/evalboard/lib/pricing.ts b/evalboard/lib/pricing.ts index a3ae1797..85b11193 100644 --- a/evalboard/lib/pricing.ts +++ b/evalboard/lib/pricing.ts @@ -128,9 +128,8 @@ const _ROUTING_PREFIXES = [ "openrouter/", ]; const _REGION_PREFIXES = ["eu.", "us.", "apac.", "global."]; -// Exported so the run header's model tally can group on the same key pricing -// looks up on. Without it a single row that recorded the qualified id counts -// as a second model and the header claims a mixed-model run. +// Exported so the run header's model tally groups on the same key pricing looks +// up on (lib/runs.ts::tallyModels). export function normalizeModel(model: string): string { let m = model.trim(); for (const pre of _ROUTING_PREFIXES) { diff --git a/evalboard/lib/runs.ts b/evalboard/lib/runs.ts index 274ddf29..010a38b2 100644 --- a/evalboard/lib/runs.ts +++ b/evalboard/lib/runs.ts @@ -693,18 +693,12 @@ async function readJson(p: string): Promise { } } -// Request-scoped memo. A run.json is 1.5–6.5 MB and several readers want the -// same one in a single render: the run page alone calls readRunSummary AND -// readRunTasks on the same id, and findMatureSourceRuns walks back through -// earlier runs that the same page may also be listing. `dedupe` in blob.ts only -// collapses the DOWNLOAD — without this each caller still paid its own read off -// the Azure Files mount plus its own JSON.parse. -// -// react/cache is per-request, so this never serves one request's data to -// another; cross-request caching stays with unstable_cache in lib/overview.ts. -// Keyed on the argument tuple, hence `source` last with a default — callers -// that omit it and callers that pass DEFAULT_SOURCE are different keys, which -// costs an extra read at worst and can never return the wrong container. +// Request-scoped memo. A run.json is 1.5-6.5 MB and the run page reads the same +// one twice (readRunSummary and readRunTasks); `dedupe` in blob.ts collapses only +// the download, not the read and parse. react/cache is per-request, so this +// never serves one request's data to another. The key is the argument tuple, so +// an omitted `source` and an explicit DEFAULT_SOURCE are two keys: one extra +// read at worst, never the wrong container. const readRunJson = cache( async ( id: string, @@ -858,17 +852,14 @@ export async function readRunSummary( // but also reports the spread so the run header can say when there was more than // one instead of silently picking a winner. // -// The vote groups on the NORMALIZED id (same key pricing resolves on), because -// the recorded string varies by code path for one and the same model: a row that -// errored before the model resolved keeps the qualified id it was configured -// with ("eu.anthropic.claude-sonnet-5") while every completed row records the -// bare one ("claude-sonnet-5"). Counting raw strings made a single errored row -// read as a second model, and the header then claimed a mixed-model run over -// what was actually one model throughout. +// The vote groups on the NORMALIZED id (same key pricing resolves on) because +// the recorded string varies by code path: a row that errored before the model +// resolved keeps the qualified id it was configured with +// ("eu.anthropic.claude-sonnet-5") while completed rows record the bare one, so +// counting raw strings let one errored row read as a second model. // -// Display stays the most common RAW string inside the winning group, so the -// chip still shows exactly what the run recorded rather than a value the header -// derived. Only the "how many models" question is answered on normalized keys. +// Display stays the most common RAW string inside the winning group, so the chip +// shows what the run recorded rather than a derived value. export function tallyModels(rows: RawTaskResult[]): { dominant: string | null; distinct: number; @@ -920,8 +911,8 @@ export async function readRunTasks( // slot (SLOT_COUNT = 5 in the runner's maturity.py), so its most recent real // execution is at most ~5 *canonical* runs back; the headroom absorbs ad-hoc / // smoke runs that listRunIds() interleaves but maturity replay ignores. The scan -// short-circuits once every task is resolved (at batch granularity — see the -// walk below), so this is a safety cap, not the typical read count. +// short-circuits once every task is resolved (at batch granularity), so this is +// a safety cap, not the typical read count. const MATURE_SOURCE_LOOKBACK = 20; // For each mature-skipped task in `fromRunId`, find the most recent *earlier* run @@ -952,14 +943,11 @@ export async function findMatureSourceRuns( const unresolved = new Set(matureTaskIds); const limit = Math.min(ids.length, start + 1 + MATURE_SOURCE_LOOKBACK); - // Read in small concurrent batches, then CONSUME each batch in index order - // so "the most recent run that executed this task" is unchanged — the walk - // is still newest-first, only the IO overlaps. Serially, a run page with - // mature rows paid 20 round-trips to Azure Files back-to-back before it - // could render, and the scan only short-circuits once every mature task - // resolves, which with ~50% of tasks carried forward usually never happens. - // The cost of overshooting is at most BATCH-1 extra reads, and those are - // request-memoized (see readRunJson) for anything else on the page. + // Read in concurrent batches, then CONSUME each batch in index order so "the + // most recent run that executed this task" is unchanged: the walk is still + // newest-first, only the IO overlaps. Serially this was up to 20 round-trips + // to Azure Files back-to-back before the page could render. Overshooting + // costs at most BATCH-1 extra reads, and those are request-memoized. const BATCH = 5; for (let i = start + 1; i < limit && unresolved.size > 0; i += BATCH) { const batch = ids.slice(i, Math.min(i + BATCH, limit)); diff --git a/evalboard/lib/zip.ts b/evalboard/lib/zip.ts index ae12b475..2f0dbd5f 100644 --- a/evalboard/lib/zip.ts +++ b/evalboard/lib/zip.ts @@ -20,12 +20,10 @@ export interface ZipEntry { data: Buffer; } -// Standard CRC-32 (IEEE 802.3 polynomial 0xEDB88320). ZIP stores one per entry -// for integrity. `zlib.crc32` is the same checksum computed natively; the -// hand-rolled table-driven loop this replaces walked the UNCOMPRESSED bytes one -// at a time on the event loop, ~47x slower measured (94 ms vs 2 ms per 50 MB), -// blocking every other request the server was serving for the duration. -// `>>> 0` keeps it unsigned, matching what the old loop returned. +// Standard CRC-32 (IEEE 802.3 polynomial 0xEDB88320), one per ZIP entry. +// `zlib.crc32` computes it natively; the hand-rolled loop it replaces walked the +// uncompressed bytes one at a time on the event loop, ~47x slower measured. +// `>>> 0` keeps it unsigned, matching what that loop returned. function entryCrc(buf: Buffer): number { return crc32(buf) >>> 0; } From fe4c1e827b4665baa99b8a511722aa6ce733d91c Mon Sep 17 00:00:00 2001 From: Bai Li Date: Thu, 3 Sep 2026 13:42:43 -0700 Subject: [PATCH 6/6] feat(evalboard): add loading skeletons to the slow routes Every heavy route is force-dynamic, so a navigation runs the server component before anything can render. With no Suspense boundary the browser sat on the PREVIOUS page for all of it: 1.4-1.6s on the front page, 1.7-2.2s on a 1,296-task run, with no sign the click registered. Adds a route-level loading.tsx for the front page (which by nesting also covers trends, watchlist, path-to-ga, scribe and runs/latest), the run page, and the task page, over shared primitives in _components/skeleton. Also replaces the run page's `Suspense fallback={null}` with the same body skeleton. That boundary left a blank hole under an already-rendered header while the review index and mature-source scan finished, which reads as a broken page rather than a loading one. Blocks mirror the real breakpoints (two stat-tile columns on a phone and five from md; the task table at md and up, cards below it) so the layout does not move when content lands. Widths are fractional or capped, and tests assert no fixed width can overflow a 320px viewport. The pulse sits on the wrapper, not each block, and drops under prefers-reduced-motion. Co-Authored-By: Claude Opus 5 (1M context) --- .../_components/__tests__/skeleton.test.tsx | 97 ++++++++++ evalboard/app/_components/skeleton.tsx | 171 ++++++++++++++++++ evalboard/app/loading.tsx | 40 ++++ evalboard/app/runs/[id]/[...task]/loading.tsx | 39 ++++ evalboard/app/runs/[id]/loading.tsx | 31 ++++ evalboard/app/runs/[id]/page.tsx | 19 +- 6 files changed, 396 insertions(+), 1 deletion(-) create mode 100644 evalboard/app/_components/__tests__/skeleton.test.tsx create mode 100644 evalboard/app/_components/skeleton.tsx create mode 100644 evalboard/app/loading.tsx create mode 100644 evalboard/app/runs/[id]/[...task]/loading.tsx create mode 100644 evalboard/app/runs/[id]/loading.tsx diff --git a/evalboard/app/_components/__tests__/skeleton.test.tsx b/evalboard/app/_components/__tests__/skeleton.test.tsx new file mode 100644 index 00000000..6794ebfe --- /dev/null +++ b/evalboard/app/_components/__tests__/skeleton.test.tsx @@ -0,0 +1,97 @@ +import { describe, expect, test } from "vitest"; +import { render } from "@testing-library/react"; +import RootLoading from "@/app/loading"; +import RunLoading from "@/app/runs/[id]/loading"; +import TaskLoading from "@/app/runs/[id]/[...task]/loading"; + +const CASES = [ + ["root", RootLoading, "Loading page"], + ["run", RunLoading, "Loading run"], + ["task", TaskLoading, "Loading task"], +] as const; + +// Every class on every rendered node, flattened into tokens. +function classTokens(root: HTMLElement): string[] { + const out: string[] = []; + for (const el of root.querySelectorAll("*")) { + out.push(...el.className.split(/\s+/).filter(Boolean)); + } + return out; +} + +// Tailwind's numeric width scale is quarter-rem, so `w-24` is 96px. Arbitrary +// values carry their own unit. +function fixedWidthPx(token: string): number | null { + const scale = /^(?:min-)?w-(\d+(?:\.\d+)?)$/.exec(token); + if (scale) return Number(scale[1]) * 4; + const arb = /^(?:min-)?w-\[(\d+(?:\.\d+)?)px\]$/.exec(token); + if (arb) return Number(arb[1]); + return null; +} + +// The narrowest phone the rest of the app is built to survive. A skeleton that +// overflows it would introduce a horizontal scrollbar that disappears the +// moment the real content arrives. +const MIN_VIEWPORT_PX = 320; +// Room for the page gutters (`px-4` each side in the root layout). +const CONTENT_BUDGET_PX = MIN_VIEWPORT_PX - 32; + +describe.each(CASES)("%s loading skeleton", (_name, Loading, label) => { + test("announces itself to a screen reader", () => { + const { getByRole } = render(); + const status = getByRole("status"); + expect(status).toHaveAttribute("aria-busy", "true"); + expect(status).toHaveAttribute("aria-label", label); + // The blocks are decorative, so the label has to carry the meaning. + expect(status.textContent).toContain(label); + }); + + test("the pulse is dropped under prefers-reduced-motion", () => { + const { getByRole } = render(); + const cls = getByRole("status").className; + expect(cls).toContain("animate-pulse"); + expect(cls).toContain("motion-reduce:animate-none"); + }); + + test("no fixed width can overflow a 320px phone", () => { + const { container } = render(); + const offenders = classTokens(container) + .map((t) => [t, fixedWidthPx(t)] as const) + .filter(([, px]) => px != null && px > CONTENT_BUDGET_PX); + expect(offenders.map(([t]) => t)).toEqual([]); + }); + + test("every wide block is capped rather than left to run", () => { + const { container } = render(); + // `w-full` on its own is fine (it fills the parent); paired with a + // `max-w-*` it is also fine. What must not appear is a fixed width + // wider than the phone, which the test above covers, or a horizontal + // overflow container, which would scroll instead of wrap. + expect(classTokens(container)).not.toContain("overflow-x-auto"); + }); + + test("the skeleton is small enough to be free", () => { + const { container } = render(); + // The pages these stand in for render thousands of nodes; the point of + // the skeleton is to appear instantly, so it must stay tiny. + expect(container.querySelectorAll("*").length).toBeLessThan(200); + }); +}); + +describe("run skeleton", () => { + test("mirrors the grid's own md breakpoint instead of picking one layout", () => { + // task-grid renders a table at md and up and a card list below it. If + // the skeleton showed only one, a phone (or a desktop) would watch the + // layout jump the moment the real content landed. + const { container } = render(); + expect(container.querySelector(".hidden.md\\:block")).not.toBeNull(); + expect(container.querySelector(".md\\:hidden")).not.toBeNull(); + }); + + test("the stat tiles match the real two-column phone grid", () => { + const { container } = render(); + const grid = container.querySelector(".grid"); + expect(grid?.className).toContain("grid-cols-2"); + expect(grid?.className).toContain("md:grid-cols-5"); + }); +}); diff --git a/evalboard/app/_components/skeleton.tsx b/evalboard/app/_components/skeleton.tsx new file mode 100644 index 00000000..11ae1068 --- /dev/null +++ b/evalboard/app/_components/skeleton.tsx @@ -0,0 +1,171 @@ +// Shared skeleton primitives for the route-level `loading.tsx` files. +// +// These exist because every heavy route here is force-dynamic: on navigation +// Next has to run the server component before it can render anything, and with +// no Suspense boundary the browser sits on the PREVIOUS page for the whole of +// it. Measured warm, that is 1.4-1.6s on the front page and 1.7-2.2s on a +// 1,296-task run, with no indication the click registered. +// +// Blocks are sized to the element they stand in for and mirror the same +// breakpoints, so the page doesn't jump when the content lands. Nothing here is +// pixel-exact; it only has to read as "this shape, still loading". +// +// Widths are fractional or capped so a phone never scrolls sideways, and the +// pulse is on the wrapper rather than each block: one animated element instead +// of dozens, all in phase. `motion-reduce` drops it entirely. + +// One grey block. `className` carries the size, so callers read as a layout. +export function SkelBar({ className = "" }: { className?: string }) { + return
; +} + +// A bordered card matching the stat tiles and panels: same border, radius and +// padding, so the skeleton occupies the height the real tile will. +export function SkelTile({ + className = "", + children, +}: { + className?: string; + children?: React.ReactNode; +}) { + return ( +
+ {children ?? ( +
+ + +
+ )} +
+ ); +} + +// A run of chip-shaped blocks, for the filter rails. Widths vary so it reads as +// text of different lengths rather than a progress bar. `flex-wrap` keeps it on +// screen at any width. +export function SkelChips({ count = 8 }: { count?: number }) { + // Deterministic, so the server and client markup agree during hydration. + const widths = ["w-16", "w-24", "w-20", "w-28", "w-14", "w-24", "w-20", "w-32"]; + return ( +
+ {Array.from({ length: count }, (_, i) => ( + + ))} +
+ ); +} + +// Table stand-in for the md-and-up layout: a header strip plus n body rows, +// inside the same bordered, clipped box the real tables use. +export function SkelTable({ + rows = 8, + className = "", +}: { + rows?: number; + className?: string; +}) { + return ( +
+
+ +
+
+ {Array.from({ length: rows }, (_, i) => ( +
+ + + +
+ ))} +
+
+ ); +} + +// Card stand-in for the below-md layout the grid switches to on a phone. +export function SkelCards({ + count = 5, + className = "", +}: { + count?: number; + className?: string; +}) { + return ( +
+ {Array.from({ length: count }, (_, i) => ( +
+ +
+ + + +
+
+ ))} +
+ ); +} + +// The body of a run page: stat tiles, filter rail, then the task listing. Used +// twice, which is why it lives here rather than in a loading.tsx: once as the +// route fallback (below a skeleton header) and once as the fallback for the +// Suspense boundary the run page wraps its grid in, where the real header has +// already streamed and only this hole is left to fill. +// +// The tile row and the listing mirror the real breakpoints exactly: two tile +// columns on a phone with the pass rate spanning both, five from md; a table at +// md and up, cards below it. Matching them is the point, so the layout doesn't +// move when the content lands. +export function RunBodySkeleton() { + return ( +
+
+ + + + +
+
+ + +
+ + +
+ ); +} + +// Wrapper every loading.tsx returns. Carries the pulse, and announces itself +// once to a screen reader instead of leaving the blocks as unlabelled noise. +export function SkeletonPage({ + label, + children, +}: { + label: string; + children: React.ReactNode; +}) { + return ( +
+ {label} + {children} +
+ ); +} diff --git a/evalboard/app/loading.tsx b/evalboard/app/loading.tsx new file mode 100644 index 00000000..e8c77819 --- /dev/null +++ b/evalboard/app/loading.tsx @@ -0,0 +1,40 @@ +import { + SkelBar, + SkelCards, + SkelTable, + SkelTile, + SkeletonPage, +} from "@/app/_components/skeleton"; + +// Fallback for the front page and, by nesting, for every route without a +// closer one: /trends, /watchlist, /path-to-ga, /scribe and /runs/latest. +// Warm server time on those ranges from 3ms (/watchlist) to 2.3s +// (/path-to-ga), and the slow ones are slow because they read run.json off the +// Azure Files mount, which is worse in production than locally. +// +// The shape is the one those pages share: a heading, a row of stat tiles, a +// wide panel (chart or prose), then a listing. Two tile columns on a phone and +// four from md, matching the real grids. +export default function Loading() { + return ( + +
+
+ + +
+
+ + + + +
+ + + + + +
+
+ ); +} diff --git a/evalboard/app/runs/[id]/[...task]/loading.tsx b/evalboard/app/runs/[id]/[...task]/loading.tsx new file mode 100644 index 00000000..914019a1 --- /dev/null +++ b/evalboard/app/runs/[id]/[...task]/loading.tsx @@ -0,0 +1,39 @@ +import { + SkelBar, + SkelTile, + SkeletonPage, +} from "@/app/_components/skeleton"; + +// Fallback for a single task. 0.7-0.9s warm once the task folder is cached, and +// longer on a cold one: opening a deep link fetches that task's artifacts from +// blob before the page can render. +// +// The shape is a breadcrumb, the task heading, its metric tiles, then the +// transcript panel that fills the rest of the page. +export default function Loading() { + return ( + +
+
+ + +
+
+ + + + +
+ +
+ + + + + +
+
+
+
+ ); +} diff --git a/evalboard/app/runs/[id]/loading.tsx b/evalboard/app/runs/[id]/loading.tsx new file mode 100644 index 00000000..bea92554 --- /dev/null +++ b/evalboard/app/runs/[id]/loading.tsx @@ -0,0 +1,31 @@ +import { + RunBodySkeleton, + SkelBar, + SkeletonPage, +} from "@/app/_components/skeleton"; + +// Fallback for a run and its activation sub-page. This is the slowest route in +// the app: 1.7-2.2s warm on a 1,296-task run, because the server parses a +// multi-MB run.json, the activation sub-run, the review index and the +// mature-source scan before it can emit anything. +// +// Covers a client navigation into the run, where nothing of the page exists +// yet, so it draws the header too. Once the header has streamed, the grid's own +// boundary in page.tsx takes over with the same body skeleton. +export default function Loading() { + return ( + +
+
+
+ + +
+ + +
+ +
+
+ ); +} diff --git a/evalboard/app/runs/[id]/page.tsx b/evalboard/app/runs/[id]/page.tsx index 70356c3a..12b51fad 100644 --- a/evalboard/app/runs/[id]/page.tsx +++ b/evalboard/app/runs/[id]/page.tsx @@ -1,5 +1,9 @@ import { Suspense } from "react"; import { notFound } from "next/navigation"; +import { + RunBodySkeleton, + SkeletonPage, +} from "@/app/_components/skeleton"; import { findMatureSourceRuns, readActivationScore, @@ -132,7 +136,20 @@ export default async function RunPage({ {analysis && } - + {/* The header above streams as soon as run.json is parsed, but the + grid waits on the review index and the mature-source scan. With + `fallback={null}` that left a blank hole below a rendered header + for the rest of the wait, which reads as a broken page rather + than a loading one. The body skeleton is the same one + loading.tsx draws, so a navigation into the run shows one + continuous shape instead of two. */} + + + + } + >