diff --git a/evalboard/app/_components/__tests__/harness-selector.test.tsx b/evalboard/app/_components/__tests__/harness-selector.test.tsx index 95b1057c..6a6d020f 100644 --- a/evalboard/app/_components/__tests__/harness-selector.test.tsx +++ b/evalboard/app/_components/__tests__/harness-selector.test.tsx @@ -62,6 +62,57 @@ describe("HarnessSelector", () => { ).toHaveAttribute("aria-pressed", "true"); }); + test("a known harness absent from the discovery window still shows", () => { + // Delegate runs weekly, so it falls out of the 12-run discovery window + // between firings. The segment must survive that, or the filter reads as + // "delegate was removed" rather than "delegate hasn't run lately". + render( + , + ); + for (const h of KNOWN_HARNESSES) { + expect( + screen.getByRole("button", { name: harnessShortLabel(h) }), + ).toBeInTheDocument(); + } + }); + + test("a discovered newcomer is kept, and sorts after the known set", () => { + render( + , + ); + const names = screen + .getAllByRole("button") + .map((b) => b.getAttribute("aria-label")); + expect(names).toContain(harnessShortLabel("zzz-new-harness")); + // Known harnesses hold display order; the newcomer lands last. + expect(names[names.length - 1]).toBe( + harnessShortLabel("zzz-new-harness"), + ); + }); + + test("no segment is duplicated when discovery and the known set overlap", () => { + render( + , + ); + const labels = screen + .getAllByRole("button") + .map((b) => b.getAttribute("aria-label")) + .filter((n): n is string => n != null); + expect(new Set(labels).size).toBe(labels.length); + }); + test("every segment is named for screen readers, not color-only", () => { render( ("*")) { + out.push(...el.className.split(/\s+/).filter(Boolean)); + } + return out; +} + +// Tailwind's numeric width scale is quarter-rem, so `w-24` is 96px. Arbitrary +// values carry their own unit. +function fixedWidthPx(token: string): number | null { + const scale = /^(?:min-)?w-(\d+(?:\.\d+)?)$/.exec(token); + if (scale) return Number(scale[1]) * 4; + const arb = /^(?:min-)?w-\[(\d+(?:\.\d+)?)px\]$/.exec(token); + if (arb) return Number(arb[1]); + return null; +} + +// The narrowest phone the rest of the app is built to survive. A skeleton that +// overflows it would introduce a horizontal scrollbar that disappears the +// moment the real content arrives. +const MIN_VIEWPORT_PX = 320; +// Room for the page gutters (`px-4` each side in the root layout). +const CONTENT_BUDGET_PX = MIN_VIEWPORT_PX - 32; + +describe.each(CASES)("%s loading skeleton", (_name, Loading, label) => { + test("announces itself to a screen reader", () => { + const { getByRole } = render(); + const status = getByRole("status"); + expect(status).toHaveAttribute("aria-busy", "true"); + expect(status).toHaveAttribute("aria-label", label); + // The blocks are decorative, so the label has to carry the meaning. + expect(status.textContent).toContain(label); + }); + + test("the pulse is dropped under prefers-reduced-motion", () => { + const { getByRole } = render(); + const cls = getByRole("status").className; + expect(cls).toContain("animate-pulse"); + expect(cls).toContain("motion-reduce:animate-none"); + }); + + test("no fixed width can overflow a 320px phone", () => { + const { container } = render(); + const offenders = classTokens(container) + .map((t) => [t, fixedWidthPx(t)] as const) + .filter(([, px]) => px != null && px > CONTENT_BUDGET_PX); + expect(offenders.map(([t]) => t)).toEqual([]); + }); + + test("every wide block is capped rather than left to run", () => { + const { container } = render(); + // `w-full` on its own is fine (it fills the parent); paired with a + // `max-w-*` it is also fine. What must not appear is a fixed width + // wider than the phone, which the test above covers, or a horizontal + // overflow container, which would scroll instead of wrap. + expect(classTokens(container)).not.toContain("overflow-x-auto"); + }); + + test("the skeleton is small enough to be free", () => { + const { container } = render(); + // The pages these stand in for render thousands of nodes; the point of + // the skeleton is to appear instantly, so it must stay tiny. + expect(container.querySelectorAll("*").length).toBeLessThan(200); + }); +}); + +describe("run skeleton", () => { + test("mirrors the grid's own md breakpoint instead of picking one layout", () => { + // task-grid renders a table at md and up and a card list below it. If + // the skeleton showed only one, a phone (or a desktop) would watch the + // layout jump the moment the real content landed. + const { container } = render(); + expect(container.querySelector(".hidden.md\\:block")).not.toBeNull(); + expect(container.querySelector(".md\\:hidden")).not.toBeNull(); + }); + + test("the stat tiles match the real two-column phone grid", () => { + const { container } = render(); + const grid = container.querySelector(".grid"); + expect(grid?.className).toContain("grid-cols-2"); + expect(grid?.className).toContain("md:grid-cols-5"); + }); +}); diff --git a/evalboard/app/_components/harness-selector.tsx b/evalboard/app/_components/harness-selector.tsx index 9e7acc0a..0646d17f 100644 --- a/evalboard/app/_components/harness-selector.tsx +++ b/evalboard/app/_components/harness-selector.tsx @@ -1,6 +1,7 @@ "use client"; import { usePathname, useRouter, useSearchParams } from "next/navigation"; +import { KNOWN_HARNESSES, orderHarnesses } from "@/lib/harness"; import { HarnessBadge, harnessShortLabel } from "./harness-badge"; // Segmented control for a page's harness scope. Sets `?h=` while @@ -41,12 +42,18 @@ export function HarnessSelector({ const qs = p.toString(); router.replace(qs ? `${pathname}?${qs}` : pathname, { scroll: false }); }; - // Always show the active harness, even if it has aged out of the recent - // window (so a deep-linked `?h=` still reads as selected rather than absent). - const opts = - current == null || harnesses.includes(current) - ? harnesses - : [current, ...harnesses]; + // Every known harness gets a segment whether or not it turned up in the + // discovery window. A weekly harness (delegate) drops out of that window + // between firings, and a control that quietly loses an option reads as + // "removed" rather than "hasn't run lately"; an empty result is honest and + // one click from recoverable. `current` is unioned in so a deep-linked `?h=` + // outside the known set still reads as selected, and orderHarnesses fixes + // the order so segments never reshuffle. + const opts = orderHarnesses([ + ...KNOWN_HARNESSES, + ...harnesses, + ...(current == null ? [] : [current]), + ]); // shrink-0 + whitespace-nowrap: a segment must keep its label on one line // and never compress, so on a narrow screen the control scrolls (see the // wrapper) instead of wrapping "Claude Code" onto two lines and stretching diff --git a/evalboard/app/_components/skeleton.tsx b/evalboard/app/_components/skeleton.tsx new file mode 100644 index 00000000..11ae1068 --- /dev/null +++ b/evalboard/app/_components/skeleton.tsx @@ -0,0 +1,171 @@ +// Shared skeleton primitives for the route-level `loading.tsx` files. +// +// These exist because every heavy route here is force-dynamic: on navigation +// Next has to run the server component before it can render anything, and with +// no Suspense boundary the browser sits on the PREVIOUS page for the whole of +// it. Measured warm, that is 1.4-1.6s on the front page and 1.7-2.2s on a +// 1,296-task run, with no indication the click registered. +// +// Blocks are sized to the element they stand in for and mirror the same +// breakpoints, so the page doesn't jump when the content lands. Nothing here is +// pixel-exact; it only has to read as "this shape, still loading". +// +// Widths are fractional or capped so a phone never scrolls sideways, and the +// pulse is on the wrapper rather than each block: one animated element instead +// of dozens, all in phase. `motion-reduce` drops it entirely. + +// One grey block. `className` carries the size, so callers read as a layout. +export function SkelBar({ className = "" }: { className?: string }) { + return
; +} + +// A bordered card matching the stat tiles and panels: same border, radius and +// padding, so the skeleton occupies the height the real tile will. +export function SkelTile({ + className = "", + children, +}: { + className?: string; + children?: React.ReactNode; +}) { + return ( +
+ {children ?? ( +
+ + +
+ )} +
+ ); +} + +// A run of chip-shaped blocks, for the filter rails. Widths vary so it reads as +// text of different lengths rather than a progress bar. `flex-wrap` keeps it on +// screen at any width. +export function SkelChips({ count = 8 }: { count?: number }) { + // Deterministic, so the server and client markup agree during hydration. + const widths = ["w-16", "w-24", "w-20", "w-28", "w-14", "w-24", "w-20", "w-32"]; + return ( +
+ {Array.from({ length: count }, (_, i) => ( + + ))} +
+ ); +} + +// Table stand-in for the md-and-up layout: a header strip plus n body rows, +// inside the same bordered, clipped box the real tables use. +export function SkelTable({ + rows = 8, + className = "", +}: { + rows?: number; + className?: string; +}) { + return ( +
+
+ +
+
+ {Array.from({ length: rows }, (_, i) => ( +
+ + + +
+ ))} +
+
+ ); +} + +// Card stand-in for the below-md layout the grid switches to on a phone. +export function SkelCards({ + count = 5, + className = "", +}: { + count?: number; + className?: string; +}) { + return ( +
+ {Array.from({ length: count }, (_, i) => ( +
+ +
+ + + +
+
+ ))} +
+ ); +} + +// The body of a run page: stat tiles, filter rail, then the task listing. Used +// twice, which is why it lives here rather than in a loading.tsx: once as the +// route fallback (below a skeleton header) and once as the fallback for the +// Suspense boundary the run page wraps its grid in, where the real header has +// already streamed and only this hole is left to fill. +// +// The tile row and the listing mirror the real breakpoints exactly: two tile +// columns on a phone with the pass rate spanning both, five from md; a table at +// md and up, cards below it. Matching them is the point, so the layout doesn't +// move when the content lands. +export function RunBodySkeleton() { + return ( +
+
+ + + + +
+
+ + +
+ + +
+ ); +} + +// Wrapper every loading.tsx returns. Carries the pulse, and announces itself +// once to a screen reader instead of leaving the blocks as unlabelled noise. +export function SkeletonPage({ + label, + children, +}: { + label: string; + children: React.ReactNode; +}) { + return ( +
+ {label} + {children} +
+ ); +} diff --git a/evalboard/app/api/download/route.ts b/evalboard/app/api/download/route.ts index 5aa7a509..79212bf6 100644 --- a/evalboard/app/api/download/route.ts +++ b/evalboard/app/api/download/route.ts @@ -1,18 +1,22 @@ import { promises as fs } from "node:fs"; import { NextResponse } from "next/server"; -import { collectRunFiles, collectTaskFiles } from "@/lib/runs"; +import { collectTaskFiles } from "@/lib/runs"; import { DEFAULT_VARIANT_ID, isValidVariantId } from "@/lib/variants"; import { sourceById } from "@/lib/sources"; import { createZip, type ZipEntry } from "@/lib/zip"; export const dynamic = "force-dynamic"; -// Bundle a task folder, or a whole run, into a zip download. +// Bundle ONE task folder into a zip download. // ?run=&task=[&v=] → that task's folder (//) -// ?run= → the entire run folder (run.json + every task dir) // minus the usual scaffolding noise, from the container named by ?src (the -// skills nightly when absent). In blob mode the collect* helpers fetch the -// needed blobs first, so this mirrors what the page would load. +// skills nightly when absent). In blob mode collectTaskFiles fetches the needed +// blobs first, so this mirrors what the page would load. +// +// `task` is REQUIRED. Omitting it used to mean "zip the whole run", which for a +// nightly meant ~10k blobs / ~400 MB fetched uncapped and buffered in memory +// before any of it was sent. That path and its button are gone; to inspect a +// whole run, use the blob container. export async function GET(req: Request) { const url = new URL(req.url); const runId = url.searchParams.get("run"); @@ -25,22 +29,19 @@ export async function GET(req: Request) { if (!runId) { return new NextResponse("missing run", { status: 400 }); } + if (!taskId) { + return new NextResponse("missing task", { status: 400 }); + } - const files = taskId - ? await collectTaskFiles(runId, taskId, source, variantId) - : await collectRunFiles(runId, source); + const files = await collectTaskFiles(runId, taskId, source, variantId); if (!files) { return new NextResponse("not found", { status: 404 }); } - // Top-level folder inside the archive: the task id for a task download, - // the run id for a whole-run download. A non-default arm is named too, so + // Top-level folder inside the archive. A non-default arm is named too, so // two arms of the same task don't produce two identically-named zips. - const root = taskId - ? variantId === DEFAULT_VARIANT_ID - ? taskId - : `${taskId}__${variantId}` - : runId; + const root = + variantId === DEFAULT_VARIANT_ID ? taskId : `${taskId}__${variantId}`; const entries: ZipEntry[] = []; for (const f of files) { const data = await fs.readFile(f.abs).catch(() => null); diff --git a/evalboard/app/api/refresh/__tests__/route.test.ts b/evalboard/app/api/refresh/__tests__/route.test.ts index 601315e3..630b2099 100644 --- a/evalboard/app/api/refresh/__tests__/route.test.ts +++ b/evalboard/app/api/refresh/__tests__/route.test.ts @@ -9,13 +9,24 @@ import { test, vi, } from "vitest"; +import { runCacheTag } from "@/lib/overview"; import { SCRIBE_SOURCE, runsDirFor } from "@/lib/sources"; +// The route calls revalidateTag to evict the run's memoized front-page +// projection alongside its on-disk copy. The real implementation asserts it is +// inside a Next request/render store, which a direct POST() call is not, so it +// is stubbed here and asserted on instead. +const revalidateTag = vi.fn(); +vi.mock("next/cache", () => ({ + revalidateTag: (tag: string) => revalidateTag(tag), +})); + // RUNS_DIR and LOCAL_RUNS_DIR are module-level consts read from env at import // time, so each scenario sets env, resets the module registry, then imports a // fresh copy of the route. async function loadPost() { vi.resetModules(); + revalidateTag.mockClear(); return (await import("../route")).POST; } @@ -117,6 +128,24 @@ describe("POST /api/refresh", () => { await expect(fs.access(runDir)).rejects.toThrow(); }); + // Without this the button is a no-op for a settled run, whose + // projection is held for a day (lib/overview.ts::perRunRevalidate). + test("valid run -> the memoized projection is invalidated too", async () => { + const POST = await loadPost(); + await POST(post("2026-06-01_04-04-22")); + + expect(revalidateTag).toHaveBeenCalledWith( + runCacheTag("skills", "2026-06-01_04-04-22"), + ); + }); + + test("a rejected id revalidates nothing", async () => { + const POST = await loadPost(); + await POST(post("..")); + + expect(revalidateTag).not.toHaveBeenCalled(); + }); + // Run ids collide across sources (both suites name runs // YYYY-MM-DD_HH-MM-SS), so the source decides WHICH cached copy is // evicted. Getting this wrong deletes a run nobody asked about and diff --git a/evalboard/app/api/refresh/route.ts b/evalboard/app/api/refresh/route.ts index 4dd1b22e..cebeedc5 100644 --- a/evalboard/app/api/refresh/route.ts +++ b/evalboard/app/api/refresh/route.ts @@ -1,5 +1,7 @@ +import { revalidateTag } from "next/cache"; import { NextResponse } from "next/server"; import { LOCAL_RUNS_DIR } from "@/lib/blob"; +import { runCacheTag } from "@/lib/overview"; import { RUNS_DIR, clearRunCacheDir } from "@/lib/runs"; import { runsDirFor, sourceById } from "@/lib/sources"; @@ -37,8 +39,11 @@ export async function POST(req: Request) { if (!(await clearRunCacheDir(runsDirFor(RUNS_DIR, source), runId))) { return NextResponse.json({ error: "invalid run" }, { status: 400 }); } + // Evicting the on-disk copy is only half the job: the front page reads a + // memoized projection of it (lib/overview.ts::cachedLoadPerRunFor), held for + // a day on a settled run. Without this the refresh would no-op for those. + revalidateTag(runCacheTag(source.id, runId)); // Re-download is lazy on next render. The run page is force-dynamic and - // reads the sidecars straight from disk, so it is fresh immediately; the - // homepage's per-run title list self-heals within its 5-minute revalidate. + // reads the sidecars straight from disk, so it is fresh immediately. return new Response(null, { status: 204 }); } diff --git a/evalboard/app/globals.css b/evalboard/app/globals.css index 94c5c56e..72a04073 100644 --- a/evalboard/app/globals.css +++ b/evalboard/app/globals.css @@ -40,3 +40,70 @@ details[open] > summary > .msg-chevron { details[open] > summary .group-chevron { transform: rotate(90deg); } + +/* Filter/tag chips (app/runs/[id]/chips.tsx). The composed utility string was + ~130 chars and rendered ~11k times on a nightly run page: 1.4 MB of identical + class text in one response. Emitting four short tokens instead moves it here, + where the browser parses it once and caches it across navigations. */ +@layer components { + .chip { + @apply rounded border transition-colors; + } + .chip-sm { + @apply text-[10px] leading-none px-1.5 py-0.5; + } + .chip-md { + @apply text-xs px-2 py-0.5; + } + /* Idle. Exactly one of idle/selected is ever emitted, so neither needs to + out-specify the other. */ + .chip-skill { + @apply bg-indigo-50 text-indigo-700 border-indigo-200 font-medium; + } + .chip-review { + @apply bg-rose-50 text-rose-700 border-rose-200; + } + .chip-tag { + @apply bg-gray-50 text-gray-500 border-gray-200; + } + /* Hover rides on .chip-act, which only the interactive (button) branch + emits, so a non-clickable span shows no affordance it can't honor. */ + .chip-act.chip-skill:hover { + @apply bg-indigo-100; + } + .chip-act.chip-review:hover { + @apply bg-rose-100; + } + .chip-act.chip-tag:hover { + @apply bg-gray-100; + } + /* Selected. */ + .chip-skill-on { + @apply bg-indigo-600 text-white border-indigo-600 font-medium; + } + .chip-review-on { + @apply bg-rose-600 text-white border-rose-600; + } + .chip-tag-on { + @apply bg-studio-blue/10 text-studio-blue border-studio-blue/30; + } + /* The md gray-tag (filter-rail) selected state is solid studio-blue, not + the softer 10% tint the sm grid chips use. */ + .chip-tag-on-md { + @apply bg-studio-blue text-white border-studio-blue; + } + + /* Per-row stat key in the grid's stacked card layout; 5.2k renders. */ + .stat-k { + @apply text-[10px] uppercase tracking-wide text-gray-400; + } + /* Numeric table cell on /trends; 5.4k renders. */ + .num-cell { + @apply py-2 px-3 tabular-nums text-right text-gray-700; + } + /* One bar of a /trends sparkline; 9.9k renders. The status color stays a + utility on the element, so only the geometry is shared. */ + .spark-bar { + @apply w-[6px] h-full rounded-sm; + } +} diff --git a/evalboard/app/loading.tsx b/evalboard/app/loading.tsx new file mode 100644 index 00000000..e8c77819 --- /dev/null +++ b/evalboard/app/loading.tsx @@ -0,0 +1,40 @@ +import { + SkelBar, + SkelCards, + SkelTable, + SkelTile, + SkeletonPage, +} from "@/app/_components/skeleton"; + +// Fallback for the front page and, by nesting, for every route without a +// closer one: /trends, /watchlist, /path-to-ga, /scribe and /runs/latest. +// Warm server time on those ranges from 3ms (/watchlist) to 2.3s +// (/path-to-ga), and the slow ones are slow because they read run.json off the +// Azure Files mount, which is worse in production than locally. +// +// The shape is the one those pages share: a heading, a row of stat tiles, a +// wide panel (chart or prose), then a listing. Two tile columns on a phone and +// four from md, matching the real grids. +export default function Loading() { + return ( + +
+
+ + +
+
+ + + + +
+ + + + + +
+ + ); +} diff --git a/evalboard/app/runs/[id]/[...task]/loading.tsx b/evalboard/app/runs/[id]/[...task]/loading.tsx new file mode 100644 index 00000000..914019a1 --- /dev/null +++ b/evalboard/app/runs/[id]/[...task]/loading.tsx @@ -0,0 +1,39 @@ +import { + SkelBar, + SkelTile, + SkeletonPage, +} from "@/app/_components/skeleton"; + +// Fallback for a single task. 0.7-0.9s warm once the task folder is cached, and +// longer on a cold one: opening a deep link fetches that task's artifacts from +// blob before the page can render. +// +// The shape is a breadcrumb, the task heading, its metric tiles, then the +// transcript panel that fills the rest of the page. +export default function Loading() { + return ( + +
+
+ + +
+
+ + + + +
+ +
+ + + + + +
+ +
+ + ); +} diff --git a/evalboard/app/runs/[id]/chips.tsx b/evalboard/app/runs/[id]/chips.tsx index dc131b58..aa11f683 100644 --- a/evalboard/app/runs/[id]/chips.tsx +++ b/evalboard/app/runs/[id]/chips.tsx @@ -8,6 +8,10 @@ export type ChipVariant = "skill" | "review" | "tag"; export type ChipSize = "sm" | "md"; +// The colour/size utilities behind these tokens live in app/globals.css under +// `@layer components`. A chip renders ~11k times on a nightly run page, and the +// composed utility string (~130 chars each) was 1.4 MB of identical class text +// in one response; short tokens move it into the cached stylesheet. const STYLES: Record< ChipVariant, { @@ -19,22 +23,22 @@ const STYLES: Record< } > = { skill: { - idle: "bg-indigo-50 text-indigo-700 border-indigo-200 hover:bg-indigo-100 font-medium", - active: "bg-indigo-600 text-white border-indigo-600 font-medium", + idle: "chip-skill", + active: "chip-skill-on", countIdle: "text-indigo-400", countActive: "text-indigo-100", title: "skill", }, review: { - idle: "bg-rose-50 text-rose-700 border-rose-200 hover:bg-rose-100", - active: "bg-rose-600 text-white border-rose-600", + idle: "chip-review", + active: "chip-review-on", countIdle: "text-rose-400", countActive: "text-white/80", title: "review tag", }, tag: { - idle: "bg-gray-50 text-gray-500 border-gray-200 hover:bg-gray-100", - active: "bg-studio-blue/10 text-studio-blue border-studio-blue/30", + idle: "chip-tag", + active: "chip-tag-on", countIdle: "text-gray-400", countActive: "text-studio-blue/70", title: "task tag", @@ -43,11 +47,11 @@ const STYLES: Record< // md gray-tag active uses solid studio-blue (the filter-rail chip variant); // sm gray-tag active uses the softer 10% tint baked into STYLES. -const TAG_MD_ACTIVE = "bg-studio-blue text-white border-studio-blue"; +const TAG_MD_ACTIVE = "chip-tag-on-md"; const SIZE_CLS: Record = { - sm: "text-[10px] leading-none px-1.5 py-0.5", - md: "text-xs px-2 py-0.5", + sm: "chip-sm", + md: "chip-md", }; export function ChipButton({ @@ -70,11 +74,11 @@ export function ChipButton({ const s = STYLES[variant]; const activeCls = variant === "tag" && size === "md" ? TAG_MD_ACTIVE : s.active; - // Strip hover utilities when there's no click handler — otherwise the - // non-interactive branch shows a hover affordance it can't honor. - const idleCls = onClick ? s.idle : s.idle.replace(/\s?hover:\S+/g, ""); + // `chip-act` arms the hover rules, and only the interactive branch gets it: + // a non-clickable must not show an affordance it can't honor. + const idleCls = onClick ? `${s.idle} chip-act` : s.idle; const stateCls = active ? activeCls : idleCls; - const baseCls = `${SIZE_CLS[size]} rounded border transition-colors ${stateCls}`; + const baseCls = `chip ${SIZE_CLS[size]} ${stateCls}`; const tooltip = title ?? s.title; const inner = ( <> diff --git a/evalboard/app/runs/[id]/loading.tsx b/evalboard/app/runs/[id]/loading.tsx new file mode 100644 index 00000000..bea92554 --- /dev/null +++ b/evalboard/app/runs/[id]/loading.tsx @@ -0,0 +1,31 @@ +import { + RunBodySkeleton, + SkelBar, + SkeletonPage, +} from "@/app/_components/skeleton"; + +// Fallback for a run and its activation sub-page. This is the slowest route in +// the app: 1.7-2.2s warm on a 1,296-task run, because the server parses a +// multi-MB run.json, the activation sub-run, the review index and the +// mature-source scan before it can emit anything. +// +// Covers a client navigation into the run, where nothing of the page exists +// yet, so it draws the header too. Once the header has streamed, the grid's own +// boundary in page.tsx takes over with the same body skeleton. +export default function Loading() { + return ( + +
+
+
+ + +
+ + +
+ +
+ + ); +} diff --git a/evalboard/app/runs/[id]/page.tsx b/evalboard/app/runs/[id]/page.tsx index 8088ffbe..12b51fad 100644 --- a/evalboard/app/runs/[id]/page.tsx +++ b/evalboard/app/runs/[id]/page.tsx @@ -1,5 +1,9 @@ import { Suspense } from "react"; import { notFound } from "next/navigation"; +import { + RunBodySkeleton, + SkeletonPage, +} from "@/app/_components/skeleton"; import { findMatureSourceRuns, readActivationScore, @@ -10,7 +14,7 @@ import { } from "@/lib/runs"; import { readRunReviewIndex, indexByTask, tagCountsForRun } from "@/lib/reviews"; import { sourceById } from "@/lib/sources"; -import { scalarParam, withSource } from "@/app/_lib/source-param"; +import { scalarParam } from "@/app/_lib/source-param"; import { fmtRunTime } from "@/lib/format"; import { AnalysisPanel } from "./analysis-panel"; import { RefreshButton } from "./refresh-button"; @@ -87,22 +91,23 @@ export default async function RunPage({ Ad-hoc )} - {/* Refresh re-pulls the run from blob storage and Download - zips it — both are internal-hosting surfaces (the public - OSS edition has no blob backend). See lib/edition.ts. */} + {/* Refresh re-pulls the run from blob storage — an + internal-hosting surface (the public OSS edition has no + blob backend). See lib/edition.ts. + + There is deliberately no whole-run download here. A + nightly run is ~10k blobs / ~400 MB, and zipping it meant + fetching every one of them with no concurrency cap, + walking the tree with a stat per file over Azure Files, + and buffering the entire archive in memory before the + first byte reached the browser — minutes of apparent + hang, against a process that already peaks at 2.7 GB RSS. + Per-task download (on the task page) is the supported + shape; it bundles a handful of files and returns in + well under a second. */} {isInternal && ( )}
@@ -131,7 +136,20 @@ export default async function RunPage({ {analysis && } - + {/* The header above streams as soon as run.json is parsed, but the + grid waits on the review index and the mature-source scan. With + `fallback={null}` that left a blank hole below a rendered header + for the rest of the wait, which reads as a broken page rather + than a loading one. The body skeleton is the same one + loading.tsx draws, so a navigation into the run shows one + continuous shape instead of two. */} + + + + } + > { + const qs = params.toString(); + window.history.replaceState( + null, + "", + qs ? `${pathname}?${qs}` : pathname, + ); + }, + [pathname], + ); + const updateParam = useCallback( (key: string, next: string[]) => { // Read the live URL on commit so a concurrent debounced write @@ -302,12 +324,9 @@ export function RunView({ const params = new URLSearchParams(window.location.search); if (next.length === 0) params.delete(key); else params.set(key, next.join(",")); - const qs = params.toString(); - router.replace(qs ? `${pathname}?${qs}` : pathname, { - scroll: false, - }); + setSearchParams(params); }, - [pathname, router], + [setSearchParams], ); const toggleTag = useCallback( @@ -339,9 +358,8 @@ export function RunView({ params.delete("q"); params.delete("tags"); params.delete("rtags"); - const qs = params.toString(); - router.replace(qs ? `${pathname}?${qs}` : pathname, { scroll: false }); - }, [pathname, router]); + setSearchParams(params); + }, [setSearchParams]); // Skill is the primary group; everything else is a secondary tag. The // secondary "Tags" rail excludes the per-task skill so it doesn't echo diff --git a/evalboard/app/runs/[id]/task-grid.tsx b/evalboard/app/runs/[id]/task-grid.tsx index de97a5e4..4209feb2 100644 --- a/evalboard/app/runs/[id]/task-grid.tsx +++ b/evalboard/app/runs/[id]/task-grid.tsx @@ -490,7 +490,7 @@ function Stat({ }) { return (
-
+
{label}
{value}
diff --git a/evalboard/app/trends/__tests__/trends-view.test.tsx b/evalboard/app/trends/__tests__/trends-view.test.tsx index 9899b127..188f0120 100644 --- a/evalboard/app/trends/__tests__/trends-view.test.tsx +++ b/evalboard/app/trends/__tests__/trends-view.test.tsx @@ -75,10 +75,12 @@ describe("TrendsView — status strip run-axis alignment", () => { const unknown = screen.getByTitle("r1 · unknown"); // The absent-run slot is the hollow stub; present runs are full bars // (including a present-but-null status, which renders as "unknown"). + // `spark-bar` is the full-height bar geometry (app/globals.css); the + // stub keeps its own short height, so the token is the discriminator. expect(gap.className).toContain("border-gray-300"); - expect(gap.className).not.toContain("h-full"); - expect(success.className).toContain("h-full"); - expect(unknown.className).toContain("h-full"); + expect(gap.className).not.toContain("spark-bar"); + expect(success.className).toContain("spark-bar"); + expect(unknown.className).toContain("spark-bar"); // Strip renders oldest → newest, left to right. const titles = [...gap.parentElement!.children].map((c) => c.getAttribute("title"), diff --git a/evalboard/app/trends/trends-view.tsx b/evalboard/app/trends/trends-view.tsx index 2bbe9d52..611b7025 100644 --- a/evalboard/app/trends/trends-view.tsx +++ b/evalboard/app/trends/trends-view.tsx @@ -195,7 +195,7 @@ function StatusBar({ ); } @@ -203,7 +203,7 @@ function StatusBar({ ); })} @@ -478,7 +478,7 @@ function TaskRow({
)} - + {t.totalRuns} - + {fmtDuration(t.avgDurationSeconds)} - + {fmtUsd(t.avgCostUsd)} - + {fmtCount(t.avgTotalTurns ?? t.avgActualCommands)} diff --git a/evalboard/lib/__tests__/collect.test.ts b/evalboard/lib/__tests__/collect.test.ts index 78fcf823..04da4a5d 100644 --- a/evalboard/lib/__tests__/collect.test.ts +++ b/evalboard/lib/__tests__/collect.test.ts @@ -3,7 +3,7 @@ import os from "node:os"; import path from "node:path"; import { afterEach, beforeEach, describe, expect, test, vi } from "vitest"; -// collectTaskFiles / collectRunFiles read RUNS_DIR, which lib/blob.ts resolves +// collectTaskFiles reads RUNS_DIR, which lib/blob.ts resolves // from EVALBOARD_LOCAL_RUNS_DIR at *import* time. So each test stubs the env to // a throwaway runs dir, then dynamically imports a fresh module copy // (vi.resetModules) so RUNS_DIR picks up that path. @@ -67,23 +67,3 @@ describe("collectTaskFiles", () => { expect(await collectTaskFiles(RUN, "nope")).toBeNull(); }); }); - -describe("collectRunFiles", () => { - test("returns run-level files + every task's files, minus noise", async () => { - const { collectRunFiles } = await loadRuns(); - const files = await collectRunFiles(RUN); - const rels = files?.map((f) => f.relPath).sort(); - expect(rels).toEqual([ - "analysis.md", - `default/${TASK}/00/artifacts/main.py`, - `default/${TASK}/00/task.json`, - `default/${TASK}/00/task.log`, - "run.json", - ]); - }); - - test("returns null for a missing run", async () => { - const { collectRunFiles } = await loadRuns(); - expect(await collectRunFiles("9999-99-99_00-00-00")).toBeNull(); - }); -}); diff --git a/evalboard/lib/__tests__/overview.test.ts b/evalboard/lib/__tests__/overview.test.ts index 52507dca..d012d39d 100644 --- a/evalboard/lib/__tests__/overview.test.ts +++ b/evalboard/lib/__tests__/overview.test.ts @@ -1,5 +1,6 @@ import { describe, expect, test, vi } from "vitest"; import { + adhocRunDate, avgRunSuccessRate, buildAdhocRows, buildTagTaskRows, @@ -1206,3 +1207,52 @@ describe("projectRunRow", () => { }); }); }); + +// The front page used to load EVERY ad-hoc candidate before sorting — 165 runs +// / ~294 MB of run.json — to render ten rows. The load is now bounded by an +// id-derived date, so these pin the extraction that decides what gets read. +describe("adhocRunDate", () => { + test("reads the date out of the ad-hoc id shapes actually in the container", () => { + const at = (id: string) => adhocRunDate(id)?.toISOString() ?? null; + // The dominant shape: `adhoc-` prefix + a full date_time. + expect(at("adhoc-2026-09-02_21-59-13")).toBe( + "2026-09-02T21:59:13.000Z", + ); + // Legacy hand-named runs: leading date, no time -> start of that day. + expect(at("2026-05-28_skills-full-codex-gpt54")).toBe( + "2026-05-28T00:00:00.000Z", + ); + // Date anywhere in the id, not just the head. + expect(at("haiku45-skills-suite-2026-05-28")).toBe( + "2026-05-28T00:00:00.000Z", + ); + }); + + test("null for an id carrying no date", () => { + // These are always loaded rather than ordered out, since there is no + // key to order them by. + expect(adhocRunDate("sdk-live-r2-final")).toBeNull(); + expect(adhocRunDate("adhoc-flow-v2-preview-20260820")).toBeNull(); + expect(adhocRunDate("deploys")).toBeNull(); + }); + + test("orders newest-first the same way run start_time does", () => { + const ids = [ + "2026-05-28_skills-full-codex-gpt54", + "adhoc-2026-09-02_21-59-13", + "adhoc-2026-09-02_11-20-47", + "adhoc-2026-08-27_19-35-31", + ]; + const sorted = [...ids].sort( + (a, b) => + (adhocRunDate(b)?.getTime() ?? 0) - + (adhocRunDate(a)?.getTime() ?? 0), + ); + expect(sorted).toEqual([ + "adhoc-2026-09-02_21-59-13", + "adhoc-2026-09-02_11-20-47", + "adhoc-2026-08-27_19-35-31", + "2026-05-28_skills-full-codex-gpt54", + ]); + }); +}); diff --git a/evalboard/lib/__tests__/runs.test.ts b/evalboard/lib/__tests__/runs.test.ts index 991419bb..b839efb5 100644 --- a/evalboard/lib/__tests__/runs.test.ts +++ b/evalboard/lib/__tests__/runs.test.ts @@ -475,6 +475,45 @@ describe("tallyModels", () => { distinct: 0, }); }); + + // A row that errors before the model resolves keeps the qualified id it was + // configured with, while completed rows record the bare one. Counting raw + // strings made that single row read as a second model, so the header showed + // "+1 more" on a run that used one model from end to end. + test("a qualified id is the same model as its bare form", () => { + const out = tallyModels([ + row("claude-sonnet-5"), + row("claude-sonnet-5"), + row("eu.anthropic.claude-sonnet-5"), + ]); + expect(out).toEqual({ dominant: "claude-sonnet-5", distinct: 1 }); + }); + + test("display keeps the raw string the run recorded, not a derived one", () => { + // Every row is qualified, so there is no bare variant to prefer; the + // chip must not invent one. + const out = tallyModels([ + row("eu.anthropic.claude-sonnet-5"), + row("eu.anthropic.claude-sonnet-5"), + ]); + expect(out).toEqual({ + dominant: "eu.anthropic.claude-sonnet-5", + distinct: 1, + }); + }); + + test("genuinely different models still count separately", () => { + // The normalization must not collapse an A/B run's real spread. + const out = tallyModels([ + row("eu.anthropic.claude-sonnet-5"), + row("us.anthropic.claude-opus-5"), + row("us.anthropic.claude-opus-5"), + ]); + expect(out).toEqual({ + dominant: "us.anthropic.claude-opus-5", + distinct: 2, + }); + }); }); describe("extractRunConfig", () => { @@ -553,6 +592,46 @@ describe("findMatureSourceRuns", () => { expect(await findMatureSourceRuns(["a"], "nope", deps)).toEqual({}); }); + // The scan reads in concurrent batches but must still consume them in index + // order — otherwise "the most recent run that executed this task" becomes + // "whichever read resolved first", which is nondeterministic and wrong. + test("within one batch, the newer run still wins", async () => { + const readRun = vi.fn(async (id: string) => { + // Resolve out of index order, so a scan that trusted completion + // order instead of index order would answer r2. + if (id === "r2") return runs.r2; + await Promise.resolve(); + return runs[id] ?? null; + }); + const out = await findMatureSourceRuns(["a"], "r5", { + listIds: async () => ids, + readRun, + }); + expect(out).toEqual({ a: "r3" }); + }); + + test("resolves across a batch boundary", async () => { + // 7 runs, so the first batch of 5 cannot resolve the task and the walk + // has to continue into the second. + const longIds = ["s7", "s6", "s5", "s4", "s3", "s2", "s1"]; + const skipped = { task_results: [{ task_id: "a", mature_skipped: true }] }; + const longRuns: Record = { + s7: skipped, + s6: skipped, + s5: skipped, + s4: skipped, + s3: skipped, + s2: skipped, + s1: { task_results: [{ task_id: "a" }] }, + }; + const out = await findMatureSourceRuns(["a"], "s7", { + listIds: async () => longIds, + readRun: async (id: string) => + (longRuns[id] as (typeof runs)[string]) ?? null, + }); + expect(out).toEqual({ a: "s1" }); + }); + test("a task with no earlier execution is omitted (stays non-clickable)", async () => { const onlyB = { listIds: async () => ["r2", "r1"], diff --git a/evalboard/lib/__tests__/zip.test.ts b/evalboard/lib/__tests__/zip.test.ts index 573632d1..2dcd1fdd 100644 --- a/evalboard/lib/__tests__/zip.test.ts +++ b/evalboard/lib/__tests__/zip.test.ts @@ -43,6 +43,31 @@ describe("createZip", () => { expect(files["task-1/artifacts/main.py"]).toBe("print('hi')\n"); }); + // The per-entry CRC-32 comes from node:zlib rather than a hand-rolled + // per-byte loop. `unzip -t` verifies every entry's checksum against the + // stored one, so a wrong CRC fails here rather than silently producing + // archives that only some readers reject. + test("stores a CRC-32 the system unzip accepts", async () => { + const zip = await createZip([ + { name: "t/empty.txt", data: Buffer.alloc(0) }, + { name: "t/small.txt", data: Buffer.from("hello") }, + // Non-UTF8 bytes, so the checksum is over real binary content. + { name: "t/bin.dat", data: Buffer.from([0x00, 0xff, 0x80, 0x7f]) }, + { name: "t/big.txt", data: Buffer.from("ab".repeat(100_000)) }, + ]); + const dir = mkdtempSync(path.join(tmpdir(), "evalboard-zip-crc-")); + try { + const zipPath = path.join(dir, "a.zip"); + writeFileSync(zipPath, zip); + const out = execFileSync("unzip", ["-t", zipPath], { + encoding: "utf-8", + }); + expect(out).toMatch(/No errors detected/); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + test("compresses large repetitive payloads (DEFLATE, not STORE)", async () => { const big = Buffer.from("a".repeat(100_000)); const zip = await createZip([{ name: "big.txt", data: big }]); diff --git a/evalboard/lib/blob.ts b/evalboard/lib/blob.ts index 041454f5..9e9624fe 100644 --- a/evalboard/lib/blob.ts +++ b/evalboard/lib/blob.ts @@ -297,29 +297,9 @@ export async function ensureRunReviewIndex( }); } -// Full fetch: every blob under the run prefix. Used by the "download whole -// run" button, which needs all task subdirs at once (the narrow per-task / -// summary fetches only cache what their page reads). `.venv` trees are -// skipped for the same reason as ensureTaskDir — no page reads them and they -// dwarf the real deliverables. -export async function ensureRunDir( - container: string, - runId: string, - destRoot: string, -): Promise { - assertValidId(runId, "runId"); - if (LOCAL_RUNS_DIR) return; - return dedupe(`run:${container}:${runId}`, async () => { - const c = await getContainer(container); - const ops: Promise[] = []; - const prefix = `${runId}/`; - for await (const blob of c.listBlobsFlat({ prefix })) { - if (blob.name.includes("/.venv/")) continue; - ops.push(downloadBlob(container, blob.name, destRoot)); - } - await Promise.all(ops); - }); -} +// There is deliberately no whole-run fetch. `ensureRunDir` used to pull EVERY +// blob under the run prefix with no concurrency cap (~10k blobs / ~400 MB for a +// nightly). Per-task download, below, is the supported shape. // Narrow fetch: run.json + just one task subdir. Used by the per-task // detail page so opening a deep link to a 50-task run doesn't pull every diff --git a/evalboard/lib/overview.ts b/evalboard/lib/overview.ts index 3a159df0..b3343b62 100644 --- a/evalboard/lib/overview.ts +++ b/evalboard/lib/overview.ts @@ -324,6 +324,29 @@ async function loadPerRunForId( }; } +// First `YYYY-MM-DD` (optionally `_HH-MM-SS`) anywhere in a run id. Ad-hoc ids +// aren't date-SHAPED (parseRunIdDate anchors, so it rejects them) but almost all +// still carry their date: `adhoc-2026-09-02_21-59-13`, `skills-2026-05-28`. +const ADHOC_DATE_RE = /(\d{4})-(\d{2})-(\d{2})(?:_(\d{2})-(\d{2})-(\d{2}))?/; + +// Id-only start date for an ad-hoc run; null when the id carries no date. A +// date-only id resolves to the start of its day, which can only lose a same-day +// tie-break. +export function adhocRunDate(id: string): Date | null { + const m = ADHOC_DATE_RE.exec(id); + if (!m) return null; + const [, y, mo, d, h, mi, s] = m; + const t = Date.UTC( + Number(y), + Number(mo) - 1, + Number(d), + Number(h ?? 0), + Number(mi ?? 0), + Number(s ?? 0), + ); + return Number.isFinite(t) ? new Date(t) : null; +} + // Cache at per-run granularity, NOT per-window. A whole-window PerRun[] for // the 30d window exceeds unstable_cache's hard 2MB ceiling (≈400 tasks × ~20 // runs), and on overflow Next.js drops the write AND hands back a truncated @@ -333,24 +356,63 @@ async function loadPerRunForId( // windows + the trends page. The cross-run aggregation downstream is cheap // in-memory work, so leaving it uncached costs nothing. // -// One cached loader per source, with `source.id` in the key parts. Run ids are -// only unique WITHIN a container — both suites name runs `YYYY-MM-DD_HH-MM-SS`, -// so a source-blind key would let a Scribe run and a skills run with the same -// id serve each other's projection. +// One cached loader per (source, run), with `source.id` in the key parts. Run +// ids are only unique WITHIN a container — both suites name runs +// `YYYY-MM-DD_HH-MM-SS`, so a source-blind key would let a Scribe run and a +// skills run with the same id serve each other's projection. +// +// Per-run rather than per-source because `unstable_cache` fixes revalidate and +// tags at construction: one shared loader can hold neither a per-run TTL nor a +// per-run tag for the refresh button to evict. type PerRunLoader = (id: string) => Promise; -const perRunLoaders = new Map(); +// Keyed `:`; grows with history, not with traffic. +const perRunLoaders = new Map Promise>(); + +// A run.json is written once, at the end of the run, so a finished run is +// immutable. Its sidecars (meta.json, reviews, analysis) are not, which is what +// `runCacheTag` is for. +const SETTLED_AFTER_MS = 24 * 60 * 60 * 1000; +// Recent enough that today's nightly may still be landing. +const FRESH_REVALIDATE_SECONDS = 300; +// Older than that: the read this avoids is a multi-MB run.json off Azure Files. +const SETTLED_REVALIDATE_SECONDS = 24 * 60 * 60; + +// Cache tag for one run's projection, so POST /api/refresh can evict it; +// without it the button would no-op for anything past SETTLED_AFTER_MS. +export function runCacheTag(sourceId: string, runId: string): string { + return `evalboard-run:${sourceId}:${runId}`; +} + +// Settled runs cache for a day, everything else for 5 minutes. An id carrying no +// date at all is treated as fresh: those are hand-uploaded and re-uploaded far +// more often than pipeline runs. Evaluated once per run per process, so a run +// that settles while the process is up keeps the short TTL until restart. +function perRunRevalidate(id: string): number { + const started = parseRunIdDate(id) ?? adhocRunDate(id); + if (started == null) return FRESH_REVALIDATE_SECONDS; + return Date.now() - started.getTime() > SETTLED_AFTER_MS + ? SETTLED_REVALIDATE_SECONDS + : FRESH_REVALIDATE_SECONDS; +} function cachedLoadPerRunFor(source: Source): PerRunLoader { - const existing = perRunLoaders.get(source.id); - if (existing) return existing; - const loader = unstable_cache( - (id: string) => loadPerRunForId(id, source), - ["evalboard-per-run", source.id], - { revalidate: 300 }, - ); - perRunLoaders.set(source.id, loader); - return loader; + return (id: string) => { + const cacheKey = `${source.id}:${id}`; + let loader = perRunLoaders.get(cacheKey); + if (!loader) { + loader = unstable_cache( + () => loadPerRunForId(id, source), + ["evalboard-per-run", source.id, id], + { + revalidate: perRunRevalidate(id), + tags: [runCacheTag(source.id, id)], + }, + ); + perRunLoaders.set(cacheKey, loader); + } + return loader(); + }; } async function loadWindowDataInner( @@ -1161,13 +1223,19 @@ export function buildAdhocRows( }; } +// Extra candidates loaded beyond `limit`, covering ids with no readable overview +// (aborted uploads, the `deploys/` prefix) that then drop out of the rows. +const ADHOC_LOAD_SLACK = 10; + // The Ad-hoc runs section (front page, below the daily listing). "Ad-hoc" here // means "not a daily-pipeline run" — i.e. the id isn't date-shaped, which is // exactly the set listRunIdsInWindow excludes from the chart and main table. -// Every ad-hoc candidate is loaded before sorting (the date lives in run.json, -// not the id, so we can't window by id and still show the most recent): the -// ad-hoc set is small by construction (manual uploads only) and per-run reads -// are memoized for 5 min, so a warm front page pays no extra IO. +// +// Only the newest `limit + ADHOC_LOAD_SLACK` candidates are loaded, ordered by +// the date in the id; loading all 165 to show ten rows cost ~294 MB per cold +// render. run.json's `start_time` stays the authoritative sort, so the id only +// decides what to READ. Ids with no date are always loaded, so they can never be +// ordered out by a key they don't have. export async function getAdhocRunListing( limit: number | null, source: Source = DEFAULT_SOURCE, @@ -1175,10 +1243,25 @@ export async function getAdhocRunListing( const ids = (await listRunIds(source)).filter( (id) => parseRunIdDate(id) == null, ); + const dated: { id: string; at: number }[] = []; + const undated: string[] = []; + for (const id of ids) { + const at = adhocRunDate(id); + if (at == null) undated.push(id); + else dated.push({ id, at: at.getTime() }); + } + dated.sort((a, b) => b.at - a.at); + // null limit = "load everything": allowed by the signature, never used. + const budget = limit == null ? dated.length : limit + ADHOC_LOAD_SLACK; + const loadedAll = budget >= dated.length; + const toLoad = [...undated, ...dated.slice(0, budget).map((d) => d.id)]; const perRun = await mapWithConcurrency( - ids, + toLoad, FETCH_CONCURRENCY, cachedLoadPerRunFor(source), ); - return buildAdhocRows(perRun, limit); + const listing = buildAdhocRows(perRun, limit); + // `total` drives "Show more", so a truncated load must report the candidate + // count or the section caps itself at the first page. + return loadedAll ? listing : { ...listing, total: ids.length }; } diff --git a/evalboard/lib/pricing.ts b/evalboard/lib/pricing.ts index 20db9c0a..b858f715 100644 --- a/evalboard/lib/pricing.ts +++ b/evalboard/lib/pricing.ts @@ -136,7 +136,9 @@ const _ROUTING_PREFIXES = [ "openrouter/", ]; const _REGION_PREFIXES = ["eu.", "us.", "apac.", "global."]; -function normalizeModel(model: string): string { +// Exported so the run header's model tally groups on the same key pricing looks +// up on (lib/runs.ts::tallyModels). +export function normalizeModel(model: string): string { let m = model.trim(); for (const pre of _ROUTING_PREFIXES) { if (m.startsWith(pre)) { diff --git a/evalboard/lib/runs.ts b/evalboard/lib/runs.ts index ad31a97d..010a38b2 100644 --- a/evalboard/lib/runs.ts +++ b/evalboard/lib/runs.ts @@ -1,10 +1,10 @@ import { promises as fs } from "node:fs"; import path from "node:path"; +import { cache } from "react"; import { LOCAL_RUNS_DIR, ensureActivationSummary, ensureRunAnalysis, - ensureRunDir, ensureRunMeta, ensureRunSummary, ensureTaskDir, @@ -16,7 +16,7 @@ import { import { DEFAULT_VARIANT_ID, isValidVariantId } from "./variants"; import { DEFAULT_SOURCE, runsDirFor, type Source } from "./sources"; import { DELIVERABLE_KINDS, DELIVERABLE_NAMES } from "./artifact-kinds"; -import { messageCostUsd } from "./pricing"; +import { messageCostUsd, normalizeModel } from "./pricing"; // Resolution order: // 1. EVALBOARD_LOCAL_RUNS_DIR — local mode, points at a coder_eval runs dir @@ -693,14 +693,22 @@ async function readJson(p: string): Promise { } } -async function readRunJson( - id: string, - source: Source = DEFAULT_SOURCE, -): Promise { - const dir = runsDirFor(RUNS_DIR, source); - await ensureRunSummary(source.container, id, dir); - return readJson(path.join(dir, id, "run.json")); -} +// Request-scoped memo. A run.json is 1.5-6.5 MB and the run page reads the same +// one twice (readRunSummary and readRunTasks); `dedupe` in blob.ts collapses only +// the download, not the read and parse. react/cache is per-request, so this +// never serves one request's data to another. The key is the argument tuple, so +// an omitted `source` and an explicit DEFAULT_SOURCE are two keys: one extra +// read at worst, never the wrong container. +const readRunJson = cache( + async ( + id: string, + source: Source = DEFAULT_SOURCE, + ): Promise => { + const dir = runsDirFor(RUNS_DIR, source); + await ensureRunSummary(source.container, id, dir); + return readJson(path.join(dir, id, "run.json")); + }, +); // The activation suite is a nested sub-run: its self-contained run.json (enriched // cases + the per-skill rollup) lives at /activation/run.json, separate from @@ -843,24 +851,50 @@ export async function readRunSummary( // appeared. Mirrors mostCommonAgentType's "the thing these tasks ran on" vote, // but also reports the spread so the run header can say when there was more than // one instead of silently picking a winner. +// +// The vote groups on the NORMALIZED id (same key pricing resolves on) because +// the recorded string varies by code path: a row that errored before the model +// resolved keeps the qualified id it was configured with +// ("eu.anthropic.claude-sonnet-5") while completed rows record the bare one, so +// counting raw strings let one errored row read as a second model. +// +// Display stays the most common RAW string inside the winning group, so the chip +// shows what the run recorded rather than a derived value. export function tallyModels(rows: RawTaskResult[]): { dominant: string | null; distinct: number; } { - const counts = new Map(); + const groups = new Map>(); for (const r of rows) { const m = r.model_used; - if (typeof m === "string" && m) counts.set(m, (counts.get(m) ?? 0) + 1); + if (typeof m !== "string" || !m) continue; + const key = normalizeModel(m); + let raw = groups.get(key); + if (!raw) { + raw = new Map(); + groups.set(key, raw); + } + raw.set(m, (raw.get(m) ?? 0) + 1); } let dominant: string | null = null; let bestN = 0; - for (const [model, n] of counts) { - if (n > bestN) { - dominant = model; - bestN = n; + for (const raws of groups.values()) { + let groupN = 0; + let groupTop: string | null = null; + let groupTopN = 0; + for (const [model, n] of raws) { + groupN += n; + if (n > groupTopN) { + groupTop = model; + groupTopN = n; + } + } + if (groupN > bestN) { + dominant = groupTop; + bestN = groupN; } } - return { dominant, distinct: counts.size }; + return { dominant, distinct: groups.size }; } export async function readRunTasks( @@ -877,8 +911,8 @@ export async function readRunTasks( // slot (SLOT_COUNT = 5 in the runner's maturity.py), so its most recent real // execution is at most ~5 *canonical* runs back; the headroom absorbs ad-hoc / // smoke runs that listRunIds() interleaves but maturity replay ignores. The scan -// short-circuits as soon as every task is resolved, so this is a safety cap, not -// the typical read count. +// short-circuits once every task is resolved (at batch granularity), so this is +// a safety cap, not the typical read count. const MATURE_SOURCE_LOOKBACK = 20; // For each mature-skipped task in `fromRunId`, find the most recent *earlier* run @@ -909,13 +943,22 @@ export async function findMatureSourceRuns( const unresolved = new Set(matureTaskIds); const limit = Math.min(ids.length, start + 1 + MATURE_SOURCE_LOOKBACK); - for (let i = start + 1; i < limit && unresolved.size > 0; i++) { - const data = await readRun(ids[i]); - for (const t of data?.task_results ?? []) { - const tid = t.task_id; - if (!tid || !unresolved.has(tid) || t.mature_skipped) continue; - out[tid] = ids[i]; - unresolved.delete(tid); + // Read in concurrent batches, then CONSUME each batch in index order so "the + // most recent run that executed this task" is unchanged: the walk is still + // newest-first, only the IO overlaps. Serially this was up to 20 round-trips + // to Azure Files back-to-back before the page could render. Overshooting + // costs at most BATCH-1 extra reads, and those are request-memoized. + const BATCH = 5; + for (let i = start + 1; i < limit && unresolved.size > 0; i += BATCH) { + const batch = ids.slice(i, Math.min(i + BATCH, limit)); + const loaded = await Promise.all(batch.map((id) => readRun(id))); + for (let j = 0; j < batch.length && unresolved.size > 0; j++) { + for (const t of loaded[j]?.task_results ?? []) { + const tid = t.task_id; + if (!tid || !unresolved.has(tid) || t.mature_skipped) continue; + out[tid] = batch[j]; + unresolved.delete(tid); + } } } return out; @@ -2462,23 +2505,6 @@ export async function collectTaskFiles( return refs.map((r) => ({ relPath: r.relPath, abs: path.join(taskDir, r.relPath) })); } -// Collect every file under a whole run (`/`) for the download-as-zip -// button on the run page. Same noise filter / symlink skip as collectTaskFiles, -// applied across all task subdirs plus run-level files (run.json, analysis.md, -// meta.json, …). Returns null for an invalid id or a missing/empty run dir. -export async function collectRunFiles( - runId: string, - source: Source = DEFAULT_SOURCE, -): Promise<{ relPath: string; abs: string }[] | null> { - if (!isValidId(runId)) return null; - const dir = runsDirFor(RUNS_DIR, source); - await ensureRunDir(source.container, runId, dir); - const runDir = path.join(dir, runId); - const refs = await walkArtifacts(runDir); - if (refs.length === 0) return null; - return refs.map((r) => ({ relPath: r.relPath, abs: path.join(runDir, r.relPath) })); -} - export async function resolveSafePath( runId: string, relPath: string, diff --git a/evalboard/lib/zip.ts b/evalboard/lib/zip.ts index 66454546..2f0dbd5f 100644 --- a/evalboard/lib/zip.ts +++ b/evalboard/lib/zip.ts @@ -1,4 +1,4 @@ -import { deflateRaw } from "node:zlib"; +import { crc32, deflateRaw } from "node:zlib"; import { promisify } from "node:util"; // Async DEFLATE so compression runs on libuv's threadpool instead of blocking @@ -20,31 +20,12 @@ export interface ZipEntry { data: Buffer; } -// Standard CRC-32 (IEEE 802.3 polynomial 0xEDB88320), computed via a lazily -// built lookup table. ZIP stores a CRC-32 per entry for integrity. -let CRC_TABLE: Uint32Array | null = null; - -function crcTable(): Uint32Array { - if (CRC_TABLE) return CRC_TABLE; - const table = new Uint32Array(256); - for (let n = 0; n < 256; n++) { - let c = n; - for (let k = 0; k < 8; k++) { - c = c & 1 ? 0xedb88320 ^ (c >>> 1) : c >>> 1; - } - table[n] = c >>> 0; - } - CRC_TABLE = table; - return table; -} - -function crc32(buf: Buffer): number { - const table = crcTable(); - let crc = 0xffffffff; - for (let i = 0; i < buf.length; i++) { - crc = table[(crc ^ buf[i]) & 0xff] ^ (crc >>> 8); - } - return (crc ^ 0xffffffff) >>> 0; +// Standard CRC-32 (IEEE 802.3 polynomial 0xEDB88320), one per ZIP entry. +// `zlib.crc32` computes it natively; the hand-rolled loop it replaces walked the +// uncompressed bytes one at a time on the event loop, ~47x slower measured. +// `>>> 0` keeps it unsigned, matching what that loop returned. +function entryCrc(buf: Buffer): number { + return crc32(buf) >>> 0; } // Convert a JS Date to the DOS date/time fields ZIP uses. Seconds have 2 s @@ -86,7 +67,7 @@ export async function createZip( const { method, body } = await compress(entry.data); return { nameBuf: Buffer.from(entry.name, "utf-8"), - crc: crc32(entry.data), + crc: entryCrc(entry.data), method, body, uncompressedSize: entry.data.length,