feat: surface eval run performance metrics

apps/framework/harness/run-eval.ts

@@ -34,6 +34,7 @@ import {
   getExperimentDisplayMetadata,
 } from '@supabase-evals/core';
 import type {
+  AgentUsage,
   ExperimentConfig,
   EvalInterface,
   EvalManifest,
@@ -360,6 +361,8 @@ async function runOne(
     transcript: TranscriptPart[];
     agentReport: string;
     stoppedReason: string;
+    usage?: AgentUsage;
+    durationMs?: number;
   }
 > {
   const prompt = parseEvalMarkdown(
@@ -388,6 +391,11 @@ async function runOne(
   let lastTranscript: TranscriptPart[] = [];
   let lastAgentReport = '';
   let lastStoppedReason = 'not_started';
+  // Performance metrics for the attempt that produced the reported score:
+  // the agent's own token/cost accounting and its wall-clock time (agent run
+  // only — sandbox boot and scoring excluded, so harnesses compare fairly).
+  let lastUsage: AgentUsage | undefined;
+  let lastDurationMs: number | undefined;
 
   for (let attempt = 1; attempt <= RUNS; attempt += 1) {
     if (ev.mode === 'local-stack') {
@@ -442,6 +450,7 @@ async function runOne(
         })
       );
 
+      const agentStart = Date.now();
       const run = await exp.agent.run({
         systemPrompt: buildSystemPrompt('local-stack', session.promptAddendum),
         userPrompt: prompt,
@@ -457,6 +466,8 @@ async function runOne(
       lastTranscript = run.transcript;
       lastAgentReport = run.agentReport;
       lastStoppedReason = run.stoppedReason;
+      lastUsage = run.usage;
+      lastDurationMs = Date.now() - agentStart;
 
       // Export the agent's workspace to the host so scorers can run host
       // tooling (vite/vitest from the repo root) against the produced files
@@ -495,6 +506,8 @@ async function runOne(
           transcript: run.transcript,
           agentReport: run.agentReport,
           stoppedReason: run.stoppedReason,
+          usage: lastUsage,
+          durationMs: lastDurationMs,
         };
       }
       logRetryAttempt(expName, ev, attempt, last);
@@ -528,6 +541,7 @@ async function runOne(
       session.promptAddendum,
       skillsPrompt
     );
+    const agentStart = Date.now();
     const run = await exp.agent.run({
       systemPrompt,
       userPrompt: prompt,
@@ -544,6 +558,8 @@ async function runOne(
     lastTranscript = run.transcript;
     lastAgentReport = run.agentReport;
     lastStoppedReason = run.stoppedReason;
+    lastUsage = run.usage;
+    lastDurationMs = Date.now() - agentStart;
     last = await (scorer as ToolScorer)({
       ...session.scoringContext,
       toolCalls: run.toolCalls,
@@ -561,6 +577,8 @@ async function runOne(
         transcript: run.transcript,
         agentReport: run.agentReport,
         stoppedReason: run.stoppedReason,
+        usage: lastUsage,
+        durationMs: lastDurationMs,
       };
     }
     logRetryAttempt(expName, ev, attempt, last);
@@ -575,6 +593,8 @@ async function runOne(
     transcript: lastTranscript,
     agentReport: lastAgentReport,
     stoppedReason: lastStoppedReason,
+    usage: lastUsage,
+    durationMs: lastDurationMs,
   };
 }
 

apps/framework/harness/types.ts

@@ -6,6 +6,7 @@ import type {
 } from '@supabase-evals/core/eval-metadata';
 
 export type {
+  AgentUsage,
   ScoreResult,
   CheckResult,
   ToolCallRecord,

apps/framework/scripts/export-results.ts

@@ -129,6 +129,8 @@ async function readResultFile(
     checks: parsedResult.checks,
     skills: parsedResult.skills,
     docs: parsedResult.docs,
+    usage: parsedResult.usage,
+    durationMs: parsedResult.durationMs,
     prompt: promptData?.prompt,
     promptSourcePath: promptData?.promptSourcePath,
     attempts: parsedResult.attempts,

apps/web/src/components/results/eval-details.tsx

@@ -7,8 +7,18 @@ import {
   XIcon,
 } from "lucide-react"
 
-import { passFailClassName } from "@/components/results/table-shared"
-import type { CheckResult, DocsCall, ParsedResult } from "@/lib/eval-results"
+import {
+  formatCost,
+  formatDuration,
+  formatTokens,
+  passFailClassName,
+} from "@/components/results/table-shared"
+import {
+  runTokens,
+  type CheckResult,
+  type DocsCall,
+  type ParsedResult,
+} from "@/lib/eval-results"
 import { formatProductLabel, formatTagLabel } from "@/lib/format"
 import { cn } from "@/lib/utils"
 
@@ -193,8 +203,22 @@ function ResultDocsCalls({ calls }: { calls: DocsCall[] }) {
   )
 }
 
+/** "1.2M tokens (982k in / 218k out)" — total with the in/out split when known. */
+function tokensLabel(result: ParsedResult): string | undefined {
+  const total = runTokens(result)
+  if (total === undefined) return undefined
+  const { inputTokens, outputTokens } = result.usage ?? {}
+  const split =
+    inputTokens !== undefined && outputTokens !== undefined
+      ? ` (${formatTokens(inputTokens)} in / ${formatTokens(outputTokens)} out)`
+      : ""
+  return `${formatTokens(total)}${split}`
+}
+
 /** Everything recorded about one run, shown when its row in the sheet is expanded. */
 export function EvalDetails({ result }: { result: ParsedResult }) {
+  const tokens = tokensLabel(result)
+
   return (
     <dl className={evalMetaGridClassName}>
       {result.prompt ? (
@@ -210,6 +234,19 @@ export function EvalDetails({ result }: { result: ParsedResult }) {
         />
       ) : null}
       <EvalMetadataRow label="Attempts" value={result.attempts ?? "-"} />
+      {result.durationMs !== undefined ? (
+        <EvalMetadataRow
+          label="Agent time"
+          value={formatDuration(result.durationMs)}
+        />
+      ) : null}
+      {tokens ? <EvalMetadataRow label="Tokens" value={tokens} /> : null}
+      {result.usage?.costUsd !== undefined ? (
+        <EvalMetadataRow
+          label="Cost"
+          value={formatCost(result.usage.costUsd)}
+        />
+      ) : null}
       <EvalMetadataRow
         label="Source"
         value={<span className="break-all">{result.sourcePath}</span>}

apps/web/src/components/results/table-shared.test.ts

@@ -1,6 +1,9 @@
 import { describe, expect, it } from "vitest"
 
 import {
+  formatCost,
+  formatDuration,
+  formatTokens,
   hasMoreContentToRight,
   scoreLabel,
 } from "@/components/results/table-shared"
@@ -13,6 +16,24 @@ describe("scoreLabel", () => {
   })
 })
 
+describe("metric formatters", () => {
+  it("formats durations as seconds under a minute, minutes above", () => {
+    expect(formatDuration(38_000)).toBe("38s")
+    expect(formatDuration(245_000)).toBe("4m 05s")
+  })
+
+  it("formats token counts compactly", () => {
+    expect(formatTokens(845)).toBe("845")
+    expect(formatTokens(12_400)).toBe("12k")
+    expect(formatTokens(1_230_000)).toBe("1.2M")
+  })
+
+  it("keeps cents-level costs from rounding away", () => {
+    expect(formatCost(0.042)).toBe("$0.042")
+    expect(formatCost(1.5)).toBe("$1.50")
+  })
+})
+
 describe("hasMoreContentToRight", () => {
   it("only reports overflow before the right edge", () => {
     expect(

apps/web/src/components/results/table-shared.ts

@@ -43,6 +43,27 @@ export function scoreLabel(
   return `${Math.round((passed / total) * 100)}%`
 }
 
+/** Formats a run duration: "38s" under a minute, "4m 05s" above. */
+export function formatDuration(ms: number) {
+  const totalSec = Math.round(ms / 1000)
+  const min = Math.floor(totalSec / 60)
+  const sec = totalSec % 60
+  if (!min) return `${sec}s`
+  return `${min}m ${String(sec).padStart(2, "0")}s`
+}
+
+/** Formats a token count compactly: "845", "12k", "1.2M". */
+export function formatTokens(count: number) {
+  if (count >= 1_000_000) return `${(count / 1_000_000).toFixed(1)}M`
+  if (count >= 1_000) return `${Math.round(count / 1_000)}k`
+  return `${Math.round(count)}`
+}
+
+/** Formats a USD cost, keeping cents-level runs from rounding to $0.00. */
+export function formatCost(usd: number) {
+  return usd < 0.1 ? `$${usd.toFixed(3)}` : `$${usd.toFixed(2)}`
+}
+
 /** Checks whether a horizontal scroller has content beyond its right edge. */
 export function hasMoreContentToRight({
   scrollLeft,