feat: upload eval-results.json to Braintrust (AI-921)

.env.example

@@ -1,3 +1,7 @@
 # AI SDK Core direct-provider credentials.
 ANTHROPIC_API_KEY=
 OPENAI_API_KEY=
+
+# Braintrust — required for `pnpm upload:braintrust` (not for `--dry`).
+# Optional: BRAINTRUST_PROJECT overrides the target project (default "supabase-evals").
+BRAINTRUST_API_KEY=

.github/workflows/eval-refresh.yml

@@ -293,6 +293,10 @@ jobs:
     needs: [prepare, run-evals]
     if: needs.prepare.outputs.evals != '[]'
     runs-on: ubuntu-latest
+    env:
+      # Surfaced to the job so the guarded Braintrust step can gate on presence.
+      # Unset (empty) when the secret isn't configured — the step is then skipped.
+      BRAINTRUST_API_KEY: ${{ secrets.BRAINTRUST_API_KEY }}
     steps:
       - name: Checkout
         uses: actions/checkout@9f698171ed81b15d1823a05fc7211befd50c8ae0 # v6.0.3
@@ -350,6 +354,17 @@ jobs:
 
           pnpm --filter @supabase-evals/framework export-results -- "${export_args[@]}"
 
+      # Best-effort mirror of the exported snapshot into Braintrust (AI-921).
+      # Gated to manual dispatch so only deliberate, canonical refreshes are
+      # mirrored — label-triggered PR runs would otherwise push transient
+      # preview results into the shared project. Skipped when BRAINTRUST_API_KEY
+      # isn't configured; never fails the run so the JSON-commit path stays
+      # authoritative while we trial it.
+      - name: Upload results to Braintrust
+        if: ${{ github.event_name == 'workflow_dispatch' && env.BRAINTRUST_API_KEY != '' }}
+        continue-on-error: true
+        run: pnpm --filter @supabase-evals/framework upload:braintrust
+
       - name: Upload exported results
         if: github.event_name == 'pull_request' || github.event_name == 'workflow_dispatch'
         uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1

README.md

@@ -89,6 +89,25 @@ Or run everything:
 pnpm eval
 ```
 
+### Upload results to Braintrust
+
+Mirror the exported `apps/web/src/data/eval-results.json` snapshot into
+[Braintrust experiments](https://www.braintrust.dev/docs/guides/evals) (AI-921) to
+trial their comparison/leaderboard UI. One Braintrust experiment is created per
+result-file experiment (Configuration) name, one event per eval; `passed` and the
+per-check pass rate become scores and the remaining fields become metadata/tags.
+
+```bash
+pnpm upload:braintrust            # upload the default snapshot
+pnpm upload:braintrust -- --dry   # map + print only, no network (no key needed)
+pnpm upload:braintrust -- --update # append to existing named experiments
+```
+
+Requires `BRAINTRUST_API_KEY` in `.env` (override the target project with
+`BRAINTRUST_PROJECT`, default `supabase-evals`). In CI, the `eval-refresh`
+workflow runs this as a best-effort step only when the `BRAINTRUST_API_KEY`
+secret is set; the committed JSON stays authoritative for now.
+
 ## Eval Shape
 
 Every eval contains:

apps/framework/lib/braintrust-mapping.test.ts

@@ -0,0 +1,163 @@
+import { describe, expect, it } from "vitest";
+import {
+  experimentMetadata,
+  groupByExperiment,
+  isResultRow,
+  metadataFor,
+  scoresFor,
+  tagsFor,
+  type ResultRow,
+} from "./braintrust-mapping.js";
+
+function makeRow(overrides: Partial<ResultRow> = {}): ResultRow {
+  return {
+    experiment: "claude-code-opus-4.8",
+    experimentSuite: "benchmark",
+    experimentDisplay: {
+      agent: "claude-code",
+      modelProvider: "anthropic",
+      modelId: "claude-opus-4-8",
+      reasoningEffort: "high",
+    },
+    eval: "build-cli-001-bootstrap-app",
+    stage: "build",
+    product: ["database", "data-api"],
+    topic: ["migrations", "rls"],
+    suite: "benchmark",
+    interface: "cli",
+    passed: true,
+    checks: [
+      { name: "a", passed: true },
+      { name: "b", passed: false },
+    ],
+    attempts: 1,
+    skills: { available: ["supabase"], loaded: ["supabase"] },
+    sourcePath: "claude-code-opus-4.8/build-cli-001-bootstrap-app.json",
+    ...overrides,
+  } as ResultRow;
+}
+
+describe("isResultRow", () => {
+  it("accepts rows with string experiment and eval", () => {
+    expect(isResultRow(makeRow())).toBe(true);
+  });
+
+  it("rejects non-objects and rows missing required keys", () => {
+    expect(isResultRow(null)).toBe(false);
+    expect(isResultRow("x")).toBe(false);
+    expect(isResultRow({ experiment: "x" })).toBe(false);
+    expect(isResultRow({ eval: "x" })).toBe(false);
+    expect(isResultRow({ experiment: 1, eval: 2 })).toBe(false);
+  });
+});
+
+describe("groupByExperiment", () => {
+  it("groups rows by experiment, preserving order", () => {
+    const rows = [
+      makeRow({ experiment: "a", eval: "e1" }),
+      makeRow({ experiment: "b", eval: "e2" }),
+      makeRow({ experiment: "a", eval: "e3" }),
+    ];
+    const groups = groupByExperiment(rows);
+    expect([...groups.keys()]).toEqual(["a", "b"]);
+    expect(groups.get("a")?.map((r) => r.eval)).toEqual(["e1", "e3"]);
+    expect(groups.get("b")?.map((r) => r.eval)).toEqual(["e2"]);
+  });
+
+  it("returns an empty map for no rows", () => {
+    expect(groupByExperiment([]).size).toBe(0);
+  });
+});
+
+describe("scoresFor", () => {
+  it("maps passed to 1/0 and computes the check pass rate", () => {
+    expect(scoresFor(makeRow({ passed: true }))).toEqual({
+      passed: 1,
+      checkPassRate: 0.5,
+    });
+    expect(scoresFor(makeRow({ passed: false }))).toEqual({
+      passed: 0,
+      checkPassRate: 0.5,
+    });
+  });
+
+  it("falls back to passed when there are no checks", () => {
+    expect(scoresFor(makeRow({ passed: true, checks: [] }))).toEqual({
+      passed: 1,
+      checkPassRate: 1,
+    });
+    expect(scoresFor(makeRow({ passed: false, checks: undefined }))).toEqual({
+      passed: 0,
+      checkPassRate: 0,
+    });
+  });
+
+  it("reports a perfect rate when every check passes", () => {
+    const row = makeRow({
+      checks: [
+        { name: "a", passed: true },
+        { name: "b", passed: true },
+      ],
+    });
+    expect(scoresFor(row).checkPassRate).toBe(1);
+  });
+});
+
+describe("tagsFor", () => {
+  it("combines interface, suite, product, and topic, deduped", () => {
+    expect(tagsFor(makeRow())).toEqual([
+      "cli",
+      "benchmark",
+      "database",
+      "data-api",
+      "migrations",
+      "rls",
+    ]);
+  });
+
+  it("drops undefined/empty values", () => {
+    const row = makeRow({
+      interface: undefined,
+      experimentSuite: undefined,
+      product: undefined,
+      topic: ["rls"],
+    });
+    expect(tagsFor(row)).toEqual(["rls"]);
+  });
+
+  it("de-duplicates overlapping product/topic values", () => {
+    const row = makeRow({ product: ["database"], topic: ["database"] } as Partial<ResultRow>);
+    expect(tagsFor(row)).toEqual(["cli", "benchmark", "database"]);
+  });
+});
+
+describe("metadataFor", () => {
+  it("flattens experimentDisplay into first-class grouping keys", () => {
+    const meta = metadataFor(makeRow());
+    expect(meta.agent).toBe("claude-code");
+    expect(meta.modelProvider).toBe("anthropic");
+    expect(meta.modelId).toBe("claude-opus-4-8");
+    expect(meta.reasoningEffort).toBe("high");
+    expect(meta.skillsLoaded).toEqual(["supabase"]);
+    expect(meta.eval).toBe("build-cli-001-bootstrap-app");
+  });
+
+  it("tolerates a missing experimentDisplay", () => {
+    const meta = metadataFor(makeRow({ experimentDisplay: undefined }));
+    expect(meta.agent).toBeUndefined();
+    expect(meta.modelId).toBeUndefined();
+  });
+});
+
+describe("experimentMetadata", () => {
+  it("carries the per-configuration dimensions and a source marker", () => {
+    expect(experimentMetadata(makeRow())).toEqual({
+      experimentSuite: "benchmark",
+      agent: "claude-code",
+      modelProvider: "anthropic",
+      modelId: "claude-opus-4-8",
+      reasoningEffort: "high",
+      source: "eval-results.json",
+    });
+  });
+});

apps/framework/lib/braintrust-mapping.ts

@@ -0,0 +1,103 @@
+/**
+ * Pure mapping from the exported `eval-results.json` shape onto the fields
+ * Braintrust's `experiment.log()` expects. Kept free of any I/O or SDK calls so
+ * it can be unit-tested without credentials or network (see the sibling test).
+ */
+import type { EvalResult } from "@supabase-evals/core/eval-metadata";
+
+// A result row as it appears in the web-facing snapshot: the core EvalResult
+// plus the display/source fields export-results.ts adds.
+export type ResultRow = EvalResult & {
+  experimentSuite?: string;
+  prompt?: string;
+  promptSourcePath?: string;
+  sourcePath?: string;
+};
+
+export function isResultRow(value: unknown): value is ResultRow {
+  return (
+    typeof value === "object" &&
+    value !== null &&
+    typeof (value as ResultRow).experiment === "string" &&
+    typeof (value as ResultRow).eval === "string"
+  );
+}
+
+export function groupByExperiment(rows: ResultRow[]): Map<string, ResultRow[]> {
+  const groups = new Map<string, ResultRow[]>();
+  for (const row of rows) {
+    const existing = groups.get(row.experiment);
+    if (existing) {
+      existing.push(row);
+    } else {
+      groups.set(row.experiment, [row]);
+    }
+  }
+  return groups;
+}
+
+/**
+ * Numeric, cross-eval-comparable scores. `passed` mirrors the run's pass@k
+ * outcome; `checkPassRate` is the fraction of individual checks that passed,
+ * falling back to `passed` when a run has no per-check breakdown.
+ */
+export function scoresFor(row: ResultRow): Record<string, number> {
+  const passed = row.passed === true ? 1 : 0;
+  const checks = row.checks ?? [];
+  const checkPassRate = checks.length
+    ? checks.filter((check) => check.passed).length / checks.length
+    : passed;
+  return { passed, checkPassRate };
+}
+
+export function metadataFor(row: ResultRow): Record<string, unknown> {
+  return {
+    eval: row.eval,
+    experiment: row.experiment,
+    experimentSuite: row.experimentSuite,
+    stage: row.stage,
+    product: row.product,
+    topic: row.topic,
+    suite: row.suite,
+    interface: row.interface,
+    cliVersion: row.cliVersion,
+    attempts: row.attempts,
+    // Flatten experimentDisplay so each dimension is a first-class grouping axis.
+    agent: row.experimentDisplay?.agent,
+    modelProvider: row.experimentDisplay?.modelProvider,
+    modelId: row.experimentDisplay?.modelId,
+    reasoningEffort: row.experimentDisplay?.reasoningEffort,
+    skillsAvailable: row.skills?.available,
+    skillsLoaded: row.skills?.loaded,
+    promptSourcePath: row.promptSourcePath,
+    sourcePath: row.sourcePath,
+  };
+}
+
+export function tagsFor(row: ResultRow): string[] {
+  const candidates: Array<string | undefined> = [
+    row.interface,
+    row.experimentSuite,
+    ...(row.product ?? []),
+    ...(row.topic ?? []),
+  ];
+  const tags = candidates.filter(
+    (tag): tag is string => typeof tag === "string" && tag.length > 0,
+  );
+  return [...new Set(tags)];
+}
+
+/**
+ * Experiment-level metadata, derived from the first row of the group. These are
+ * constant per Configuration, so they belong on the experiment, not each event.
+ */
+export function experimentMetadata(row: ResultRow): Record<string, unknown> {
+  return {
+    experimentSuite: row.experimentSuite,
+    agent: row.experimentDisplay?.agent,
+    modelProvider: row.experimentDisplay?.modelProvider,
+    modelId: row.experimentDisplay?.modelId,
+    reasoningEffort: row.experimentDisplay?.reasoningEffort,
+    source: "eval-results.json",
+  };
+}

apps/framework/package.json

@@ -4,13 +4,15 @@
   "version": "0.0.1",
   "type": "module",
   "scripts": {
-    "check": "pnpm typecheck && pnpm test:framework",
+    "check": "pnpm typecheck && pnpm test:unit && pnpm test:framework",
+    "test:unit": "vitest run",
     "eval": "node --env-file=../../.env --import tsx/esm harness/run-eval.ts",
     "eval:dry": "node --env-file=../../.env --import tsx/esm harness/run-eval.ts --dry",
     "eval:smoke": "node --env-file=../../.env --import tsx/esm harness/run-eval.ts --smoke",
     "typecheck": "tsc --noEmit",
     "test:framework": "node --env-file-if-exists=../../.env --import tsx/esm scripts/smoke-framework.ts",
     "export-results": "node --import tsx/esm scripts/export-results.ts",
+    "upload:braintrust": "node --env-file-if-exists=../../.env --import tsx/esm scripts/upload-to-braintrust.ts",
     "demo:mcp": "node --env-file=../../.env --import tsx/esm scripts/mcp-demo.ts",
     "demo:executor": "node --env-file=../../.env --import tsx/esm scripts/executor-demo.ts"
   },
@@ -27,6 +29,7 @@
     "@testing-library/react": "^16.3.2",
     "@vitejs/plugin-react": "catalog:",
     "ai": "catalog:",
+    "braintrust": "catalog:",
     "happy-dom": "^20.9.0",
     "@supabase/lite": "catalog:",
     "react": "^19.2.5",