From acce9594df60f3291a2a85d032a435e70b71b09a Mon Sep 17 00:00:00 2001 From: Pedro Rodrigues Date: Tue, 14 Jul 2026 17:27:53 +0100 Subject: [PATCH 1/3] feat: upload eval-results.json to Braintrust (AI-921) Adds a pnpm upload:braintrust command that mirrors the exported eval-results.json snapshot into Braintrust experiments (one experiment per Configuration, one event per eval; passed/checkPassRate as scores, everything else as metadata/tags) to trial Braintrust's comparison UI. Wires a best-effort, secret-gated upload into the eval-refresh workflow, limited to manual workflow_dispatch refreshes so the committed JSON stays authoritative and PR-preview runs don't pollute the shared project. Co-Authored-By: Claude Opus 4.8 (1M context) --- .github/workflows/eval-refresh.yml | 15 + README.md | 19 ++ apps/framework/package.json | 2 + .../framework/scripts/upload-to-braintrust.ts | 238 +++++++++++++++ package.json | 1 + pnpm-lock.yaml | 288 ++++++++++++++++++ pnpm-workspace.yaml | 1 + 7 files changed, 564 insertions(+) create mode 100644 apps/framework/scripts/upload-to-braintrust.ts diff --git a/.github/workflows/eval-refresh.yml b/.github/workflows/eval-refresh.yml index 8af591d9..3c93c642 100644 --- a/.github/workflows/eval-refresh.yml +++ b/.github/workflows/eval-refresh.yml @@ -293,6 +293,10 @@ jobs: needs: [prepare, run-evals] if: needs.prepare.outputs.evals != '[]' runs-on: ubuntu-latest + env: + # Surfaced to the job so the guarded Braintrust step can gate on presence. + # Unset (empty) when the secret isn't configured — the step is then skipped. + BRAINTRUST_API_KEY: ${{ secrets.BRAINTRUST_API_KEY }} steps: - name: Checkout uses: actions/checkout@9f698171ed81b15d1823a05fc7211befd50c8ae0 # v6.0.3 @@ -350,6 +354,17 @@ jobs: pnpm --filter @supabase-evals/framework export-results -- "${export_args[@]}" + # Best-effort mirror of the exported snapshot into Braintrust (AI-921). + # Gated to manual dispatch so only deliberate, canonical refreshes are + # mirrored — label-triggered PR runs would otherwise push transient + # preview results into the shared project. Skipped when BRAINTRUST_API_KEY + # isn't configured; never fails the run so the JSON-commit path stays + # authoritative while we trial it. + - name: Upload results to Braintrust + if: ${{ github.event_name == 'workflow_dispatch' && env.BRAINTRUST_API_KEY != '' }} + continue-on-error: true + run: pnpm --filter @supabase-evals/framework upload:braintrust + - name: Upload exported results if: github.event_name == 'pull_request' || github.event_name == 'workflow_dispatch' uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 diff --git a/README.md b/README.md index a40ac58d..3f5d1811 100644 --- a/README.md +++ b/README.md @@ -89,6 +89,25 @@ Or run everything: pnpm eval ``` +### Upload results to Braintrust + +Mirror the exported `apps/web/src/data/eval-results.json` snapshot into +[Braintrust experiments](https://www.braintrust.dev/docs/guides/evals) (AI-921) to +trial their comparison/leaderboard UI. One Braintrust experiment is created per +result-file experiment (Configuration) name, one event per eval; `passed` and the +per-check pass rate become scores and the remaining fields become metadata/tags. + +```bash +pnpm upload:braintrust # upload the default snapshot +pnpm upload:braintrust -- --dry # map + print only, no network (no key needed) +pnpm upload:braintrust -- --update # append to existing named experiments +``` + +Requires `BRAINTRUST_API_KEY` in `.env` (override the target project with +`BRAINTRUST_PROJECT`, default `supabase-evals`). In CI, the `eval-refresh` +workflow runs this as a best-effort step only when the `BRAINTRUST_API_KEY` +secret is set; the committed JSON stays authoritative for now. + ## Eval Shape Every eval contains: diff --git a/apps/framework/package.json b/apps/framework/package.json index c726bc37..826cb934 100644 --- a/apps/framework/package.json +++ b/apps/framework/package.json @@ -11,6 +11,7 @@ "typecheck": "tsc --noEmit", "test:framework": "node --env-file-if-exists=../../.env --import tsx/esm scripts/smoke-framework.ts", "export-results": "node --import tsx/esm scripts/export-results.ts", + "upload:braintrust": "node --env-file-if-exists=../../.env --import tsx/esm scripts/upload-to-braintrust.ts", "demo:mcp": "node --env-file=../../.env --import tsx/esm scripts/mcp-demo.ts", "demo:executor": "node --env-file=../../.env --import tsx/esm scripts/executor-demo.ts" }, @@ -27,6 +28,7 @@ "@testing-library/react": "^16.3.2", "@vitejs/plugin-react": "catalog:", "ai": "catalog:", + "braintrust": "catalog:", "happy-dom": "^20.9.0", "@supabase/lite": "catalog:", "react": "^19.2.5", diff --git a/apps/framework/scripts/upload-to-braintrust.ts b/apps/framework/scripts/upload-to-braintrust.ts new file mode 100644 index 00000000..b25cbd68 --- /dev/null +++ b/apps/framework/scripts/upload-to-braintrust.ts @@ -0,0 +1,238 @@ +#!/usr/bin/env tsx +/** + * Upload the exported `eval-results.json` snapshot to Braintrust. + * + * This is the first, deliberately small step of AI-921: store eval results in + * Braintrust instead of (or alongside) the committed JSON file. It does NOT run + * evals — it reads the same snapshot the web app consumes and mirrors it into + * Braintrust experiments so we can try their built-in comparison/leaderboard UI. + * + * Mapping (one Braintrust **experiment** per result-file `experiment` / + * Configuration name, one **event** per eval): + * - id ← eval id (stable, so `--update` re-logs the same row) + * - input ← { eval, prompt } + * - output ← { passed, checks } + * - scores ← { passed: 0|1, checkPassRate: 0..1 } (comparable across evals) + * - metadata ← everything else, with experimentDisplay flattened to top-level + * keys (agent, modelProvider, modelId, reasoningEffort) so they + * can be used directly as Braintrust grouping axes + * - tags ← product + topic + interface + experimentSuite (for filtering) + * + * Usage: + * pnpm upload:braintrust # upload default snapshot + * pnpm upload:braintrust -- --dry # map + print, no network (no key needed) + * pnpm upload:braintrust -- --update # append to existing named experiments + * pnpm upload:braintrust -- --project my-project --input path/to/results.json + * + * Requires BRAINTRUST_API_KEY in the environment (loaded from repo-root .env) + * unless running with --dry. + */ +import { readFile } from "node:fs/promises"; +import { dirname, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; +import { init } from "braintrust"; +import type { EvalResult } from "@supabase-evals/core/eval-metadata"; + +const __dirname = dirname(fileURLToPath(import.meta.url)); +const ROOT = resolve(__dirname, "..", "..", ".."); +const DEFAULT_INPUT = resolve( + ROOT, + "apps", + "web", + "src", + "data", + "eval-results.json", +); +const DEFAULT_PROJECT = process.env.BRAINTRUST_PROJECT ?? "supabase-evals"; + +function readFlag(args: string[], name: string): string | undefined { + const index = args.indexOf(`--${name}`); + if (index === -1 || index === args.length - 1) { + return undefined; + } + return args[index + 1]; +} + +const rawArgs = process.argv.slice(2); +const DRY = rawArgs.includes("--dry"); +const UPDATE = rawArgs.includes("--update"); +const INPUT_PATH = resolve(ROOT, readFlag(rawArgs, "input") ?? DEFAULT_INPUT); +const PROJECT = readFlag(rawArgs, "project") ?? DEFAULT_PROJECT; + +// A result row as it appears in the web-facing snapshot: the core EvalResult +// plus the display/source fields export-results.ts adds. +type ResultRow = EvalResult & { + experimentSuite?: string; + prompt?: string; + promptSourcePath?: string; + sourcePath?: string; +}; + +function isResultRow(value: unknown): value is ResultRow { + return ( + typeof value === "object" && + value !== null && + typeof (value as ResultRow).experiment === "string" && + typeof (value as ResultRow).eval === "string" + ); +} + +async function loadResults(path: string): Promise { + const parsed: unknown = JSON.parse(await readFile(path, "utf8")); + if (!Array.isArray(parsed)) { + throw new Error(`Expected an array of results in ${path}`); + } + const rows = parsed.filter(isResultRow); + const skipped = parsed.length - rows.length; + if (skipped > 0) { + console.warn( + `Skipped ${skipped} malformed record(s) (missing experiment/eval).`, + ); + } + return rows; +} + +function groupByExperiment(rows: ResultRow[]): Map { + const groups = new Map(); + for (const row of rows) { + const existing = groups.get(row.experiment); + if (existing) { + existing.push(row); + } else { + groups.set(row.experiment, [row]); + } + } + return groups; +} + +function scoresFor(row: ResultRow): Record { + const passed = row.passed === true ? 1 : 0; + const checks = row.checks ?? []; + const checkPassRate = checks.length + ? checks.filter((check) => check.passed).length / checks.length + : passed; + return { passed, checkPassRate }; +} + +function metadataFor(row: ResultRow): Record { + return { + eval: row.eval, + experiment: row.experiment, + experimentSuite: row.experimentSuite, + stage: row.stage, + product: row.product, + topic: row.topic, + suite: row.suite, + interface: row.interface, + cliVersion: row.cliVersion, + attempts: row.attempts, + // Flatten experimentDisplay so each dimension is a first-class grouping axis. + agent: row.experimentDisplay?.agent, + modelProvider: row.experimentDisplay?.modelProvider, + modelId: row.experimentDisplay?.modelId, + reasoningEffort: row.experimentDisplay?.reasoningEffort, + skillsAvailable: row.skills?.available, + skillsLoaded: row.skills?.loaded, + promptSourcePath: row.promptSourcePath, + sourcePath: row.sourcePath, + }; +} + +function tagsFor(row: ResultRow): string[] { + const candidates: Array = [ + row.interface, + row.experimentSuite, + ...(row.product ?? []), + ...(row.topic ?? []), + ]; + const tags = candidates.filter( + (tag): tag is string => typeof tag === "string" && tag.length > 0, + ); + return [...new Set(tags)]; +} + +// Experiment-level metadata, derived from the first row of the group. These are +// constant per Configuration, so they belong on the experiment, not each event. +function experimentMetadata(row: ResultRow): Record { + return { + experimentSuite: row.experimentSuite, + agent: row.experimentDisplay?.agent, + modelProvider: row.experimentDisplay?.modelProvider, + modelId: row.experimentDisplay?.modelId, + reasoningEffort: row.experimentDisplay?.reasoningEffort, + source: "eval-results.json", + }; +} + +async function uploadExperiment(name: string, rows: ResultRow[]): Promise { + const experiment = init(PROJECT, { + experiment: name, + update: UPDATE, + metadata: experimentMetadata(rows[0]), + }); + + for (const row of rows) { + experiment.log({ + id: row.eval, + input: { eval: row.eval, prompt: row.prompt }, + output: { passed: row.passed === true, checks: row.checks ?? [] }, + scores: scoresFor(row), + metadata: metadataFor(row), + tags: tagsFor(row), + }); + } + + await experiment.flush(); + + const passed = rows.filter((row) => row.passed === true).length; + const summary = await experiment.summarize({ summarizeScores: false }); + console.log( + ` ${name}: logged ${rows.length} result(s) (${passed} pass, ${rows.length - passed} fail) → ${summary.experimentUrl}`, + ); +} + +function printDryRun(groups: Map): void { + console.log( + `[dry run] would upload to Braintrust project "${PROJECT}"` + + (UPDATE ? " (update mode)" : "") + + ` from ${INPUT_PATH}`, + ); + for (const [name, rows] of groups) { + const passed = rows.filter((row) => row.passed === true).length; + console.log( + ` experiment "${name}": ${rows.length} event(s) (${passed} pass, ${rows.length - passed} fail)`, + ); + } +} + +async function main(): Promise { + const rows = await loadResults(INPUT_PATH); + if (rows.length === 0) { + throw new Error(`No usable results found in ${INPUT_PATH}`); + } + const groups = groupByExperiment(rows); + + if (DRY) { + printDryRun(groups); + return; + } + + if (!process.env.BRAINTRUST_API_KEY) { + throw new Error( + "BRAINTRUST_API_KEY is not set. Add it to your .env (or run with --dry).", + ); + } + + console.log( + `Uploading ${rows.length} result(s) across ${groups.size} experiment(s) to Braintrust project "${PROJECT}"...`, + ); + for (const [name, experimentRows] of groups) { + await uploadExperiment(name, experimentRows); + } + console.log("Done."); +} + +main().catch((error: unknown) => { + console.error(error); + process.exit(1); +}); diff --git a/package.json b/package.json index cf70715a..5a7ddee2 100644 --- a/package.json +++ b/package.json @@ -14,6 +14,7 @@ "eval:force": "pnpm --filter @supabase-evals/framework eval:force", "test:framework": "pnpm --filter @supabase-evals/framework test:framework", "export-results": "pnpm --filter @supabase-evals/framework export-results", + "upload:braintrust": "pnpm --filter @supabase-evals/framework upload:braintrust", "typecheck": "pnpm --filter @supabase-evals/framework typecheck && pnpm --filter @supabase-evals/web typecheck", "web": "pnpm --filter @supabase-evals/web dev", "web:build": "pnpm --filter @supabase-evals/web build", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 0f259178..4c266284 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -39,6 +39,9 @@ catalogs: ai: specifier: ^6.0.174 version: 6.0.199 + braintrust: + specifier: ^3.9.0 + version: 3.22.0 openai: specifier: ^6.44.0 version: 6.44.0 @@ -126,6 +129,9 @@ importers: ai: specifier: 'catalog:' version: 6.0.199(zod@4.4.3) + braintrust: + specifier: 'catalog:' + version: 3.22.0(zod@4.4.3) happy-dom: specifier: ^20.9.0 version: 20.10.2 @@ -549,6 +555,45 @@ packages: resolution: {integrity: sha512-4zBIxpPzowiZpusoFkyGVwakdRJUyuH5PxQ/PrqghfdFWWasvnCdPfQXHrenDai+gyLARulZjZowCOj6fjT4pA==} engines: {node: '>=6.9.0'} + '@braintrust/bt-darwin-arm64@0.12.0': + resolution: {integrity: sha512-mY6VW/3VwcOQOGN8sYHS6F0xzHTFwgZcNlj7zlQttI6OXOCGt/bhonGIqd03QBhxmj0M31ymSS7TqSeCK/RYIQ==} + cpu: [arm64] + os: [darwin] + + '@braintrust/bt-darwin-x64@0.12.0': + resolution: {integrity: sha512-woyRyDv2DfCF8+von+3X9f1cddAbFnKLSfCbEkrBQVc0ZsAM60as8nehYv7DkafJF/67yKFx/sUraV65sukesQ==} + cpu: [x64] + os: [darwin] + + '@braintrust/bt-linux-arm64@0.12.0': + resolution: {integrity: sha512-J9/7f3EIMKmFmSSQHQnAQLpUgB8YJENrOlkABjlTprBgsIYVgz7tSH7yg2P3ur+2Jem7PpGnrDxHGh/UjRfjsg==} + cpu: [arm64] + os: [linux] + + '@braintrust/bt-linux-x64-musl@0.12.0': + resolution: {integrity: sha512-KnLgENOoztBXcH+mLFJe4bYzi67kSOuELOQrm4Hler35GjgaBMI1fFLIUjKRPAmyT9VIhBMhy1ICfGEFLueMEQ==} + cpu: [x64] + os: [linux] + + '@braintrust/bt-linux-x64@0.12.0': + resolution: {integrity: sha512-HJFbUl3HYYY1Ivw8OYBeIMcIsU9r7+fC9BzLWB5FdH7881ToKee2wbbLZssMCxFbMVeUxzEo+SzGD/1Z9Gk9CA==} + cpu: [x64] + os: [linux] + + '@braintrust/bt-win32-arm64@0.12.0': + resolution: {integrity: sha512-+7PkdBAmiqEJsWteGHP/Zs62F75PkHPA8m/ej4TOz/mHGKulUU0bZibw91KRqIB7J7jGi5ItvCiqkiGaLKLnyw==} + cpu: [arm64] + os: [win32] + + '@braintrust/bt-win32-x64@0.12.0': + resolution: {integrity: sha512-Q0c+pVUGm82n/7N8QJ5s6j/G/Q0GFzqOaTipHjkmElLbAOOEcfRH/uljdtIPNBiEQcrzqirS0GmOgiFkeQ3Txg==} + cpu: [x64] + os: [win32] + + '@colors/colors@1.5.0': + resolution: {integrity: sha512-ooWCrlZP11i8GImSjTHYHLkvFDP48nS4+204nGb1RiX/WXYHmJA2III9/e2DWVabCESdW7hBAEzHRqUn9OUVvQ==} + engines: {node: '>=0.1.90'} + '@commander-js/extra-typings@14.0.0': resolution: {integrity: sha512-hIn0ncNaJRLkZrxBIp5AsW/eXEHNKYQBh0aPdoUqNgD+Io3NIykQqpKFyKcuasZhicGaEZJX/JBSIkZ4e5x8Dg==} peerDependencies: @@ -983,6 +1028,12 @@ packages: '@jridgewell/trace-mapping@0.3.31': resolution: {integrity: sha512-zzNR+SdQSDJzc8joaeP8QQoCQr8NuYx2dIIytl1QeBEZHJ9uW6hebsrYgbz8hJwUQao3TWCMtmfV8Nu1twOLAw==} + '@kwsites/file-exists@1.1.1': + resolution: {integrity: sha512-m9/5YGR18lIwxSFDwfE3oA7bWuq9kdau6ugN4H2rJeyhFQZcG9AgSHkQtSD15a8WvTgfz9aikZMrKPHvbpqFiw==} + + '@kwsites/promise-deferred@1.1.1': + resolution: {integrity: sha512-GaHYm+c0O9MjZRu0ongGBRbinu8gVAMd2UZjji6jVmqKtZluZnptXGWhz1E8j8D2HJ3f/yMxKAUC0b+57wncIw==} + '@launchql/protobufjs@7.2.6': resolution: {integrity: sha512-vwi1nG2/heVFsIMHQU1KxTjUp5c757CTtRAZn/jutApCkFlle1iv8tzM/DHlSZJKDldxaYqnNYTg0pTyp8Bbtg==} engines: {node: '>=12.0.0'} @@ -1006,6 +1057,9 @@ packages: '@cfworker/json-schema': optional: true + '@next/env@14.2.35': + resolution: {integrity: sha512-DuhvCtj4t9Gwrx80dmz2F4t/zKQ4ktN8WrMwOuVzkJfBilwAwGr6v16M5eI8yCuZ63H9TTuEU09Iu2HqkzFPVQ==} + '@noble/ciphers@1.3.0': resolution: {integrity: sha512-2I0gnIVPtfnMw9ee9h1dJG7tp81+8Ob3OJb3Mv37rx5L40/b0i7djjCVvGOVqc9AEIQyvyu1i6ypKdFw8R8gQw==} engines: {node: ^14.21.3 || >=16} @@ -1981,6 +2035,12 @@ packages: '@opentelemetry/sdk-trace-base': ^1.30.1 || ^2.1.0 '@opentelemetry/semantic-conventions': ^1.39.0 + '@simple-git/args-pathspec@1.0.3': + resolution: {integrity: sha512-ngJMaHlsWDTfjyq9F3VIQ8b7NXbBLq5j9i5bJ6XLYtD6qlDXT7fdKY2KscWWUF8t18xx052Y/PUO1K1TRc9yKA==} + + '@simple-git/argv-parser@1.1.1': + resolution: {integrity: sha512-Q9lBcfQ+VQCpQqGJFHe5yooOS5hGdLFFbJ5R+R5aDsnkPCahtn1hSkMcORX65J2Z5lxSkD0lQorMsncuBQxYUw==} + '@sindresorhus/merge-streams@4.0.0': resolution: {integrity: sha512-tlqY9xq5ukxTUZBmoOp+m61cqwQD5pHJtFY3Mn8CA8ps6yghLH/Hw8UPdqg4OLmFW3IFlcXnQNmo/dh8HzXYIQ==} engines: {node: '>=18'} @@ -2293,6 +2353,15 @@ packages: resolution: {integrity: sha512-VYNCgUc0nOmC4WJmWw9GkrKdfr8Zl4/rxhC5SvgacBgxiW9W/9NRttUoHHXV8xdII3MaRgkZZVX8Ikzc/Jmjag==} engines: {node: '>=14'} + '@vercel/functions@1.6.0': + resolution: {integrity: sha512-R6FKQrYT5MZs5IE1SqeCJWxMuBdHawFcCZboKKw8p7s+6/mcd55Gx6tWmyKnQTyrSEA04NH73Tc9CbqpEle8RA==} + engines: {node: '>= 16'} + peerDependencies: + '@aws-sdk/credential-provider-web-identity': '*' + peerDependenciesMeta: + '@aws-sdk/credential-provider-web-identity': + optional: true + '@vercel/oidc@3.2.0': resolution: {integrity: sha512-UycprH3T6n3jH0k44NHMa7pnFHGu/N05MjojYr+Mc6I7obkoLIJujSWwin1pCvdy/eOxrI/l3uDLQsmcrOb4ug==} engines: {node: '>= 20'} @@ -2420,6 +2489,10 @@ packages: resolution: {integrity: sha512-6t10qk83GOG8p0vKmaCr8eiilZwO171AvbROMtvvNiwrTly62t+7XkA8RdIIVbpMhCASAsxgAzdRSwh6nw/5Dg==} engines: {node: '>=4'} + astring@1.9.0: + resolution: {integrity: sha512-LElXdjswlqjWrPpJFg1Fx4wpkOCxj1TDHlSV4PlaRxHGWko024xICaa97ZkMfs6DRKlCguiAI+rbXv5GWwXIkg==} + hasBin: true + balanced-match@1.0.2: resolution: {integrity: sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==} @@ -2454,6 +2527,12 @@ packages: resolution: {integrity: sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA==} engines: {node: '>=8'} + braintrust@3.22.0: + resolution: {integrity: sha512-r3RmoLmZ/UsR3uReYWCV/B5CFA4kZjt+iXI41b8Hl6B7CDn8OLbeBxAX8ryMVGRJERNlGmbXLn0+n0IUqMFTqQ==} + hasBin: true + peerDependencies: + zod: ^3.25.34 || ^4.0 + browserslist@4.28.2: resolution: {integrity: sha512-48xSriZYYg+8qXna9kwqjIVzuQxi+KYWp2+5nCYnYKPTr0LvD89Jqk2Or5ogxz0NUMfIjhh2lIUX/LyX9B4oIg==} engines: {node: ^6 || ^7 || ^8 || ^9 || ^10 || ^11 || ^12 || >=13.7} @@ -2519,10 +2598,18 @@ packages: resolution: {integrity: sha512-aCj4O5wKyszjMmDT4tZj93kxyydN/K5zPWSCe6/0AV/AA1pqe5ZBIw0a2ZfPQV7lL5/yb5HsUreJ6UFAF1tEQw==} engines: {node: '>=18'} + cli-progress@3.12.0: + resolution: {integrity: sha512-tRkV3HJ1ASwm19THiiLIXLO7Im7wlTuKnvkYaTkyoAPefqjNg7W7DHKUlGRxy9vxDvbyCYQkQozvptuMkGCg8A==} + engines: {node: '>=4'} + cli-spinners@2.9.2: resolution: {integrity: sha512-ywqV+5MmyL4E7ybXgKys4DugZbX0FC6LnwrhjuykIjnK9k8OQacQ7axGKnjDXWNhns0xot3bZI5h55H8yo9cJg==} engines: {node: '>=6'} + cli-table3@0.6.5: + resolution: {integrity: sha512-+W/5efTR7y5HRD7gACw9yQjqMVvEMLBHmboM/kPWam+H+Hmyrgjh6YncVKK122YZkXrLudzTuAukUw9FnMf7IQ==} + engines: {node: 10.* || >= 12.*} + clsx@2.1.1: resolution: {integrity: sha512-eYm0QWBtUrBWZWG0d386OGAw16Z995PiOVo2B7bjWSbHedGl5e0ZWaq65kOGgUSNesEIDkB9ISbTg/JK9dhCZA==} engines: {node: '>=6'} @@ -2610,6 +2697,9 @@ packages: resolution: {integrity: sha512-0R9ikRb668HB7QDxT1vkpuUBtqc53YyAwMwGeUFKRojY/NWKvdZ+9UYtRfGmhqNbRkTSVpMbmyhXipFFv2cb/A==} engines: {node: '>= 12'} + dc-browser@1.0.4: + resolution: {integrity: sha512-7oEtnzNlcE+hr4OvO3GR6Gndgw8BhW+wKOEwMqSleyY7N29jbAxzyW5BaJl7qBCw+6OIxfMWtY0T+6dxq8RWLw==} + debug@4.4.3: resolution: {integrity: sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==} engines: {node: '>=6.0'} @@ -2671,6 +2761,10 @@ packages: dom-accessibility-api@0.6.3: resolution: {integrity: sha512-7ZgogeTnjuHbo+ct10G9Ffp0mif17idi0IyWNVA/wcwcm7NPOD/WEHVP3n7n3MhXqxoIYm8d6MuZohYWIZ4T3w==} + dotenv@16.6.1: + resolution: {integrity: sha512-uBq4egWHTcTt33a72vpSG0z3HnPuIl6NqYcTrKEg2azoEyl2hpW0zqlxysq2pK9HlDIHyHyakeYaYnSAwd8bow==} + engines: {node: '>=12'} + dotenv@17.4.2: resolution: {integrity: sha512-nI4U3TottKAcAD9LLud4Cb7b2QztQMUEfHbvhTH09bqXTxnSie8WnjPALV/WMCrJZ6UV/qHJ6L03OqO3LcdYZw==} engines: {node: '>=12'} @@ -2692,6 +2786,9 @@ packages: emoji-regex@10.6.0: resolution: {integrity: sha512-toUI84YS5YmxW219erniWD0CIVOo46xGKColeNQRgOzDorgBi1v4D71/OFzgD9GO2UGKIv1C3Sp8DAn0+j5w7A==} + emoji-regex@8.0.0: + resolution: {integrity: sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==} + encodeurl@2.0.0: resolution: {integrity: sha512-Q0n9HRi4m6JuGIV1eFlmvJB7ZEVxu93IrMyiMsGC0lrMJMWzRgx6WGquyfQgZVb31vhGgXnfmPNNXmxnOkRBrg==} engines: {node: '>= 0.8'} @@ -2820,6 +2917,10 @@ packages: resolution: {integrity: sha512-aIL5Fx7mawVa300al2BnEE4iNvo1qETxLrPI/o05L7z6go7fCw1J6EQmbK4FmJ2AS7kgVF/KEZWufBfdClMcPg==} engines: {node: '>= 0.6'} + eventsource-parser@1.1.2: + resolution: {integrity: sha512-v0eOBUbiaFojBu2s2NPBfYUoRR9GjcDNvCXVaqEf5vVfpIAh9f8RCo4vXTP8c63QRKCFwoLpMpTdPwwhEKVgzA==} + engines: {node: '>=14.18'} + eventsource-parser@3.1.0: resolution: {integrity: sha512-kJezFj9YFAMLeORyi7aCLxLbD5/qWMQnoMVlVPyHIll7lgRJCc3JVln9Vgl9nwQi0YkMnhdGTMNn7CkRRAptMg==} engines: {node: '>=18.0.0'} @@ -3160,6 +3261,10 @@ packages: resolution: {integrity: sha512-SbKbANkN603Vi4jEZv49LeVJMn4yGwsbzZworEoyEiutsN3nJYdbO36zfhGJ6QEDpOZIFkDtnq5JRxmvl3jsoQ==} engines: {node: '>=0.10.0'} + is-fullwidth-code-point@3.0.0: + resolution: {integrity: sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==} + engines: {node: '>=8'} + is-glob@4.0.3: resolution: {integrity: sha512-xelSayHH36ZgE7ZWhli7pW34hNbNl8Ojv5KVmkJD4hBdD3th8Tfk9vYasLM+mXWOZhFkgZfxhLSnrwRr4elSSg==} engines: {node: '>=0.10.0'} @@ -3449,6 +3554,10 @@ packages: resolution: {integrity: sha512-8q7VEgMJW4J8tcfVPy8g09NcQwZdbwFEqhe/WZkoIzjn/3TGDwtOCYtXGxA3O8tPzpczCCDgv+P2P5y00ZJOOg==} engines: {node: '>= 8'} + meriyah@6.1.4: + resolution: {integrity: sha512-Sz8FzjzI0kN13GK/6MVEsVzMZEPvOhnmmI1lU5+/1cGOiK3QUahntrNNtdVeihrO7t9JpoH75iMNXg6R6uWflQ==} + engines: {node: '>=18.0.0'} + micromatch@4.0.8: resolution: {integrity: sha512-PXwfBhYu0hBCPw8Dn0E+WDYb7af3dSLVWKi3HGv84IdF4TyFoC0ysxFd0Goxw7nSv4T/PzEJQxsYsEiFCKo2BA==} engines: {node: '>=8.6'} @@ -3496,6 +3605,10 @@ packages: ms@2.1.3: resolution: {integrity: sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==} + mustache@4.2.0: + resolution: {integrity: sha512-71ippSywq5Yb7/tVYyGbkBggbU8H3u5Rz56fH60jGFgr8uHwxs+aSKeqmluIVzM0m0kB7xQjKS6qPfd0b2ZoqQ==} + hasBin: true + nanoid@3.3.12: resolution: {integrity: sha512-ZB9RH/39qpq5Vu6Y+NmUaFhQR6pp+M2Xt76XBnEwDaGcVAqhlvxrl3B2bKS5D3NH3QR76v3aSrKaF/Kiy7lEtQ==} engines: {node: ^10 || ^12 || ^13.7 || ^14 || >=15.0.1} @@ -3959,6 +4072,9 @@ packages: resolution: {integrity: sha512-vfD3pmTzGpufjScBh50YHKzEu2lxBWhVEHsNGoEXmCmn2hKGfeNLYMzCJpe8cD7gqX7TJluOVpBkAequ6dgMmA==} engines: {node: '>=4'} + semifies@1.0.0: + resolution: {integrity: sha512-xXR3KGeoxTNWPD4aBvL5NUpMTT7WMANr3EWnaS190QVkY52lqqcVRD7Q05UVbBhiWDGWMlJEUam9m7uFFGVScw==} + semver@6.3.1: resolution: {integrity: sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA==} hasBin: true @@ -4017,6 +4133,9 @@ packages: resolution: {integrity: sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw==} engines: {node: '>=14'} + simple-git@3.36.0: + resolution: {integrity: sha512-cGQjLjK8bxJw4QuYT7gxHw3/IouVESbhahSsHrX97MzCL1gu2u7oy38W6L2ZIGECEfIBG4BabsWDPjBxJENv9Q==} + sisteransi@1.0.5: resolution: {integrity: sha512-bLGGlR1QxBcynn2d5YmDX4MGjlZvy2MRBDRNHLJ8VI6l6+9FUiyTFNJ0IveOSP0bcXgVDPRcfGqA0pjaqUpfVg==} @@ -4028,6 +4147,10 @@ packages: resolution: {integrity: sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==} engines: {node: '>=0.10.0'} + source-map@0.7.6: + resolution: {integrity: sha512-i5uvt8C3ikiWeNZSVZNWcfZPItFQOsYTUAOkcUPGd8DqDy1uOUikjt5dG+uRlwyvR108Fb9DOd4GvXfT0N2/uQ==} + engines: {node: '>= 12'} + split2@4.2.0: resolution: {integrity: sha512-UcjcJOWknrNkF6PLX83qcHM6KHgVKNkV62Y8a5uYDVv9ydGQVwAHMKqHdJje1VTWpljG0WYpCDhrCdAOYH4TWg==} engines: {node: '>= 10.x'} @@ -4055,6 +4178,10 @@ packages: strfy-js@3.2.2: resolution: {integrity: sha512-hUgJ5k2PR1ivhq4uObxnin5j6GcOr0Y0N1lzi3z6SRhxNqu4rzpDfyoC2ToUAyM8yXNXM0zs6f4KIiqj8NqheQ==} + string-width@4.2.3: + resolution: {integrity: sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==} + engines: {node: '>=8'} + string-width@7.2.0: resolution: {integrity: sha512-tsaTIkKW9b4N+AEj+SVA+WhJzV7/zMhcSu78mLKWSk7cXMOSHsBKFWUs0fWwq8QyK3MgJBQRX6Gbi4kYbdvGkQ==} engines: {node: '>=18'} @@ -4113,6 +4240,10 @@ packages: resolution: {integrity: sha512-uxc/zpqFg6x7C8vOE7lh6Lbda8eEL9zmVm/PLeTPBRhh1xCgdWaQ+J1CUieGpIfm2HdtsUpRv+HshiasBMcc6A==} engines: {node: '>=6'} + termi-link@1.1.0: + resolution: {integrity: sha512-2qSN6TnomHgVLtk+htSWbaYs4Rd2MH/RU7VpHTy6MBstyNyWbM4yKd1DCYpE3fDg8dmGWojXCngNi/MHCzGuAA==} + engines: {node: '>=12'} + tiny-invariant@1.3.3: resolution: {integrity: sha512-+FbBPE1o9QAYvviau/qC5SE3caw21q3xkvWKBtja5vgqOWIHHJ3ioaq1VPfn/Szqctz2bU/oYeKd9/z5BL+PVg==} @@ -4208,6 +4339,10 @@ packages: resolution: {integrity: sha512-pjy2bYhSsufwWlKwPc+l3cN7+wuJlK6uz0YdJEOlQDbl6jo/YlPi4mb8agUkVC8BF7V8NuzeyPNqRksA3hztKQ==} engines: {node: '>= 0.8'} + unplugin@2.3.11: + resolution: {integrity: sha512-5uKD0nqiYVzlmCRs01Fhs2BdkEgBS3SAVP6ndrBsuK42iC2+JHyxM05Rm9G8+5mkmRtzMZGY8Ct5+mliZxU/Ww==} + engines: {node: '>=18.12.0'} + update-browserslist-db@1.2.3: resolution: {integrity: sha512-Js0m9cx+qOgDxo0eMiFGEueWztz+d4+M3rGlmKPT+T4IS/jP4ylw3Nwpu6cpTTP8R1MAC1kF4VbdLt3ARf209w==} hasBin: true @@ -4243,6 +4378,10 @@ packages: util-deprecate@1.0.2: resolution: {integrity: sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==} + uuid@11.1.1: + resolution: {integrity: sha512-vIYxrBCC/N/K+Js3qSN88go7kIfNPssr/hHCesKCQNAjmgvYS2oqr69kIufEG+O4+PfezOH4EbIeHCfFov8ZgQ==} + hasBin: true + uuid@13.0.2: resolution: {integrity: sha512-vzi9uRZ926x4XV73S/4qQaTwPXM2JBj6/6lI/byHH1jOpCzb0zDbfytgA9LcN/hzb2l7WQSQnxITOVx5un/wGw==} hasBin: true @@ -4340,6 +4479,9 @@ packages: resolution: {integrity: sha512-d2JWLCivmZYTSIoge9MsgFCZrt571BikcWGYkjC1khllbTeDlGqZ2D8vD8E/lJa8WGWbb7Plm8/XJYV7IJHZZw==} engines: {node: '>= 8'} + webpack-virtual-modules@0.6.2: + resolution: {integrity: sha512-66/V2i5hQanC51vBQKPH4aI8NMAcBW59FVBs+rC7eGHupMyfn34q7rZIE+ETlJ+XTevqfUhVVBgSUNSW2flEUQ==} + whatwg-mimetype@3.0.0: resolution: {integrity: sha512-nt+N2dzIutVRxARx1nghPKGv1xHikU7HKdfafKkLNLindmPU/ch3U31NOCGGA/dmPcmb1VlofO0vnKAcsm0o/Q==} engines: {node: '>=12'} @@ -4688,6 +4830,30 @@ snapshots: '@babel/helper-string-parser': 7.29.7 '@babel/helper-validator-identifier': 7.29.7 + '@braintrust/bt-darwin-arm64@0.12.0': + optional: true + + '@braintrust/bt-darwin-x64@0.12.0': + optional: true + + '@braintrust/bt-linux-arm64@0.12.0': + optional: true + + '@braintrust/bt-linux-x64-musl@0.12.0': + optional: true + + '@braintrust/bt-linux-x64@0.12.0': + optional: true + + '@braintrust/bt-win32-arm64@0.12.0': + optional: true + + '@braintrust/bt-win32-x64@0.12.0': + optional: true + + '@colors/colors@1.5.0': + optional: true + '@commander-js/extra-typings@14.0.0(commander@14.0.3)': dependencies: commander: 14.0.3 @@ -4976,6 +5142,14 @@ snapshots: '@jridgewell/resolve-uri': 3.1.2 '@jridgewell/sourcemap-codec': 1.5.5 + '@kwsites/file-exists@1.1.1': + dependencies: + debug: 4.4.3(supports-color@10.2.2) + transitivePeerDependencies: + - supports-color + + '@kwsites/promise-deferred@1.1.1': {} + '@launchql/protobufjs@7.2.6': dependencies: '@protobufjs/aspromise': 1.1.2 @@ -5046,6 +5220,8 @@ snapshots: transitivePeerDependencies: - supports-color + '@next/env@14.2.35': {} + '@noble/ciphers@1.3.0': {} '@noble/curves@1.9.7': @@ -6023,6 +6199,12 @@ snapshots: '@opentelemetry/semantic-conventions': 1.41.1 '@sentry/core': 10.57.0 + '@simple-git/args-pathspec@1.0.3': {} + + '@simple-git/argv-parser@1.1.1': + dependencies: + '@simple-git/args-pathspec': 1.0.3 + '@sindresorhus/merge-streams@4.0.0': {} '@stablelib/base64@1.0.1': {} @@ -6388,6 +6570,8 @@ snapshots: '@vercel/detect-agent@1.2.3': {} + '@vercel/functions@1.6.0': {} + '@vercel/oidc@3.2.0': {} '@vitejs/plugin-react@5.2.0(vite@7.3.5(@types/node@22.19.20)(jiti@2.7.0)(lightningcss@1.32.0)(tsx@4.22.4))': @@ -6528,6 +6712,8 @@ snapshots: dependencies: tslib: 2.8.1 + astring@1.9.0: {} + balanced-match@1.0.2: {} balanced-match@4.0.4: {} @@ -6567,6 +6753,51 @@ snapshots: dependencies: fill-range: 7.1.1 + braintrust@3.22.0(zod@4.4.3): + dependencies: + '@next/env': 14.2.35 + '@vercel/functions': 1.6.0 + acorn: 8.16.0 + acorn-import-attributes: 1.9.5(acorn@8.16.0) + ajv: 8.20.0 + argparse: 2.0.1 + astring: 1.9.0 + cjs-module-lexer: 2.2.0 + cli-progress: 3.12.0 + cli-table3: 0.6.5 + cors: 2.8.6 + dc-browser: 1.0.4 + dotenv: 16.6.1 + esbuild: 0.28.0 + esquery: 1.7.0 + eventsource-parser: 1.1.2 + express: 5.2.1 + http-errors: 2.0.1 + meriyah: 6.1.4 + minimatch: 10.2.5 + module-details-from-path: 1.0.4 + mustache: 4.2.0 + pluralize: 8.0.0 + semifies: 1.0.0 + simple-git: 3.36.0 + source-map: 0.7.6 + termi-link: 1.1.0 + unplugin: 2.3.11 + uuid: 11.1.1 + zod: 4.4.3 + zod-to-json-schema: 3.25.2(zod@4.4.3) + optionalDependencies: + '@braintrust/bt-darwin-arm64': 0.12.0 + '@braintrust/bt-darwin-x64': 0.12.0 + '@braintrust/bt-linux-arm64': 0.12.0 + '@braintrust/bt-linux-x64': 0.12.0 + '@braintrust/bt-linux-x64-musl': 0.12.0 + '@braintrust/bt-win32-arm64': 0.12.0 + '@braintrust/bt-win32-x64': 0.12.0 + transitivePeerDependencies: + - '@aws-sdk/credential-provider-web-identity' + - supports-color + browserslist@4.28.2: dependencies: baseline-browser-mapping: 2.10.35 @@ -6626,8 +6857,18 @@ snapshots: dependencies: restore-cursor: 5.1.0 + cli-progress@3.12.0: + dependencies: + string-width: 4.2.3 + cli-spinners@2.9.2: {} + cli-table3@0.6.5: + dependencies: + string-width: 4.2.3 + optionalDependencies: + '@colors/colors': 1.5.0 + clsx@2.1.1: {} code-block-writer@13.0.3: {} @@ -6688,6 +6929,8 @@ snapshots: data-uri-to-buffer@4.0.1: {} + dc-browser@1.0.4: {} + debug@4.4.3(supports-color@10.2.2): dependencies: ms: 2.1.3 @@ -6723,6 +6966,8 @@ snapshots: dom-accessibility-api@0.6.3: {} + dotenv@16.6.1: {} + dotenv@17.4.2: {} dunder-proto@1.0.1: @@ -6744,6 +6989,8 @@ snapshots: emoji-regex@10.6.0: {} + emoji-regex@8.0.0: {} + encodeurl@2.0.0: {} enhanced-resolve@5.23.0: @@ -6931,6 +7178,8 @@ snapshots: etag@1.8.1: {} + eventsource-parser@1.1.2: {} + eventsource-parser@3.1.0: {} eventsource@3.0.7: @@ -7283,6 +7532,8 @@ snapshots: is-extglob@2.1.1: {} + is-fullwidth-code-point@3.0.0: {} + is-glob@4.0.3: dependencies: is-extglob: 2.1.1 @@ -7488,6 +7739,8 @@ snapshots: merge2@1.4.1: {} + meriyah@6.1.4: {} + micromatch@4.0.8: dependencies: braces: 3.0.3 @@ -7525,6 +7778,8 @@ snapshots: ms@2.1.3: {} + mustache@4.2.0: {} + nanoid@3.3.12: {} natural-compare@1.4.0: {} @@ -8011,6 +8266,8 @@ snapshots: extend-shallow: 2.0.1 kind-of: 6.0.3 + semifies@1.0.0: {} + semver@6.3.1: {} semver@7.8.3: {} @@ -8123,12 +8380,24 @@ snapshots: signal-exit@4.1.0: {} + simple-git@3.36.0: + dependencies: + '@kwsites/file-exists': 1.1.1 + '@kwsites/promise-deferred': 1.1.1 + '@simple-git/args-pathspec': 1.0.3 + '@simple-git/argv-parser': 1.1.1 + debug: 4.4.3(supports-color@10.2.2) + transitivePeerDependencies: + - supports-color + sisteransi@1.0.5: {} source-map-js@1.2.1: {} source-map@0.6.1: {} + source-map@0.7.6: {} + split2@4.2.0: {} sprintf-js@1.0.3: {} @@ -8150,6 +8419,12 @@ snapshots: dependencies: minimatch: 10.2.5 + string-width@4.2.3: + dependencies: + emoji-regex: 8.0.0 + is-fullwidth-code-point: 3.0.0 + strip-ansi: 6.0.1 + string-width@7.2.0: dependencies: emoji-regex: 10.6.0 @@ -8196,6 +8471,8 @@ snapshots: tapable@2.3.3: {} + termi-link@1.1.0: {} + tiny-invariant@1.3.3: {} tinybench@2.9.0: {} @@ -8277,6 +8554,13 @@ snapshots: unpipe@1.0.0: {} + unplugin@2.3.11: + dependencies: + '@jridgewell/remapping': 2.3.5 + acorn: 8.16.0 + picomatch: 4.0.4 + webpack-virtual-modules: 0.6.2 + update-browserslist-db@1.2.3(browserslist@4.28.2): dependencies: browserslist: 4.28.2 @@ -8306,6 +8590,8 @@ snapshots: util-deprecate@1.0.2: {} + uuid@11.1.1: {} + uuid@13.0.2: {} validate-npm-package-name@7.0.2: {} @@ -8402,6 +8688,8 @@ snapshots: web-streams-polyfill@3.3.3: {} + webpack-virtual-modules@0.6.2: {} + whatwg-mimetype@3.0.0: {} which@2.0.2: diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index 3083cd81..fb316f23 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -8,6 +8,7 @@ catalog: '@ai-sdk/anthropic': ^3.0.71 '@ai-sdk/mcp': ^1.0.39 '@ai-sdk/openai': ^3.0.66 + braintrust: ^3.9.0 '@electric-sql/pglite': 0.4.5 '@electric-sql/pglite-socket': 0.1.5 '@supabase/lite': 0.5.1-next.1 From 802a20150d3814a4dc7b88951e165e2eb452e735 Mon Sep 17 00:00:00 2001 From: Pedro Rodrigues Date: Tue, 14 Jul 2026 18:10:04 +0100 Subject: [PATCH 2/3] test: unit-test the Braintrust result mapping (AI-921) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Extract the pure snapshot→Braintrust mapping (scoresFor, tagsFor, metadataFor, experimentMetadata, groupByExperiment, isResultRow) into lib/braintrust-mapping.ts and cover it with vitest. Runs credential-free as part of `pnpm check`. Co-Authored-By: Claude Opus 4.8 (1M context) --- apps/framework/lib/braintrust-mapping.test.ts | 163 ++++++++++++++++++ apps/framework/lib/braintrust-mapping.ts | 103 +++++++++++ apps/framework/package.json | 3 +- .../framework/scripts/upload-to-braintrust.ts | 100 +---------- 4 files changed, 277 insertions(+), 92 deletions(-) create mode 100644 apps/framework/lib/braintrust-mapping.test.ts create mode 100644 apps/framework/lib/braintrust-mapping.ts diff --git a/apps/framework/lib/braintrust-mapping.test.ts b/apps/framework/lib/braintrust-mapping.test.ts new file mode 100644 index 00000000..9fbd1bd9 --- /dev/null +++ b/apps/framework/lib/braintrust-mapping.test.ts @@ -0,0 +1,163 @@ +import { describe, expect, it } from "vitest"; +import { + experimentMetadata, + groupByExperiment, + isResultRow, + metadataFor, + scoresFor, + tagsFor, + type ResultRow, +} from "./braintrust-mapping.js"; + +function makeRow(overrides: Partial = {}): ResultRow { + return { + experiment: "claude-code-opus-4.8", + experimentSuite: "benchmark", + experimentDisplay: { + agent: "claude-code", + modelProvider: "anthropic", + modelId: "claude-opus-4-8", + reasoningEffort: "high", + }, + eval: "build-cli-001-bootstrap-app", + stage: "build", + product: ["database", "data-api"], + topic: ["migrations", "rls"], + suite: "benchmark", + interface: "cli", + passed: true, + checks: [ + { name: "a", passed: true }, + { name: "b", passed: false }, + ], + attempts: 1, + skills: { available: ["supabase"], loaded: ["supabase"] }, + sourcePath: "claude-code-opus-4.8/build-cli-001-bootstrap-app.json", + ...overrides, + } as ResultRow; +} + +describe("isResultRow", () => { + it("accepts rows with string experiment and eval", () => { + expect(isResultRow(makeRow())).toBe(true); + }); + + it("rejects non-objects and rows missing required keys", () => { + expect(isResultRow(null)).toBe(false); + expect(isResultRow("x")).toBe(false); + expect(isResultRow({ experiment: "x" })).toBe(false); + expect(isResultRow({ eval: "x" })).toBe(false); + expect(isResultRow({ experiment: 1, eval: 2 })).toBe(false); + }); +}); + +describe("groupByExperiment", () => { + it("groups rows by experiment, preserving order", () => { + const rows = [ + makeRow({ experiment: "a", eval: "e1" }), + makeRow({ experiment: "b", eval: "e2" }), + makeRow({ experiment: "a", eval: "e3" }), + ]; + const groups = groupByExperiment(rows); + expect([...groups.keys()]).toEqual(["a", "b"]); + expect(groups.get("a")?.map((r) => r.eval)).toEqual(["e1", "e3"]); + expect(groups.get("b")?.map((r) => r.eval)).toEqual(["e2"]); + }); + + it("returns an empty map for no rows", () => { + expect(groupByExperiment([]).size).toBe(0); + }); +}); + +describe("scoresFor", () => { + it("maps passed to 1/0 and computes the check pass rate", () => { + expect(scoresFor(makeRow({ passed: true }))).toEqual({ + passed: 1, + checkPassRate: 0.5, + }); + expect(scoresFor(makeRow({ passed: false }))).toEqual({ + passed: 0, + checkPassRate: 0.5, + }); + }); + + it("falls back to passed when there are no checks", () => { + expect(scoresFor(makeRow({ passed: true, checks: [] }))).toEqual({ + passed: 1, + checkPassRate: 1, + }); + expect(scoresFor(makeRow({ passed: false, checks: undefined }))).toEqual({ + passed: 0, + checkPassRate: 0, + }); + }); + + it("reports a perfect rate when every check passes", () => { + const row = makeRow({ + checks: [ + { name: "a", passed: true }, + { name: "b", passed: true }, + ], + }); + expect(scoresFor(row).checkPassRate).toBe(1); + }); +}); + +describe("tagsFor", () => { + it("combines interface, suite, product, and topic, deduped", () => { + expect(tagsFor(makeRow())).toEqual([ + "cli", + "benchmark", + "database", + "data-api", + "migrations", + "rls", + ]); + }); + + it("drops undefined/empty values", () => { + const row = makeRow({ + interface: undefined, + experimentSuite: undefined, + product: undefined, + topic: ["rls"], + }); + expect(tagsFor(row)).toEqual(["rls"]); + }); + + it("de-duplicates overlapping product/topic values", () => { + const row = makeRow({ product: ["database"], topic: ["database"] } as Partial); + expect(tagsFor(row)).toEqual(["cli", "benchmark", "database"]); + }); +}); + +describe("metadataFor", () => { + it("flattens experimentDisplay into first-class grouping keys", () => { + const meta = metadataFor(makeRow()); + expect(meta.agent).toBe("claude-code"); + expect(meta.modelProvider).toBe("anthropic"); + expect(meta.modelId).toBe("claude-opus-4-8"); + expect(meta.reasoningEffort).toBe("high"); + expect(meta.skillsLoaded).toEqual(["supabase"]); + expect(meta.eval).toBe("build-cli-001-bootstrap-app"); + }); + + it("tolerates a missing experimentDisplay", () => { + const meta = metadataFor(makeRow({ experimentDisplay: undefined })); + expect(meta.agent).toBeUndefined(); + expect(meta.modelId).toBeUndefined(); + }); +}); + +describe("experimentMetadata", () => { + it("carries the per-configuration dimensions and a source marker", () => { + expect(experimentMetadata(makeRow())).toEqual({ + experimentSuite: "benchmark", + agent: "claude-code", + modelProvider: "anthropic", + modelId: "claude-opus-4-8", + reasoningEffort: "high", + source: "eval-results.json", + }); + }); +}); diff --git a/apps/framework/lib/braintrust-mapping.ts b/apps/framework/lib/braintrust-mapping.ts new file mode 100644 index 00000000..186f0cc8 --- /dev/null +++ b/apps/framework/lib/braintrust-mapping.ts @@ -0,0 +1,103 @@ +/** + * Pure mapping from the exported `eval-results.json` shape onto the fields + * Braintrust's `experiment.log()` expects. Kept free of any I/O or SDK calls so + * it can be unit-tested without credentials or network (see the sibling test). + */ +import type { EvalResult } from "@supabase-evals/core/eval-metadata"; + +// A result row as it appears in the web-facing snapshot: the core EvalResult +// plus the display/source fields export-results.ts adds. +export type ResultRow = EvalResult & { + experimentSuite?: string; + prompt?: string; + promptSourcePath?: string; + sourcePath?: string; +}; + +export function isResultRow(value: unknown): value is ResultRow { + return ( + typeof value === "object" && + value !== null && + typeof (value as ResultRow).experiment === "string" && + typeof (value as ResultRow).eval === "string" + ); +} + +export function groupByExperiment(rows: ResultRow[]): Map { + const groups = new Map(); + for (const row of rows) { + const existing = groups.get(row.experiment); + if (existing) { + existing.push(row); + } else { + groups.set(row.experiment, [row]); + } + } + return groups; +} + +/** + * Numeric, cross-eval-comparable scores. `passed` mirrors the run's pass@k + * outcome; `checkPassRate` is the fraction of individual checks that passed, + * falling back to `passed` when a run has no per-check breakdown. + */ +export function scoresFor(row: ResultRow): Record { + const passed = row.passed === true ? 1 : 0; + const checks = row.checks ?? []; + const checkPassRate = checks.length + ? checks.filter((check) => check.passed).length / checks.length + : passed; + return { passed, checkPassRate }; +} + +export function metadataFor(row: ResultRow): Record { + return { + eval: row.eval, + experiment: row.experiment, + experimentSuite: row.experimentSuite, + stage: row.stage, + product: row.product, + topic: row.topic, + suite: row.suite, + interface: row.interface, + cliVersion: row.cliVersion, + attempts: row.attempts, + // Flatten experimentDisplay so each dimension is a first-class grouping axis. + agent: row.experimentDisplay?.agent, + modelProvider: row.experimentDisplay?.modelProvider, + modelId: row.experimentDisplay?.modelId, + reasoningEffort: row.experimentDisplay?.reasoningEffort, + skillsAvailable: row.skills?.available, + skillsLoaded: row.skills?.loaded, + promptSourcePath: row.promptSourcePath, + sourcePath: row.sourcePath, + }; +} + +export function tagsFor(row: ResultRow): string[] { + const candidates: Array = [ + row.interface, + row.experimentSuite, + ...(row.product ?? []), + ...(row.topic ?? []), + ]; + const tags = candidates.filter( + (tag): tag is string => typeof tag === "string" && tag.length > 0, + ); + return [...new Set(tags)]; +} + +/** + * Experiment-level metadata, derived from the first row of the group. These are + * constant per Configuration, so they belong on the experiment, not each event. + */ +export function experimentMetadata(row: ResultRow): Record { + return { + experimentSuite: row.experimentSuite, + agent: row.experimentDisplay?.agent, + modelProvider: row.experimentDisplay?.modelProvider, + modelId: row.experimentDisplay?.modelId, + reasoningEffort: row.experimentDisplay?.reasoningEffort, + source: "eval-results.json", + }; +} diff --git a/apps/framework/package.json b/apps/framework/package.json index 826cb934..bea4d581 100644 --- a/apps/framework/package.json +++ b/apps/framework/package.json @@ -4,7 +4,8 @@ "version": "0.0.1", "type": "module", "scripts": { - "check": "pnpm typecheck && pnpm test:framework", + "check": "pnpm typecheck && pnpm test:unit && pnpm test:framework", + "test:unit": "vitest run", "eval": "node --env-file=../../.env --import tsx/esm harness/run-eval.ts", "eval:dry": "node --env-file=../../.env --import tsx/esm harness/run-eval.ts --dry", "eval:smoke": "node --env-file=../../.env --import tsx/esm harness/run-eval.ts --smoke", diff --git a/apps/framework/scripts/upload-to-braintrust.ts b/apps/framework/scripts/upload-to-braintrust.ts index b25cbd68..a8c8197e 100644 --- a/apps/framework/scripts/upload-to-braintrust.ts +++ b/apps/framework/scripts/upload-to-braintrust.ts @@ -31,7 +31,15 @@ import { readFile } from "node:fs/promises"; import { dirname, resolve } from "node:path"; import { fileURLToPath } from "node:url"; import { init } from "braintrust"; -import type { EvalResult } from "@supabase-evals/core/eval-metadata"; +import { + experimentMetadata, + groupByExperiment, + isResultRow, + metadataFor, + scoresFor, + tagsFor, + type ResultRow, +} from "../lib/braintrust-mapping.js"; const __dirname = dirname(fileURLToPath(import.meta.url)); const ROOT = resolve(__dirname, "..", "..", ".."); @@ -59,24 +67,6 @@ const UPDATE = rawArgs.includes("--update"); const INPUT_PATH = resolve(ROOT, readFlag(rawArgs, "input") ?? DEFAULT_INPUT); const PROJECT = readFlag(rawArgs, "project") ?? DEFAULT_PROJECT; -// A result row as it appears in the web-facing snapshot: the core EvalResult -// plus the display/source fields export-results.ts adds. -type ResultRow = EvalResult & { - experimentSuite?: string; - prompt?: string; - promptSourcePath?: string; - sourcePath?: string; -}; - -function isResultRow(value: unknown): value is ResultRow { - return ( - typeof value === "object" && - value !== null && - typeof (value as ResultRow).experiment === "string" && - typeof (value as ResultRow).eval === "string" - ); -} - async function loadResults(path: string): Promise { const parsed: unknown = JSON.parse(await readFile(path, "utf8")); if (!Array.isArray(parsed)) { @@ -92,78 +82,6 @@ async function loadResults(path: string): Promise { return rows; } -function groupByExperiment(rows: ResultRow[]): Map { - const groups = new Map(); - for (const row of rows) { - const existing = groups.get(row.experiment); - if (existing) { - existing.push(row); - } else { - groups.set(row.experiment, [row]); - } - } - return groups; -} - -function scoresFor(row: ResultRow): Record { - const passed = row.passed === true ? 1 : 0; - const checks = row.checks ?? []; - const checkPassRate = checks.length - ? checks.filter((check) => check.passed).length / checks.length - : passed; - return { passed, checkPassRate }; -} - -function metadataFor(row: ResultRow): Record { - return { - eval: row.eval, - experiment: row.experiment, - experimentSuite: row.experimentSuite, - stage: row.stage, - product: row.product, - topic: row.topic, - suite: row.suite, - interface: row.interface, - cliVersion: row.cliVersion, - attempts: row.attempts, - // Flatten experimentDisplay so each dimension is a first-class grouping axis. - agent: row.experimentDisplay?.agent, - modelProvider: row.experimentDisplay?.modelProvider, - modelId: row.experimentDisplay?.modelId, - reasoningEffort: row.experimentDisplay?.reasoningEffort, - skillsAvailable: row.skills?.available, - skillsLoaded: row.skills?.loaded, - promptSourcePath: row.promptSourcePath, - sourcePath: row.sourcePath, - }; -} - -function tagsFor(row: ResultRow): string[] { - const candidates: Array = [ - row.interface, - row.experimentSuite, - ...(row.product ?? []), - ...(row.topic ?? []), - ]; - const tags = candidates.filter( - (tag): tag is string => typeof tag === "string" && tag.length > 0, - ); - return [...new Set(tags)]; -} - -// Experiment-level metadata, derived from the first row of the group. These are -// constant per Configuration, so they belong on the experiment, not each event. -function experimentMetadata(row: ResultRow): Record { - return { - experimentSuite: row.experimentSuite, - agent: row.experimentDisplay?.agent, - modelProvider: row.experimentDisplay?.modelProvider, - modelId: row.experimentDisplay?.modelId, - reasoningEffort: row.experimentDisplay?.reasoningEffort, - source: "eval-results.json", - }; -} - async function uploadExperiment(name: string, rows: ResultRow[]): Promise { const experiment = init(PROJECT, { experiment: name, From 3c5ae004e40bf84eb32a0dc87f21b29cbc46794d Mon Sep 17 00:00:00 2001 From: Pedro Rodrigues Date: Tue, 14 Jul 2026 18:10:38 +0100 Subject: [PATCH 3/3] docs: document BRAINTRUST_API_KEY in .env.example (AI-921) Co-Authored-By: Claude Opus 4.8 (1M context) --- .env.example | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.env.example b/.env.example index 19a0696f..cbf64118 100644 --- a/.env.example +++ b/.env.example @@ -1,3 +1,7 @@ # AI SDK Core direct-provider credentials. ANTHROPIC_API_KEY= OPENAI_API_KEY= + +# Braintrust — required for `pnpm upload:braintrust` (not for `--dry`). +# Optional: BRAINTRUST_PROJECT overrides the target project (default "supabase-evals"). +BRAINTRUST_API_KEY=