feat: store eval results in a Supabase project (AI-922)

.github/workflows/eval-refresh.yml

@@ -32,24 +32,20 @@ on:
         type: boolean
         required: false
         default: false
-      commit_to_branch:
-        description: "Commit exported results to the dispatched branch instead of opening a PR"
-        type: boolean
-        required: false
-        default: false
   pull_request:
     # Run whenever a PR carrying the run-evals label is opened, pushed to, or
     # receives the run-evals label. The job-level `if` gates on those cases.
     types: [opened, synchronize, labeled]
 
+# Results are published to Supabase, not committed back to the repo, so the
+# workflow only needs read access to check out and run.
 permissions:
-  contents: write
-  pull-requests: write
+  contents: read
 
 concurrency:
   group: eval-refresh-${{ github.event_name == 'pull_request' && github.event.pull_request.number || github.ref }}
   # Supersede an in-flight run when a new commit is pushed to the same PR, but
-  # never cancel a workflow_dispatch run (those open the refresh PR).
+  # never cancel a workflow_dispatch run (those publish the canonical results).
   cancel-in-progress: ${{ github.event_name == 'pull_request' }}
 
 jobs:
@@ -109,8 +105,11 @@ jobs:
             filter_changed="true"
           fi
 
-          # do_merge drives the export --merge (graft into existing results). It's
-          # always on for the changed path, and opt-in for manual dispatch.
+          # do_merge drives the export --merge, which grafts into an existing
+          # eval-results.json. In CI that file is no longer committed, so a fresh
+          # checkout has none and --merge only affects this run's own export;
+          # cross-run accumulation now comes from the Supabase upsert. Kept for
+          # local runs (where the gitignored file persists between exports).
           do_merge="$filter_changed"
           if [ "${{ github.event_name }}" = "workflow_dispatch" ] && [ "${{ inputs.merge }}" = "true" ]; then
             do_merge="true"
@@ -293,6 +292,9 @@ jobs:
     needs: [prepare, run-evals]
     if: needs.prepare.outputs.evals != '[]'
     runs-on: ubuntu-latest
+    env:
+      SUPABASE_URL: ${{ secrets.SUPABASE_URL }}
+      SUPABASE_SERVICE_ROLE_KEY: ${{ secrets.SUPABASE_SERVICE_ROLE_KEY }}
     steps:
       - name: Checkout
         uses: actions/checkout@9f698171ed81b15d1823a05fc7211befd50c8ae0 # v6.0.3
@@ -358,50 +360,17 @@ jobs:
           path: apps/web/src/data/eval-results.json
           retention-days: 7
 
-      - name: Commit exported results to branch
-        # PR runs commit to the PR head branch. Manual dispatch commits to the
-        # selected branch only when commit_to_branch is enabled.
-        if: >-
-          github.event_name == 'pull_request' ||
-          (github.event_name == 'workflow_dispatch' && inputs.commit_to_branch)
-        shell: bash
+      # Publish the exported snapshot to the Supabase store — the
+      # leaderboard's durable source of truth, replacing the committed JSON.
+      # Upserts by (experiment, eval); the web app reads from Supabase at runtime.
+      - name: Upload results to Supabase
+        if: ${{ env.SUPABASE_URL != '' && env.SUPABASE_SERVICE_ROLE_KEY != '' }}
+        run: pnpm --filter @supabase-evals/framework upload:supabase
+
+      # Supabase is now the only persistence path — a missing secret means results
+      # silently vanish. Fail loudly so a misconfigured run doesn't report success.
+      - name: Fail if Supabase not configured
+        if: ${{ env.SUPABASE_URL == '' || env.SUPABASE_SERVICE_ROLE_KEY == '' }}
         run: |
-          set -euo pipefail
-
-          git config user.name "github-actions[bot]"
-          # github-actions[bot]'s noreply email uses its public user ID: https://github.com/actions/checkout#push-a-commit-using-the-built-in-token
-          git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
-
-          git add apps/web/src/data/eval-results.json
-
-          if git diff --cached --quiet; then
-            echo "No eval result changes to commit"
-            exit 0
-          fi
-
-          git commit -m "chore: refresh eval results"
-          git push
-
-      - name: Generate GitHub App token
-        id: generate-token
-        if: github.event_name == 'workflow_dispatch' && !inputs.commit_to_branch
-        uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0
-        with:
-          app-id: ${{ secrets.GH_APP_ID }}
-          private-key: ${{ secrets.GH_APP_PRIVATE_KEY }}
-
-      - name: Create results pull request
-        if: github.event_name == 'workflow_dispatch' && !inputs.commit_to_branch
-        uses: peter-evans/create-pull-request@5f6978faf089d4d20b00c7766989d076bb2fc7f1 # v8.1.1
-        with:
-          token: ${{ steps.generate-token.outputs.token }}
-          add-paths: apps/web/src/data/eval-results.json
-          # Per-base head branch so a branch refresh PR can't collide with main's.
-          branch: chore/refresh-eval-results-${{ github.ref_name }}
-          base: ${{ github.ref_name }}
-          commit-message: "chore: refresh eval results"
-          title: "chore: refresh eval results"
-          body: |
-            Refreshes `apps/web/src/data/eval-results.json` from the latest automated eval run.
-          draft: true
-          delete-branch: true
+          echo "::error::SUPABASE_URL / SUPABASE_SERVICE_ROLE_KEY are not set — results were NOT published to Supabase. Configure the repo secrets."
+          exit 1

.gitignore

@@ -6,3 +6,5 @@ dist/
 results/*/
 .sync-tmp/
 
+apps/web/src/data/eval-results.json
+

apps/framework/package.json

@@ -11,6 +11,7 @@
     "typecheck": "tsc --noEmit",
     "test:framework": "node --env-file-if-exists=../../.env --import tsx/esm scripts/smoke-framework.ts",
     "export-results": "node --import tsx/esm scripts/export-results.ts",
+    "upload:supabase": "node --env-file-if-exists=../../.env --import tsx/esm scripts/upload-to-supabase.ts",
     "demo:mcp": "node --env-file=../../.env --import tsx/esm scripts/mcp-demo.ts",
     "demo:executor": "node --env-file=../../.env --import tsx/esm scripts/executor-demo.ts"
   },

apps/framework/scripts/upload-to-supabase.ts

@@ -0,0 +1,184 @@
+#!/usr/bin/env tsx
+/**
+ * Upsert the exported `eval-results.json` snapshot into the Supabase eval-results
+ * store — the durable, queryable source of truth for the public
+ * leaderboard, replacing the committed JSON file.
+ *
+ * One row per (experiment, eval); re-running upserts on that key so the store
+ * always reflects the latest snapshot. Writes use the service-role key (bypasses
+ * RLS); the web app reads via the anon key under the "public read" policy.
+ *
+ * Usage:
+ *   pnpm upload:supabase                 # upsert the default snapshot
+ *   pnpm upload:supabase -- --dry        # map + print, no network (no keys needed)
+ *   pnpm upload:supabase -- --input path/to/results.json
+ *
+ * Requires SUPABASE_URL and SUPABASE_SERVICE_ROLE_KEY in the environment
+ * (loaded from repo-root .env) unless running with --dry.
+ */
+import { readFile } from "node:fs/promises";
+import { dirname, resolve } from "node:path";
+import { fileURLToPath } from "node:url";
+import { createClient } from "@supabase/supabase-js";
+import type { EvalResult } from "@supabase-evals/core/eval-metadata";
+
+const __dirname = dirname(fileURLToPath(import.meta.url));
+const ROOT = resolve(__dirname, "..", "..", "..");
+const DEFAULT_INPUT = resolve(
+  ROOT,
+  "apps",
+  "web",
+  "src",
+  "data",
+  "eval-results.json",
+);
+const TABLE = "eval_results";
+const CHUNK_SIZE = 500;
+
+function readFlag(args: string[], name: string): string | undefined {
+  const index = args.indexOf(`--${name}`);
+  if (index === -1 || index === args.length - 1) {
+    return undefined;
+  }
+  return args[index + 1];
+}
+
+const rawArgs = process.argv.slice(2);
+const DRY = rawArgs.includes("--dry");
+const INPUT_PATH = resolve(ROOT, readFlag(rawArgs, "input") ?? DEFAULT_INPUT);
+
+// A result row as it appears in the web-facing snapshot: the core EvalResult
+// plus the display/source fields export-results.ts adds.
+type ResultRow = EvalResult & {
+  experimentSuite?: string;
+  prompt?: string;
+  promptSourcePath?: string;
+  sourcePath?: string;
+};
+
+// One row of the public.eval_results table (snake_case columns).
+type EvalResultRow = {
+  experiment: string;
+  eval: string;
+  experiment_suite: string | null;
+  agent: string | null;
+  model_provider: string | null;
+  model_id: string | null;
+  reasoning_effort: string | null;
+  stage: string | null;
+  product: string[] | null;
+  topic: string[] | null;
+  suite: string | null;
+  interface: string | null;
+  cli_version: string | null;
+  passed: boolean;
+  checks: unknown;
+  attempts: number | null;
+  skills: unknown;
+  prompt: string | null;
+  prompt_source_path: string | null;
+  source_path: string | null;
+};
+
+function isResultRow(value: unknown): value is ResultRow {
+  return (
+    typeof value === "object" &&
+    value !== null &&
+    typeof (value as ResultRow).experiment === "string" &&
+    typeof (value as ResultRow).eval === "string"
+  );
+}
+
+/** Map a snapshot row onto an eval_results table row. */
+export function toEvalResultRow(row: ResultRow): EvalResultRow {
+  return {
+    experiment: row.experiment,
+    eval: row.eval,
+    experiment_suite: row.experimentSuite ?? null,
+    agent: row.experimentDisplay?.agent ?? null,
+    model_provider: row.experimentDisplay?.modelProvider ?? null,
+    model_id: row.experimentDisplay?.modelId ?? null,
+    reasoning_effort: row.experimentDisplay?.reasoningEffort ?? null,
+    stage: row.stage ?? null,
+    product: row.product ?? null,
+    topic: row.topic ?? null,
+    suite: row.suite ?? null,
+    interface: row.interface ?? null,
+    cli_version: row.cliVersion ?? null,
+    passed: row.passed === true,
+    checks: row.checks ?? null,
+    attempts: row.attempts ?? null,
+    skills: row.skills ?? null,
+    prompt: row.prompt ?? null,
+    prompt_source_path: row.promptSourcePath ?? null,
+    source_path: row.sourcePath ?? null,
+  };
+}
+
+async function loadResults(path: string): Promise<ResultRow[]> {
+  const parsed: unknown = JSON.parse(await readFile(path, "utf8"));
+  if (!Array.isArray(parsed)) {
+    throw new Error(`Expected an array of results in ${path}`);
+  }
+  const rows = parsed.filter(isResultRow);
+  const skipped = parsed.length - rows.length;
+  if (skipped > 0) {
+    console.warn(
+      `Skipped ${skipped} malformed record(s) (missing experiment/eval).`,
+    );
+  }
+  return rows;
+}
+
+function chunk<T>(items: T[], size: number): T[][] {
+  const chunks: T[][] = [];
+  for (let i = 0; i < items.length; i += size) {
+    chunks.push(items.slice(i, i + size));
+  }
+  return chunks;
+}
+
+async function main(): Promise<void> {
+  const rows = (await loadResults(INPUT_PATH)).map(toEvalResultRow);
+  if (rows.length === 0) {
+    throw new Error(`No usable results found in ${INPUT_PATH}`);
+  }
+
+  if (DRY) {
+    const passed = rows.filter((row) => row.passed).length;
+    console.log(
+      `[dry run] would upsert ${rows.length} row(s) into "${TABLE}" ` +
+        `(${passed} pass, ${rows.length - passed} fail) from ${INPUT_PATH}`,
+    );
+    console.log(JSON.stringify(rows[0], null, 2));
+    return;
+  }
+
+  const url = process.env.SUPABASE_URL;
+  const serviceRoleKey = process.env.SUPABASE_SERVICE_ROLE_KEY;
+  if (!url || !serviceRoleKey) {
+    throw new Error(
+      "SUPABASE_URL and SUPABASE_SERVICE_ROLE_KEY must be set (or run with --dry).",
+    );
+  }
+
+  const supabase = createClient(url, serviceRoleKey, {
+    auth: { persistSession: false },
+  });
+
+  console.log(`Upserting ${rows.length} row(s) into "${TABLE}" at ${url}...`);
+  for (const batch of chunk(rows, CHUNK_SIZE)) {
+    const { error } = await supabase
+      .from(TABLE)
+      .upsert(batch, { onConflict: "experiment,eval" });
+    if (error) {
+      throw new Error(`Upsert failed: ${error.message}`);
+    }
+  }
+  console.log("Done.");
+}
+
+main().catch((error: unknown) => {
+  console.error(error);
+  process.exit(1);
+});

apps/web/.env.example

@@ -0,0 +1,5 @@
+# Supabase eval-results store - the leaderboard's read path.
+# Use the project's publishable key (sb_publishable_...), which is safe for the
+# browser; reads are gated by the table's "public read" RLS policy.
+VITE_SUPABASE_URL=
+VITE_SUPABASE_PUBLISHABLE_KEY=

apps/web/package.json

@@ -14,6 +14,7 @@
   "dependencies": {
     "@fontsource-variable/inter": "^5.2.8",
     "@supabase-evals/core": "workspace:*",
+    "@supabase/supabase-js": "catalog:",
     "@tailwindcss/vite": "^4.2.1",
     "class-variance-authority": "^0.7.1",
     "clsx": "^2.1.1",