From 52787353977a2afe7ea9f6a951a8d177eb23553c Mon Sep 17 00:00:00 2001
From: Barry Roodt <barry.roodt@supabase.io>
Date: Tue, 28 Jul 2026 15:47:35 +0200
Subject: [PATCH 1/6] refactor(framework): run framework tests through Vitest

AI-934. The 659-line smoke-framework.ts script becomes Vitest files: six
scorer tests next to the EVAL.ts they cover, four harness tests next to the
harness module they cover, shared plumbing in harness/scorer-test-kit.ts.
Adds a CI job so these actually run on PRs, which is the point of the ticket.

Colocation lets each scorer test import its scorer directly, so the old
dynamic loadScorer(path) is gone and the scorer/test contract is typechecked.

Two things the old script hid:

- Every failure was invisible. It stubbed console.error globally to silence
  expected-fail scorer noise, then reported failures through console.error, so
  a failing run printed nothing and exited 1. Vitest reports failures itself,
  so the stub moves to an onConsoleLog filter that cannot swallow a result.
- The frontend build test was failing for real (AI-975). Fixed here because a
  CI job that ships red is worse than no CI job: project-runner hardcoded
  <repo>/node_modules/{vite,vitest}, which pnpm's isolated layout never
  creates, and the frontend toolchain was declared in apps/framework where a
  scored workspace under results/ cannot resolve it. Bins now resolve via each
  package's exported package.json rather than a deep subpath that exports may
  stop publishing, and the workspace-facing toolchain moves to the root
  manifest, matching the documented contract that scoring runs repo-root
  vite/vitest so the sandbox need not carry it. Nothing in apps/framework
  imported those packages; they appeared only in generated workspace config
  and fixture strings. build-frontend-001-todos-app has never produced a
  result row, which is why nobody noticed.

The suite is hermetic: investigate-security-001's scorer calls a real OpenAI
judge, so that test mocks judge() alone and keeps every other core export
real. Verified green with OPENAI_API_KEY/ANTHROPIC_API_KEY unset.
---
 .github/workflows/biome.yml                   |  25 +
 .../harness/platform-backend.test.ts          |  69 ++
 apps/framework/harness/project-runner.test.ts | 141 ++++
 apps/framework/harness/project-runner.ts      |  19 +-
 apps/framework/harness/scorer-test-kit.ts     | 124 ++++
 apps/framework/package.json                   |  11 +-
 apps/framework/scripts/smoke-framework.ts     | 659 ------------------
 apps/framework/tsconfig.json                  |   3 +-
 apps/framework/vitest.config.ts               |  22 +
 .../EVAL.test.ts                              |  52 ++
 .../EVAL.test.ts                              |  61 ++
 .../EVAL.test.ts                              | 141 ++++
 .../EVAL.test.ts                              |  29 +
 .../EVAL.test.ts                              |  26 +
 .../EVAL.test.ts                              |  84 +++
 package.json                                  |  12 +-
 pnpm-lock.yaml                                |  57 +-
 17 files changed, 832 insertions(+), 703 deletions(-)
 create mode 100644 apps/framework/harness/platform-backend.test.ts
 create mode 100644 apps/framework/harness/project-runner.test.ts
 create mode 100644 apps/framework/harness/scorer-test-kit.ts
 delete mode 100644 apps/framework/scripts/smoke-framework.ts
 create mode 100644 apps/framework/vitest.config.ts
 create mode 100644 evals/build-functions-001-order-total/EVAL.test.ts
 create mode 100644 evals/build-functions-002-edge-auth-db/EVAL.test.ts
 create mode 100644 evals/build-functions-004-service-role-bypass/EVAL.test.ts
 create mode 100644 evals/build-rls-002-own-todos-client/EVAL.test.ts
 create mode 100644 evals/investigate-logs-001-top-error-function/EVAL.test.ts
 create mode 100644 evals/investigate-security-001-public-table/EVAL.test.ts

diff --git a/.github/workflows/biome.yml b/.github/workflows/biome.yml
index fc635388..e49bc2a7 100644
--- a/.github/workflows/biome.yml
+++ b/.github/workflows/biome.yml
@@ -29,3 +29,28 @@ jobs:
 
       - name: Check formatting
         run: pnpm format:check
+
+  check:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@9f698171ed81b15d1823a05fc7211befd50c8ae0 # v6.0.3
+        with:
+          persist-credentials: false
+
+      - name: Install pnpm
+        uses: pnpm/action-setup@fc06bc1257f339d1d5d8b3a19a8cae5388b55320 # v5.0.0
+
+      - name: Setup Node.js
+        uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
+        with:
+          node-version-file: .node-version
+          cache: pnpm
+
+      - name: Install dependencies
+        run: pnpm install --frozen-lockfile
+
+      # Typechecks plus the Vitest suites. No API keys: every test that would
+      # reach a model stubs it, so this must never need credentials or spend.
+      - name: Run checks
+        run: pnpm check
diff --git a/apps/framework/harness/platform-backend.test.ts b/apps/framework/harness/platform-backend.test.ts
new file mode 100644
index 00000000..6b45fd9f
--- /dev/null
+++ b/apps/framework/harness/platform-backend.test.ts
@@ -0,0 +1,69 @@
+import { expect, test } from 'vitest';
+import { bootPlatformBackend } from './platform-backend.js';
+import { seedPath, withBackend } from './scorer-test-kit.js';
+
+const LOGS_EVAL = 'evals/investigate-logs-001-top-error-function';
+
+test('supalite auth issues a session supabase-js can write and read under RLS', async () => {
+  await withBackend({}, async (backend) => {
+    await backend.query(`
+CREATE TABLE todos (
+  id uuid PRIMARY KEY DEFAULT gen_random_uuid(),
+  user_id uuid NOT NULL,
+  body text NOT NULL
+);
+
+ALTER TABLE todos ENABLE ROW LEVEL SECURITY;
+GRANT SELECT, INSERT ON todos TO authenticated;
+
+CREATE POLICY "users can insert their own todos" ON todos FOR INSERT TO authenticated WITH CHECK (user_id = auth.uid());
+CREATE POLICY "users can read their own todos" ON todos FOR SELECT TO authenticated USING (user_id = auth.uid());
+    `);
+
+    const client = backend.client;
+    const { data: signup, error: signupError } = await client.auth.signUp({
+      email: `smoke-${Date.now()}@example.com`,
+      password: 'secret123',
+    });
+    expect(signupError).toBeNull();
+    expect(signup.user?.id).toBeTruthy();
+
+    const { error: insertError } = await client.from('todos').insert({
+      user_id: signup.user?.id,
+      body: 'verify supabase-js path',
+    });
+    expect(insertError).toBeNull();
+
+    const { data: rows, error: selectError } = await client
+      .from('todos')
+      .select('body')
+      .eq('user_id', signup.user?.id);
+    expect(selectError).toBeNull();
+    expect(rows).toEqual([{ body: 'verify supabase-js path' }]);
+  });
+});
+
+test('close disposes the platform, and is idempotent', async () => {
+  const backend = await bootPlatformBackend({});
+  await backend.query('select 1 as n');
+  await backend.close();
+
+  await expect(backend.query('select 1 as n')).rejects.toThrow();
+  await expect(backend.close()).resolves.not.toThrow();
+});
+
+test('seeded logs are queryable over the analytics endpoint', async () => {
+  await withBackend(
+    { logsSeedJsonl: seedPath(LOGS_EVAL, 'logs.jsonl') },
+    async ({ url, ref, accessToken }) => {
+      const sql = 'SELECT count(*)::int AS n FROM edge_logs';
+      const res = await fetch(
+        `${url}/v1/projects/${ref}/analytics/endpoints/logs.all?sql=${encodeURIComponent(sql)}`,
+        { headers: { Authorization: `Bearer ${accessToken}` } }
+      );
+      const body = (await res.json()) as { result: Array<{ n: number }> };
+
+      expect(body.result[0]?.n).toBeGreaterThan(0);
+    }
+  );
+});
diff --git a/apps/framework/harness/project-runner.test.ts b/apps/framework/harness/project-runner.test.ts
new file mode 100644
index 00000000..7897806a
--- /dev/null
+++ b/apps/framework/harness/project-runner.test.ts
@@ -0,0 +1,141 @@
+import { cpSync, existsSync, mkdirSync, rmSync, writeFileSync } from 'node:fs';
+import { dirname, join } from 'node:path';
+import { expect, test } from 'vitest';
+import { resolvePackageBin, viteBuild, vitestRun } from './project-runner.js';
+import { ROOT } from './scorer-test-kit.js';
+
+const FRONTEND_EVAL = 'evals/build-frontend-001-todos-app';
+
+const GOOD_FRONTEND_APP = `
+import { useState } from "react";
+import { createClient } from "@supabase/supabase-js";
+
+type Todo = { id: string; body: string; done: boolean };
+
+export const supabase = createClient(
+  import.meta.env.VITE_SUPABASE_URL,
+  import.meta.env.VITE_SUPABASE_ANON_KEY
+);
+
+export default function App() {
+  const [email, setEmail] = useState("");
+  const [password, setPassword] = useState("");
+  const [newTodo, setNewTodo] = useState("");
+  const [signedIn, setSignedIn] = useState(false);
+  const [todos, setTodos] = useState<Todo[]>([]);
+  const [error, setError] = useState("");
+
+  async function loadTodos() {
+    const { data, error } = await supabase.from("todos").select("id,body,done").order("created_at", { ascending: true });
+    if (error) throw error;
+    setTodos(data ?? []);
+  }
+
+  async function handleSignIn(event: React.FormEvent) {
+    event.preventDefault();
+    setError("");
+    const { error } = await supabase.auth.signInWithPassword({ email, password });
+    if (error) { setError(error.message); return; }
+    setSignedIn(true);
+    await loadTodos();
+  }
+
+  async function handleAddTodo(event: React.FormEvent) {
+    event.preventDefault();
+    const body = newTodo.trim();
+    if (!body) return;
+    setError("");
+    const { data, error } = await supabase.from("todos").insert({ body }).select("id,body,done").single();
+    if (error) { setError(error.message); return; }
+    setTodos((current) => [...current, data]);
+    setNewTodo("");
+  }
+
+  async function handleToggleTodo(todo: Todo) {
+    setError("");
+    const { data, error } = await supabase.from("todos").update({ done: !todo.done }).eq("id", todo.id).select("id,body,done").single();
+    if (error) { setError(error.message); return; }
+    setTodos((current) => current.map((item) => (item.id === todo.id ? data : item)));
+  }
+
+  return (
+    <main>
+      <h1>Todos</h1>
+      <form onSubmit={handleSignIn}>
+        <input data-testid="email-input" placeholder="Email" value={email} onChange={(e) => setEmail(e.target.value)} />
+        <input data-testid="password-input" placeholder="Password" type="password" value={password} onChange={(e) => setPassword(e.target.value)} />
+        <button data-testid="sign-in-button" type="submit">Sign in</button>
+      </form>
+      {signedIn ? <p data-testid="signed-in">Signed in</p> : null}
+      {error ? <p role="alert">{error}</p> : null}
+      <form onSubmit={handleAddTodo}>
+        <input data-testid="todo-input" placeholder="New todo" value={newTodo} onChange={(e) => setNewTodo(e.target.value)} />
+        <button data-testid="add-button" type="submit">Add</button>
+      </form>
+      <ul data-testid="todo-list">
+        {todos.map((todo) => (
+          <li key={todo.id}>
+            <label>
+              <input data-testid={\`todo-checkbox-\${todo.body}\`} type="checkbox" checked={todo.done} onChange={() => void handleToggleTodo(todo)} />
+              {todo.body}
+            </label>
+          </li>
+        ))}
+      </ul>
+    </main>
+  );
+}
+`;
+
+// Regression guard for AI-975: pnpm's isolated layout has no hoisted
+// `<repo>/node_modules/<pkg>`, so a repo-root path missed every time and the
+// bin never even launched.
+test.each([
+  ['vite', 'bin/vite.js'],
+  ['vitest', 'vitest.mjs'],
+])('resolves the %s binary that actually exists on disk', (pkg, entry) => {
+  const resolved = resolvePackageBin(pkg, entry);
+
+  expect(existsSync(resolved), `${pkg} bin missing at ${resolved}`).toBe(true);
+});
+
+// AI-975, second layer: the workspace's own `vite.config.ts` imports `vite`,
+// and vite compiles that config into `<repo>/node_modules/.vite-temp/`, so
+// resolution anchors at the repo root. The scored workspace lives under
+// `results/` and can only walk up to the repo root too. That is the documented
+// contract (README, and the `copyToHost` doc comment: score with repo-root
+// vite/vitest so the toolchain need not exist in the sandbox), so the root
+// manifest owns the frontend toolchain. This fixture copies `local/` with no
+// `node_modules`, exactly as that contract assumes.
+test('builds and tests a known-good frontend workspace', async () => {
+  const workspace = join(
+    ROOT,
+    'results',
+    '_smoke',
+    'build-frontend-001-todos-app'
+  );
+  rmSync(workspace, { recursive: true, force: true });
+  mkdirSync(dirname(workspace), { recursive: true });
+  cpSync(join(ROOT, FRONTEND_EVAL, 'local'), workspace, {
+    recursive: true,
+    filter: (src) => !src.endsWith('/EVAL.ts'),
+  });
+  cpSync(join(ROOT, FRONTEND_EVAL, 'tests'), join(workspace, 'tests'), {
+    recursive: true,
+  });
+  writeFileSync(
+    join(workspace, '.env.local'),
+    [
+      'VITE_SUPABASE_URL=http://supabase-evals.local',
+      'VITE_SUPABASE_ANON_KEY=supabase-evals-anon-key',
+      '',
+    ].join('\n')
+  );
+  writeFileSync(join(workspace, 'src', 'App.tsx'), GOOD_FRONTEND_APP);
+
+  const build = await viteBuild(workspace);
+  expect(build.ok, build.stderr || build.stdout).toBe(true);
+
+  const vitest = await vitestRun(workspace);
+  expect(vitest.ok, vitest.stderr || vitest.stdout).toBe(true);
+});
diff --git a/apps/framework/harness/project-runner.ts b/apps/framework/harness/project-runner.ts
index d3a75035..500b7302 100644
--- a/apps/framework/harness/project-runner.ts
+++ b/apps/framework/harness/project-runner.ts
@@ -1,15 +1,24 @@
 import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs';
 import { dirname, join } from 'node:path';
-import { fileURLToPath } from 'node:url';
+import { createRequire } from 'node:module';
 import { spawn } from 'node:child_process';
 import type { CommandResult, VitestResult } from '@supabase-evals/core';
 
-const __dirname = dirname(fileURLToPath(import.meta.url));
-const ROOT = join(__dirname, '..', '..', '..');
+const nodeRequire = createRequire(import.meta.url);
+
+/**
+ * Resolve a dependency's bundled entry script. pnpm's isolated layout never
+ * creates a hoisted `<repo>/node_modules/<pkg>`, so anchoring on the repo root
+ * misses every time. Anchor on the `package.json` each package exports instead
+ * of requesting a deep subpath, which `exports` may stop publishing on a bump.
+ */
+export function resolvePackageBin(pkg: string, entry: string): string {
+  return join(dirname(nodeRequire.resolve(`${pkg}/package.json`)), entry);
+}
 
 export async function viteBuild(workspace: string): Promise<CommandResult> {
   return runNodeBin(
-    join(ROOT, 'node_modules', 'vite', 'bin', 'vite.js'),
+    resolvePackageBin('vite', 'bin/vite.js'),
     ['build'],
     workspace
   );
@@ -38,7 +47,7 @@ export async function vitestRun(workspace: string): Promise<VitestResult> {
     ].join('\n')
   );
   const result = await runNodeBin(
-    join(ROOT, 'node_modules', 'vitest', 'vitest.mjs'),
+    resolvePackageBin('vitest', 'vitest.mjs'),
     [
       'run',
       '--config',
diff --git a/apps/framework/harness/scorer-test-kit.ts b/apps/framework/harness/scorer-test-kit.ts
new file mode 100644
index 00000000..5d7cc78e
--- /dev/null
+++ b/apps/framework/harness/scorer-test-kit.ts
@@ -0,0 +1,124 @@
+/**
+ * Helpers for testing eval scorers against a real platform-lite backend.
+ *
+ * Scorer tests live next to the scorer they cover (`evals/<id>/EVAL.test.ts`)
+ * and import it directly; harness tests live beside the harness module they
+ * cover. Both import from here so a scorer test stays a description of the
+ * scenario rather than a pile of backend plumbing.
+ */
+import { dirname, join } from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { bootPlatformBackend } from './platform-backend.js';
+import type { PlatformBackend } from './platform-backend.js';
+import type {
+  EdgeFunctionsInvokeResult,
+  ToolEvalContext,
+  TranscriptPart,
+} from './types.js';
+
+/** Repo root, from `apps/framework/harness/`. */
+export const ROOT = join(
+  dirname(fileURLToPath(import.meta.url)),
+  '..',
+  '..',
+  '..'
+);
+
+/** Path to a file in an eval's `remote/` seed directory. */
+export function seedPath(relDir: string, file: string): string {
+  return join(ROOT, relDir, 'remote', file);
+}
+
+/** A `ToolEvalContext` backed by a live platform-lite project. */
+export function scorerCtx(
+  backend: PlatformBackend,
+  extra?: { agentReport?: string; transcript?: TranscriptPart[] }
+): ToolEvalContext {
+  return {
+    mgmt: backend.mgmt,
+    ref: backend.ref,
+    client: backend.client,
+    getClient: backend.getClient,
+    query: backend.query,
+    invokeFunction: backend.invokeFunction,
+    toolCalls: [],
+    transcript: extra?.transcript ?? [],
+    agentReport: extra?.agentReport,
+  };
+}
+
+/** Boot a backend, run `fn` against it, and always close it. */
+export async function withBackend<T>(
+  opts: { projectSeedSql?: string; logsSeedJsonl?: string },
+  fn: (backend: PlatformBackend) => Promise<T>
+): Promise<T> {
+  const backend = await bootPlatformBackend(opts);
+  try {
+    return await fn(backend);
+  } finally {
+    await backend.close();
+  }
+}
+
+/** Serialized checks, for use as an assertion failure message. */
+export function checksMessage(result: { checks?: unknown[] }) {
+  return JSON.stringify(result.checks ?? []);
+}
+
+/** Names of the checks that failed, for asserting on *which* check broke. */
+export function failedCheckNames(result: {
+  checks?: { name: string; passed: boolean }[];
+}) {
+  return (
+    result.checks
+      ?.filter((check) => !check.passed)
+      .map((check) => check.name) ?? []
+  );
+}
+
+/** An edge function response, as the scorer's `invokeFunction` would see it. */
+export function functionResponse(
+  status: number,
+  body = '',
+  outboundBearerTokens: string[] = []
+): EdgeFunctionsInvokeResult {
+  return { type: 'response', status, headers: {}, body, outboundBearerTokens };
+}
+
+/**
+ * Deploy an edge function through platform-lite's management API, the same
+ * path the real harness uses.
+ */
+export async function deployFunction(
+  backend: PlatformBackend,
+  slug: string,
+  source: string,
+  opts: { verifyJwt: boolean }
+): Promise<void> {
+  const form = new FormData();
+  form.append(
+    'metadata',
+    JSON.stringify({
+      name: slug,
+      verify_jwt: opts.verifyJwt,
+      entrypoint_path: 'index.ts',
+    })
+  );
+  form.append(
+    'file',
+    new File([source], 'index.ts', { type: 'application/typescript' })
+  );
+
+  const res = await fetch(
+    `${backend.url}/v1/projects/${backend.ref}/functions/deploy?slug=${slug}`,
+    {
+      method: 'POST',
+      headers: { Authorization: `Bearer ${backend.accessToken}` },
+      body: form,
+    }
+  );
+
+  if (res.status !== 201) {
+    throw new Error(`deploy of ${slug} failed: ${await res.text()}`);
+  }
+}
diff --git a/apps/framework/package.json b/apps/framework/package.json
index c726bc37..710155d9 100644
--- a/apps/framework/package.json
+++ b/apps/framework/package.json
@@ -9,7 +9,7 @@
     "eval:dry": "node --env-file=../../.env --import tsx/esm harness/run-eval.ts --dry",
     "eval:smoke": "node --env-file=../../.env --import tsx/esm harness/run-eval.ts --smoke",
     "typecheck": "tsc --noEmit",
-    "test:framework": "node --env-file-if-exists=../../.env --import tsx/esm scripts/smoke-framework.ts",
+    "test:framework": "vitest run",
     "export-results": "node --import tsx/esm scripts/export-results.ts",
     "demo:mcp": "node --env-file=../../.env --import tsx/esm scripts/mcp-demo.ts",
     "demo:executor": "node --env-file=../../.env --import tsx/esm scripts/executor-demo.ts"
@@ -22,16 +22,7 @@
     "@electric-sql/pglite": "catalog:",
     "@supabase-evals/core": "workspace:*",
     "@supabase-evals/sandbox": "workspace:*",
-    "@supabase/supabase-js": "catalog:",
-    "@testing-library/jest-dom": "^6.9.1",
-    "@testing-library/react": "^16.3.2",
-    "@vitejs/plugin-react": "catalog:",
     "ai": "catalog:",
-    "happy-dom": "^20.9.0",
-    "@supabase/lite": "catalog:",
-    "react": "^19.2.5",
-    "react-dom": "^19.2.5",
-    "vite": "catalog:",
     "vitest": "catalog:"
   },
   "devDependencies": {
diff --git a/apps/framework/scripts/smoke-framework.ts b/apps/framework/scripts/smoke-framework.ts
deleted file mode 100644
index 6709e3be..00000000
--- a/apps/framework/scripts/smoke-framework.ts
+++ /dev/null
@@ -1,659 +0,0 @@
-import assert from 'node:assert/strict';
-import { cpSync, mkdirSync, rmSync, writeFileSync } from 'node:fs';
-import { dirname, join } from 'node:path';
-import { fileURLToPath, pathToFileURL } from 'node:url';
-import { bootPlatformBackend } from '../harness/platform-backend.js';
-import { viteBuild, vitestRun } from '../harness/project-runner.js';
-import type {
-  EdgeFunctionsInvokeResult,
-  ToolEvalContext,
-  ToolScorer,
-  TranscriptPart,
-} from '../harness/types.js';
-import type { PlatformBackend } from '../harness/platform-backend.js';
-const __dirname = dirname(fileURLToPath(import.meta.url));
-const ROOT = join(__dirname, '..', '..', '..');
-
-const DEBUG = process.argv.includes('--debug');
-
-// Scorers trigger expected-fail supabase-js calls that log via console.error.
-// Silence it globally so smoke runs stay quiet on those.
-const stderr = console.error;
-if (!DEBUG) console.error = () => undefined;
-
-const CLIENT_RLS_EVAL = 'evals/build-rls-002-own-todos-client';
-const FUNCTIONS_EVAL = 'evals/build-functions-001-order-total';
-const EDGE_AUTH_DB_EVAL = 'evals/build-functions-002-edge-auth-db';
-const SERVICE_ROLE_BYPASS_EVAL =
-  'evals/build-functions-004-service-role-bypass';
-const INVESTIGATE_LOGS_EVAL = 'evals/investigate-logs-001-top-error-function';
-const INVESTIGATE_SECURITY_EVAL = 'evals/investigate-security-001-public-table';
-const FRONTEND_EVAL = 'evals/build-frontend-001-todos-app';
-
-async function loadScorer(relDir: string): Promise<ToolScorer> {
-  const mod = await import(pathToFileURL(join(ROOT, relDir, 'EVAL.ts')).href);
-  return mod.default as ToolScorer;
-}
-
-function scorerCtx(
-  backend: PlatformBackend,
-  extra?: { agentReport?: string; transcript?: TranscriptPart[] }
-) {
-  return {
-    mgmt: backend.mgmt,
-    ref: backend.ref,
-    client: backend.client,
-    getClient: backend.getClient,
-    query: backend.query,
-    invokeFunction: backend.invokeFunction,
-    toolCalls: [],
-    transcript: extra?.transcript ?? [],
-    agentReport: extra?.agentReport,
-  };
-}
-
-function checksMessage(result: { checks?: unknown[] }) {
-  return JSON.stringify(result.checks ?? []);
-}
-
-function failedCheckNames(result: {
-  checks?: { name: string; passed: boolean }[];
-}) {
-  return (
-    result.checks
-      ?.filter((check) => !check.passed)
-      .map((check) => check.name) ?? []
-  );
-}
-
-async function withBackend<T>(
-  opts: { projectSeedSql?: string; logsSeedJsonl?: string },
-  fn: (backend: PlatformBackend) => Promise<T>
-): Promise<T> {
-  const backend = await bootPlatformBackend(opts);
-  try {
-    return await fn(backend);
-  } finally {
-    await backend.close();
-  }
-}
-
-function seedPath(relDir: string, file: string): string {
-  return join(ROOT, relDir, 'remote', file);
-}
-
-async function smokeClientRlsEval() {
-  const scorer = await loadScorer(CLIENT_RLS_EVAL);
-
-  await withBackend(
-    { projectSeedSql: seedPath(CLIENT_RLS_EVAL, 'project.sql') },
-    async (backend) => {
-      await backend.query(`
-ALTER TABLE todos ENABLE ROW LEVEL SECURITY;
-
-CREATE POLICY "users can read own todos" ON todos FOR SELECT TO authenticated USING (user_id = auth.uid());
-CREATE POLICY "users can insert own todos" ON todos FOR INSERT TO authenticated WITH CHECK (user_id = auth.uid());
-CREATE POLICY "users can update own todos" ON todos FOR UPDATE TO authenticated USING (user_id = auth.uid()) WITH CHECK (user_id = auth.uid());
-CREATE POLICY "users can delete own todos" ON todos FOR DELETE TO authenticated USING (user_id = auth.uid());
-      `);
-
-      const result = await scorer(scorerCtx(backend));
-      assert.equal(result.passed, true, checksMessage(result));
-    }
-  );
-
-  console.log('PASS client-scored RLS scorer + supabase-js');
-}
-
-async function smokeFunctionsEval() {
-  const scorer = await loadScorer(FUNCTIONS_EVAL);
-
-  await withBackend({}, async (backend) => {
-    const before = await scorer(scorerCtx(backend));
-    assert.equal(before.passed, false);
-    assert.match(checksMessage(before), /function not found/i);
-
-    const deployUrl = `${backend.url}/v1/projects/${backend.ref}/functions/deploy?slug=order-total`;
-    const form = new FormData();
-    form.append(
-      'metadata',
-      JSON.stringify({
-        name: 'order-total',
-        verify_jwt: false,
-        entrypoint_path: 'index.ts',
-      })
-    );
-    form.append(
-      'file',
-      new File([ORDER_TOTAL_SOURCE], 'index.ts', {
-        type: 'application/typescript',
-      })
-    );
-
-    const deployRes = await fetch(deployUrl, {
-      method: 'POST',
-      headers: { Authorization: `Bearer ${backend.accessToken}` },
-      body: form,
-    });
-    assert.equal(
-      deployRes.status,
-      201,
-      `deploy failed: ${await deployRes.text()}`
-    );
-
-    const after = await scorer(scorerCtx(backend));
-    assert.equal(after.passed, true, checksMessage(after));
-  });
-
-  console.log('PASS functions scorer + edge-functions dispatcher');
-}
-
-function functionResponse(
-  status: number,
-  body = '',
-  outboundBearerTokens: string[] = []
-): EdgeFunctionsInvokeResult {
-  return { type: 'response', status, headers: {}, body, outboundBearerTokens };
-}
-
-// Unlike the other smokes, this one fakes the ToolEvalContext instead of
-// booting a real backend: the scorer's *decision logic* (which statuses and
-// bodies pass) is what we want to pin, and driving that through a real stack
-// would mean deploying six edge-function variants. `responses` are consumed in
-// the scorer's invocation order (missingAuth, ownNotes, aRequestsB, bRequestsA);
-// `serviceRoleResponses` builds them in that same order.
-function serviceRoleBypassCtx(
-  responses: EdgeFunctionsInvokeResult[]
-): ToolEvalContext {
-  const authResult = (id: string, accessToken: string) => ({
-    data: { user: { id }, session: { access_token: accessToken } },
-    error: null,
-  });
-  const clientA = {
-    auth: { signUp: async () => authResult('user-a', 'token-a') },
-  } as unknown as ToolEvalContext['client'];
-  const clientB = {
-    auth: { signUp: async () => authResult('user-b', 'token-b') },
-  } as unknown as ToolEvalContext['client'];
-
-  return {
-    mgmt: {} as ToolEvalContext['mgmt'],
-    ref: 'test-ref',
-    client: clientA,
-    getClient: () => clientB,
-    query: async () => ({ rows: [] }),
-    invokeFunction: async () => {
-      const response = responses.shift();
-      if (!response) throw new Error('missing fake function response');
-      return response;
-    },
-    toolCalls: [],
-    transcript: [],
-  };
-}
-
-// Order matches the scorer's invocation sequence; overrides are keyed by role.
-function serviceRoleResponses(
-  overrides: Partial<
-    Record<
-      'missingAuth' | 'ownNotes' | 'aRequestsB' | 'bRequestsA',
-      EdgeFunctionsInvokeResult
-    >
-  > = {}
-): EdgeFunctionsInvokeResult[] {
-  return [
-    overrides.missingAuth ?? functionResponse(401),
-    overrides.ownNotes ??
-      functionResponse(200, 'user A private note', ['token-a']),
-    overrides.aRequestsB ?? functionResponse(401, 'unauthorized'),
-    overrides.bRequestsA ?? functionResponse(403, 'forbidden'),
-  ];
-}
-
-async function smokeServiceRoleBypassEval() {
-  const scorer = await loadScorer(SERVICE_ROLE_BYPASS_EVAL);
-  const runScorer = (overrides?: Parameters<typeof serviceRoleResponses>[0]) =>
-    scorer(serviceRoleBypassCtx(serviceRoleResponses(overrides)));
-
-  // The recommended secure fix: reject anonymous access, serve the caller their
-  // own note over their JWT, and deny forced cross-user reads.
-  const secure = await runScorer();
-  assert.equal(secure.passed, true, checksMessage(secure));
-  assert.match(checksMessage(secure), /"notes":"status=401"/);
-  assert.match(checksMessage(secure), /"notes":"status=200"/);
-  assert.match(checksMessage(secure), /"notes":"status=403"/);
-  assert.match(
-    checksMessage(secure),
-    /"notes":"bearer_tokens=1, all_match=true"/
-  );
-
-  // Ignoring the spoofed user_id and returning the caller's own note is the
-  // other secure shape the scorer must accept.
-  const callerScoped = await runScorer({
-    aRequestsB: functionResponse(200, 'user A private note', ['token-a']),
-    bRequestsA: functionResponse(200, 'user B private note', ['token-b']),
-  });
-  assert.equal(callerScoped.passed, true, checksMessage(callerScoped));
-
-  // An RLS-scoped read of another user's note returns no rows, which a function
-  // may surface as 404. That is still a non-leaking denial.
-  const notFound = await runScorer({
-    aRequestsB: functionResponse(404, 'not found'),
-    bRequestsA: functionResponse(404, 'not found'),
-  });
-  assert.equal(notFound.passed, true, checksMessage(notFound));
-
-  // A leaked note fails regardless of status (200 or 403).
-  const leaky = await runScorer({
-    aRequestsB: functionResponse(200, 'user B private note'),
-    bRequestsA: functionResponse(403, 'user A private note'),
-  });
-  assert.equal(leaky.passed, false, checksMessage(leaky));
-  assert.deepEqual(failedCheckNames(leaky), [
-    'user A cannot force-read user B note',
-    'user B cannot force-read user A note',
-  ]);
-  assert.match(checksMessage(leaky), /"notes":"status=200"/);
-  assert.match(checksMessage(leaky), /"notes":"status=403"/);
-
-  // Serving data to an unauthenticated caller fails, even with no leaked note.
-  const ignoredAuth = await runScorer({ missingAuth: functionResponse(200) });
-  assert.equal(ignoredAuth.passed, false, checksMessage(ignoredAuth));
-  assert.deepEqual(failedCheckNames(ignoredAuth), ['rejects missing auth']);
-
-  // A 401 that still echoes a note is a leak, not a denial.
-  const unauthenticatedLeak = await runScorer({
-    missingAuth: functionResponse(401, 'user A private note'),
-  });
-  assert.equal(
-    unauthenticatedLeak.passed,
-    false,
-    checksMessage(unauthenticatedLeak)
-  );
-  assert.deepEqual(failedCheckNames(unauthenticatedLeak), [
-    'rejects missing auth',
-  ]);
-
-  // Reading via the service-role key instead of the caller's JWT fails.
-  const serviceRoleRead = await runScorer({
-    ownNotes: functionResponse(200, 'user A private note', [
-      'service-role-key',
-    ]),
-  });
-  assert.equal(serviceRoleRead.passed, false, checksMessage(serviceRoleRead));
-  assert.deepEqual(failedCheckNames(serviceRoleRead), [
-    "reads only with the caller's JWT",
-  ]);
-
-  console.log('PASS service-role bypass scorer security contract');
-}
-
-async function smokeSupaliteClient() {
-  await withBackend({}, async (backend) => {
-    await backend.query(`
-CREATE TABLE todos (
-  id uuid PRIMARY KEY DEFAULT gen_random_uuid(),
-  user_id uuid NOT NULL,
-  body text NOT NULL
-);
-
-ALTER TABLE todos ENABLE ROW LEVEL SECURITY;
-GRANT SELECT, INSERT ON todos TO authenticated;
-
-CREATE POLICY "users can insert their own todos" ON todos FOR INSERT TO authenticated WITH CHECK (user_id = auth.uid());
-CREATE POLICY "users can read their own todos" ON todos FOR SELECT TO authenticated USING (user_id = auth.uid());
-    `);
-
-    const client = backend.client;
-    const email = `smoke-${Date.now()}@example.com`;
-    const { data: signup, error: signupError } = await client.auth.signUp({
-      email,
-      password: 'secret123',
-    });
-    assert.equal(signupError, null);
-    assert(signup.user?.id);
-
-    const { error: insertError } = await client.from('todos').insert({
-      user_id: signup.user.id,
-      body: 'verify supabase-js path',
-    });
-    assert.equal(insertError, null);
-
-    const { data: rows, error: selectError } = await client
-      .from('todos')
-      .select('body')
-      .eq('user_id', signup.user.id);
-    assert.equal(selectError, null);
-    assert.deepEqual(rows, [{ body: 'verify supabase-js path' }]);
-  });
-
-  console.log('PASS supalite auth + supabase-js client');
-}
-
-async function smokePlatformBackendClose() {
-  const backend = await bootPlatformBackend({});
-  await backend.query('select 1 as n');
-  await backend.close();
-  await assert.rejects(() => backend.query('select 1 as n'));
-  await backend.close();
-
-  console.log('PASS platform backend close disposes platform');
-}
-
-async function smokeEdgeAuthDbEval() {
-  const scorer = await loadScorer(EDGE_AUTH_DB_EVAL);
-
-  await withBackend(
-    { projectSeedSql: seedPath(EDGE_AUTH_DB_EVAL, 'project.sql') },
-    async (backend) => {
-      // Deploy the function via platform-lite's HTTP management API
-      const deployUrl = `${backend.url}/v1/projects/${backend.ref}/functions/deploy?slug=todo-create`;
-      const form = new FormData();
-      form.append(
-        'metadata',
-        JSON.stringify({
-          name: 'todo-create',
-          verify_jwt: true,
-          entrypoint_path: 'index.ts',
-        })
-      );
-      form.append(
-        'file',
-        new File([TODO_CREATE_SOURCE], 'index.ts', {
-          type: 'application/typescript',
-        })
-      );
-
-      const deployRes = await fetch(deployUrl, {
-        method: 'POST',
-        headers: { Authorization: `Bearer ${backend.accessToken}` },
-        body: form,
-      });
-      assert.equal(
-        deployRes.status,
-        201,
-        `deploy failed: ${await deployRes.text()}`
-      );
-
-      const result = await scorer(scorerCtx(backend));
-      assert.equal(result.passed, true, checksMessage(result));
-    }
-  );
-
-  console.log('PASS edge function auth + supabase-js DB scorer');
-}
-
-async function smokeLogsSeeding() {
-  await withBackend(
-    { logsSeedJsonl: seedPath(INVESTIGATE_LOGS_EVAL, 'logs.jsonl') },
-    async ({ url, ref, accessToken }) => {
-      const logsUrl = `${url}/v1/projects/${ref}/analytics/endpoints/logs.all?sql=${encodeURIComponent('SELECT count(*)::int AS n FROM edge_logs')}`;
-      const res = await fetch(logsUrl, {
-        headers: { Authorization: `Bearer ${accessToken}` },
-      });
-      const body = (await res.json()) as { result: Array<{ n: number }> };
-      assert(body.result[0] && body.result[0].n > 0, 'expected seeded logs');
-    }
-  );
-
-  console.log('PASS logs seeding via platform-lite');
-}
-
-async function smokeInvestigateLogsEval() {
-  const scorer = await loadScorer(INVESTIGATE_LOGS_EVAL);
-
-  await withBackend(
-    { logsSeedJsonl: seedPath(INVESTIGATE_LOGS_EVAL, 'logs.jsonl') },
-    async (backend) => {
-      const result = await scorer(
-        scorerCtx(backend, {
-          agentReport:
-            'stripe-webhook had the most errors with 9 errors out of 50 events.',
-        })
-      );
-      assert.equal(result.passed, true, checksMessage(result));
-    }
-  );
-
-  console.log('PASS investigate logs scorer');
-}
-
-async function smokeInvestigateSecurityEval() {
-  const scorer = await loadScorer(INVESTIGATE_SECURITY_EVAL);
-
-  await withBackend(
-    {
-      projectSeedSql: seedPath(INVESTIGATE_SECURITY_EVAL, 'project.sql'),
-      logsSeedJsonl: seedPath(INVESTIGATE_SECURITY_EVAL, 'logs.jsonl'),
-    },
-    async (backend) => {
-      const { rows } = await backend.query(`
-SELECT grantee FROM information_schema.role_table_grants
-WHERE table_name = 'customer_payment_methods' AND privilege_type = 'SELECT'
-ORDER BY grantee;
-      `);
-      assert(
-        (rows as Array<{ grantee: string }>).some(
-          (row) => row.grantee === 'anon'
-        )
-      );
-
-      const report = [
-        'customer_payment_methods is exposed to anon.',
-        'Fix by REVOKE SELECT ON customer_payment_methods FROM anon and enable row level security.',
-      ].join(' ');
-
-      // The judge reads the transcript, so surface the report as assistant text.
-      const transcript: TranscriptPart[] = [
-        { type: 'message', role: 'assistant', content: report },
-      ];
-
-      const result = await scorer(
-        scorerCtx(backend, { agentReport: report, transcript })
-      );
-      assert.equal(result.passed, true, checksMessage(result));
-    }
-  );
-
-  console.log('PASS investigate security scorer + database shim');
-}
-
-async function smokeFrontendBuildTooling() {
-  const source = join(ROOT, FRONTEND_EVAL, 'local');
-  const workspace = join(
-    ROOT,
-    'results',
-    '_smoke',
-    'build-frontend-001-todos-app'
-  );
-  rmSync(workspace, { recursive: true, force: true });
-  mkdirSync(dirname(workspace), { recursive: true });
-  cpSync(source, workspace, {
-    recursive: true,
-    filter: (src) => !src.endsWith('/EVAL.ts'),
-  });
-  cpSync(join(ROOT, FRONTEND_EVAL, 'tests'), join(workspace, 'tests'), {
-    recursive: true,
-  });
-  writeFileSync(
-    join(workspace, '.env.local'),
-    [
-      'VITE_SUPABASE_URL=http://supabase-evals.local',
-      'VITE_SUPABASE_ANON_KEY=supabase-evals-anon-key',
-      '',
-    ].join('\n')
-  );
-  writeFileSync(join(workspace, 'src', 'App.tsx'), GOOD_FRONTEND_APP);
-
-  const build = await viteBuild(workspace);
-  assert.equal(build.ok, true, build.stderr || build.stdout);
-  const vitest = await vitestRun(workspace);
-  assert.equal(vitest.ok, true, vitest.stderr || vitest.stdout);
-
-  console.log('PASS frontend vite/react/supalite build + test tooling');
-}
-
-async function main() {
-  await smokeClientRlsEval();
-  await smokeFunctionsEval();
-  await smokeServiceRoleBypassEval();
-  await smokeSupaliteClient();
-  await smokePlatformBackendClose();
-  await smokeEdgeAuthDbEval();
-  await smokeLogsSeeding();
-  await smokeInvestigateLogsEval();
-  await smokeInvestigateSecurityEval();
-  await smokeFrontendBuildTooling();
-  console.log('PASS framework smoke');
-}
-
-main().catch((error) => {
-  console.error(error);
-  process.exit(1);
-});
-
-const ORDER_TOTAL_SOURCE = `
-Deno.serve(async (req) => {
-  const json = (payload, status = 200) =>
-    new Response(JSON.stringify(payload), { status, headers: { "content-type": "application/json" } });
-
-  if (req.method !== "POST") return json({ error: "method not allowed" }, 405);
-
-  let body;
-  try { body = await req.json(); } catch { return json({ error: "invalid json" }, 400); }
-  if (!Array.isArray(body.items) || body.items.length === 0) return json({ error: "items are required" }, 400);
-
-  let subtotal = 0;
-  for (const item of body.items) {
-    if (!Number.isFinite(item.unit_price_cents) || !Number.isFinite(item.quantity) || item.unit_price_cents <= 0 || item.quantity <= 0) {
-      return json({ error: "invalid item" }, 400);
-    }
-    subtotal += item.unit_price_cents * item.quantity;
-  }
-
-  const couponDiscount = body.coupon === "WELCOME10" ? Math.min(Math.round(subtotal * 0.1), 2000) : 0;
-  const enterpriseDiscount = body.customer_tier === "enterprise" ? Math.round(subtotal * 0.15) : 0;
-  const discount = Math.max(couponDiscount, enterpriseDiscount);
-  const taxable = subtotal - discount;
-  const tax = Math.round(taxable * 0.0725);
-
-  return json({ subtotal_cents: subtotal, discount_cents: discount, tax_cents: tax, total_cents: taxable + tax });
-});
-`;
-
-const TODO_CREATE_SOURCE = `
-import { createClient } from "@supabase/supabase-js";
-
-Deno.serve(async (req) => {
-  const json = (payload, status = 200) =>
-    new Response(JSON.stringify(payload), {
-      status,
-      headers: { "content-type": "application/json" },
-    });
-
-  if (req.method !== "POST") return json({ error: "method not allowed" }, 405);
-
-  const authorization = req.headers.get("authorization");
-  if (!authorization) return json({ error: "missing authorization" }, 401);
-
-  let body;
-  try { body = await req.json(); } catch { return json({ error: "invalid json" }, 400); }
-  if (typeof body.body !== "string" || body.body.trim() === "") return json({ error: "body is required" }, 400);
-
-  const supabase = createClient(
-    Deno.env.get("SUPABASE_URL"),
-    Deno.env.get("SUPABASE_ANON_KEY"),
-    { global: { headers: { Authorization: authorization } } }
-  );
-
-  const { data, error } = await supabase
-    .from("todos")
-    .insert({ body: body.body })
-    .select("id,body,user_id")
-    .single();
-
-  if (error) return json({ error: error.message }, 400);
-  return json(data, 201);
-});
-`;
-
-const GOOD_FRONTEND_APP = `
-import { useState } from "react";
-import { createClient } from "@supabase/supabase-js";
-
-type Todo = { id: string; body: string; done: boolean };
-
-export const supabase = createClient(
-  import.meta.env.VITE_SUPABASE_URL,
-  import.meta.env.VITE_SUPABASE_ANON_KEY
-);
-
-export default function App() {
-  const [email, setEmail] = useState("");
-  const [password, setPassword] = useState("");
-  const [newTodo, setNewTodo] = useState("");
-  const [signedIn, setSignedIn] = useState(false);
-  const [todos, setTodos] = useState<Todo[]>([]);
-  const [error, setError] = useState("");
-
-  async function loadTodos() {
-    const { data, error } = await supabase.from("todos").select("id,body,done").order("created_at", { ascending: true });
-    if (error) throw error;
-    setTodos(data ?? []);
-  }
-
-  async function handleSignIn(event: React.FormEvent) {
-    event.preventDefault();
-    setError("");
-    const { error } = await supabase.auth.signInWithPassword({ email, password });
-    if (error) { setError(error.message); return; }
-    setSignedIn(true);
-    await loadTodos();
-  }
-
-  async function handleAddTodo(event: React.FormEvent) {
-    event.preventDefault();
-    const body = newTodo.trim();
-    if (!body) return;
-    setError("");
-    const { data, error } = await supabase.from("todos").insert({ body }).select("id,body,done").single();
-    if (error) { setError(error.message); return; }
-    setTodos((current) => [...current, data]);
-    setNewTodo("");
-  }
-
-  async function handleToggleTodo(todo: Todo) {
-    setError("");
-    const { data, error } = await supabase.from("todos").update({ done: !todo.done }).eq("id", todo.id).select("id,body,done").single();
-    if (error) { setError(error.message); return; }
-    setTodos((current) => current.map((item) => (item.id === todo.id ? data : item)));
-  }
-
-  return (
-    <main>
-      <h1>Todos</h1>
-      <form onSubmit={handleSignIn}>
-        <input data-testid="email-input" placeholder="Email" value={email} onChange={(e) => setEmail(e.target.value)} />
-        <input data-testid="password-input" placeholder="Password" type="password" value={password} onChange={(e) => setPassword(e.target.value)} />
-        <button data-testid="sign-in-button" type="submit">Sign in</button>
-      </form>
-      {signedIn ? <p data-testid="signed-in">Signed in</p> : null}
-      {error ? <p role="alert">{error}</p> : null}
-      <form onSubmit={handleAddTodo}>
-        <input data-testid="todo-input" placeholder="New todo" value={newTodo} onChange={(e) => setNewTodo(e.target.value)} />
-        <button data-testid="add-button" type="submit">Add</button>
-      </form>
-      <ul data-testid="todo-list">
-        {todos.map((todo) => (
-          <li key={todo.id}>
-            <label>
-              <input data-testid={\`todo-checkbox-\${todo.body}\`} type="checkbox" checked={todo.done} onChange={() => void handleToggleTodo(todo)} />
-              {todo.body}
-            </label>
-          </li>
-        ))}
-      </ul>
-    </main>
-  );
-}
-`;
diff --git a/apps/framework/tsconfig.json b/apps/framework/tsconfig.json
index 0b21c008..4a852699 100644
--- a/apps/framework/tsconfig.json
+++ b/apps/framework/tsconfig.json
@@ -5,6 +5,7 @@
     "harness",
     "shims",
     "scripts",
-    "../../evals/**/EVAL.ts"
+    "../../evals/**/EVAL.ts",
+    "../../evals/*/EVAL.test.ts"
   ]
 }
diff --git a/apps/framework/vitest.config.ts b/apps/framework/vitest.config.ts
new file mode 100644
index 00000000..cf955168
--- /dev/null
+++ b/apps/framework/vitest.config.ts
@@ -0,0 +1,22 @@
+import { defineConfig } from 'vitest/config';
+
+export default defineConfig({
+  test: {
+    environment: 'node',
+    // Booting platform-lite and shelling out to vite/vitest is slow.
+    testTimeout: 120000,
+    hookTimeout: 120000,
+    include: [
+      'harness/**/*.test.ts',
+      // Scorer tests only. Deliberately not `evals/**/*.test.ts`: several evals
+      // ship withheld tests under `evals/<id>/tests/` that exist to score an
+      // agent's workspace, and must never run as part of this suite.
+      '../../evals/*/EVAL.test.ts',
+    ],
+    // Scorers make expected-fail supabase-js calls that log through
+    // console.error. Vitest reports failures through its own reporter, so
+    // dropping this noise cannot hide a failing test.
+    onConsoleLog: (_log, type) =>
+      type === 'stderr' && !process.env.DEBUG_TESTS ? false : undefined,
+  },
+});
diff --git a/evals/build-functions-001-order-total/EVAL.test.ts b/evals/build-functions-001-order-total/EVAL.test.ts
new file mode 100644
index 00000000..8362baa6
--- /dev/null
+++ b/evals/build-functions-001-order-total/EVAL.test.ts
@@ -0,0 +1,52 @@
+import { expect, test } from 'vitest';
+import {
+  checksMessage,
+  deployFunction,
+  scorerCtx,
+  withBackend,
+} from '../../apps/framework/harness/scorer-test-kit.js';
+import scorer from './EVAL.js';
+
+const ORDER_TOTAL_SOURCE = `
+Deno.serve(async (req) => {
+  const json = (payload, status = 200) =>
+    new Response(JSON.stringify(payload), { status, headers: { "content-type": "application/json" } });
+
+  if (req.method !== "POST") return json({ error: "method not allowed" }, 405);
+
+  let body;
+  try { body = await req.json(); } catch { return json({ error: "invalid json" }, 400); }
+  if (!Array.isArray(body.items) || body.items.length === 0) return json({ error: "items are required" }, 400);
+
+  let subtotal = 0;
+  for (const item of body.items) {
+    if (!Number.isFinite(item.unit_price_cents) || !Number.isFinite(item.quantity) || item.unit_price_cents <= 0 || item.quantity <= 0) {
+      return json({ error: "invalid item" }, 400);
+    }
+    subtotal += item.unit_price_cents * item.quantity;
+  }
+
+  const couponDiscount = body.coupon === "WELCOME10" ? Math.min(Math.round(subtotal * 0.1), 2000) : 0;
+  const enterpriseDiscount = body.customer_tier === "enterprise" ? Math.round(subtotal * 0.15) : 0;
+  const discount = Math.max(couponDiscount, enterpriseDiscount);
+  const taxable = subtotal - discount;
+  const tax = Math.round(taxable * 0.0725);
+
+  return json({ subtotal_cents: subtotal, discount_cents: discount, tax_cents: tax, total_cents: taxable + tax });
+});
+`;
+
+test('fails before the function exists, passes once it is deployed', async () => {
+  await withBackend({}, async (backend) => {
+    const before = await scorer(scorerCtx(backend));
+    expect(before.passed).toBe(false);
+    expect(checksMessage(before)).toMatch(/function not found/i);
+
+    await deployFunction(backend, 'order-total', ORDER_TOTAL_SOURCE, {
+      verifyJwt: false,
+    });
+
+    const after = await scorer(scorerCtx(backend));
+    expect(after.passed, checksMessage(after)).toBe(true);
+  });
+});
diff --git a/evals/build-functions-002-edge-auth-db/EVAL.test.ts b/evals/build-functions-002-edge-auth-db/EVAL.test.ts
new file mode 100644
index 00000000..e885cd73
--- /dev/null
+++ b/evals/build-functions-002-edge-auth-db/EVAL.test.ts
@@ -0,0 +1,61 @@
+import { expect, test } from 'vitest';
+import {
+  checksMessage,
+  deployFunction,
+  scorerCtx,
+  seedPath,
+  withBackend,
+} from '../../apps/framework/harness/scorer-test-kit.js';
+import scorer from './EVAL.js';
+
+const EVAL_DIR = 'evals/build-functions-002-edge-auth-db';
+
+const TODO_CREATE_SOURCE = `
+import { createClient } from "@supabase/supabase-js";
+
+Deno.serve(async (req) => {
+  const json = (payload, status = 200) =>
+    new Response(JSON.stringify(payload), {
+      status,
+      headers: { "content-type": "application/json" },
+    });
+
+  if (req.method !== "POST") return json({ error: "method not allowed" }, 405);
+
+  const authorization = req.headers.get("authorization");
+  if (!authorization) return json({ error: "missing authorization" }, 401);
+
+  let body;
+  try { body = await req.json(); } catch { return json({ error: "invalid json" }, 400); }
+  if (typeof body.body !== "string" || body.body.trim() === "") return json({ error: "body is required" }, 400);
+
+  const supabase = createClient(
+    Deno.env.get("SUPABASE_URL"),
+    Deno.env.get("SUPABASE_ANON_KEY"),
+    { global: { headers: { Authorization: authorization } } }
+  );
+
+  const { data, error } = await supabase
+    .from("todos")
+    .insert({ body: body.body })
+    .select("id,body,user_id")
+    .single();
+
+  if (error) return json({ error: error.message }, 400);
+  return json(data, 201);
+});
+`;
+
+test('passes for a JWT-verifying function that inserts as the caller', async () => {
+  await withBackend(
+    { projectSeedSql: seedPath(EVAL_DIR, 'project.sql') },
+    async (backend) => {
+      await deployFunction(backend, 'todo-create', TODO_CREATE_SOURCE, {
+        verifyJwt: true,
+      });
+
+      const result = await scorer(scorerCtx(backend));
+      expect(result.passed, checksMessage(result)).toBe(true);
+    }
+  );
+});
diff --git a/evals/build-functions-004-service-role-bypass/EVAL.test.ts b/evals/build-functions-004-service-role-bypass/EVAL.test.ts
new file mode 100644
index 00000000..695cf7ca
--- /dev/null
+++ b/evals/build-functions-004-service-role-bypass/EVAL.test.ts
@@ -0,0 +1,141 @@
+import { describe, expect, test } from 'vitest';
+import {
+  checksMessage,
+  failedCheckNames,
+  functionResponse,
+} from '../../apps/framework/harness/scorer-test-kit.js';
+import type {
+  EdgeFunctionsInvokeResult,
+  ToolEvalContext,
+} from '../../apps/framework/harness/types.js';
+import scorer from './EVAL.js';
+
+/**
+ * Unlike the other scorer tests, this one fakes the `ToolEvalContext` instead
+ * of booting a real backend: the scorer's *decision logic* (which statuses and
+ * bodies pass) is what we want to pin, and driving that through a real stack
+ * would mean deploying six edge-function variants.
+ */
+function fakeCtx(responses: EdgeFunctionsInvokeResult[]): ToolEvalContext {
+  const authResult = (id: string, accessToken: string) => ({
+    data: { user: { id }, session: { access_token: accessToken } },
+    error: null,
+  });
+  const clientA = {
+    auth: { signUp: async () => authResult('user-a', 'token-a') },
+  } as unknown as ToolEvalContext['client'];
+  const clientB = {
+    auth: { signUp: async () => authResult('user-b', 'token-b') },
+  } as unknown as ToolEvalContext['client'];
+
+  return {
+    mgmt: {} as ToolEvalContext['mgmt'],
+    ref: 'test-ref',
+    client: clientA,
+    getClient: () => clientB,
+    query: async () => ({ rows: [] }),
+    invokeFunction: async () => {
+      const response = responses.shift();
+      if (!response) throw new Error('missing fake function response');
+      return response;
+    },
+    toolCalls: [],
+    transcript: [],
+  };
+}
+
+type Role = 'missingAuth' | 'ownNotes' | 'aRequestsB' | 'bRequestsA';
+
+/**
+ * Responses are consumed in the scorer's invocation order (missingAuth,
+ * ownNotes, aRequestsB, bRequestsA); overrides are keyed by role.
+ */
+function runScorer(
+  overrides: Partial<Record<Role, EdgeFunctionsInvokeResult>> = {}
+) {
+  return scorer(
+    fakeCtx([
+      overrides.missingAuth ?? functionResponse(401),
+      overrides.ownNotes ??
+        functionResponse(200, 'user A private note', ['token-a']),
+      overrides.aRequestsB ?? functionResponse(401, 'unauthorized'),
+      overrides.bRequestsA ?? functionResponse(403, 'forbidden'),
+    ])
+  );
+}
+
+describe('secure shapes the scorer must accept', () => {
+  test('rejects anonymous access, serves own note, denies cross-user reads', async () => {
+    const result = await runScorer();
+
+    expect(result.passed, checksMessage(result)).toBe(true);
+    expect(checksMessage(result)).toMatch(/"notes":"status=401"/);
+    expect(checksMessage(result)).toMatch(/"notes":"status=200"/);
+    expect(checksMessage(result)).toMatch(/"notes":"status=403"/);
+    expect(checksMessage(result)).toMatch(
+      /"notes":"bearer_tokens=1, all_match=true"/
+    );
+  });
+
+  test('ignoring the spoofed user_id and returning the caller their own note', async () => {
+    const result = await runScorer({
+      aRequestsB: functionResponse(200, 'user A private note', ['token-a']),
+      bRequestsA: functionResponse(200, 'user B private note', ['token-b']),
+    });
+
+    expect(result.passed, checksMessage(result)).toBe(true);
+  });
+
+  test('404 from an RLS-scoped read that returns no rows', async () => {
+    const result = await runScorer({
+      aRequestsB: functionResponse(404, 'not found'),
+      bRequestsA: functionResponse(404, 'not found'),
+    });
+
+    expect(result.passed, checksMessage(result)).toBe(true);
+  });
+});
+
+describe('leaks and bypasses the scorer must reject', () => {
+  test('a leaked note fails regardless of status', async () => {
+    const result = await runScorer({
+      aRequestsB: functionResponse(200, 'user B private note'),
+      bRequestsA: functionResponse(403, 'user A private note'),
+    });
+
+    expect(result.passed, checksMessage(result)).toBe(false);
+    expect(failedCheckNames(result)).toEqual([
+      'user A cannot force-read user B note',
+      'user B cannot force-read user A note',
+    ]);
+  });
+
+  test('serving data to an unauthenticated caller fails without any leak', async () => {
+    const result = await runScorer({ missingAuth: functionResponse(200) });
+
+    expect(result.passed, checksMessage(result)).toBe(false);
+    expect(failedCheckNames(result)).toEqual(['rejects missing auth']);
+  });
+
+  test('a 401 that still echoes a note is a leak, not a denial', async () => {
+    const result = await runScorer({
+      missingAuth: functionResponse(401, 'user A private note'),
+    });
+
+    expect(result.passed, checksMessage(result)).toBe(false);
+    expect(failedCheckNames(result)).toEqual(['rejects missing auth']);
+  });
+
+  test('reading via the service-role key instead of the caller JWT fails', async () => {
+    const result = await runScorer({
+      ownNotes: functionResponse(200, 'user A private note', [
+        'service-role-key',
+      ]),
+    });
+
+    expect(result.passed, checksMessage(result)).toBe(false);
+    expect(failedCheckNames(result)).toEqual([
+      "reads only with the caller's JWT",
+    ]);
+  });
+});
diff --git a/evals/build-rls-002-own-todos-client/EVAL.test.ts b/evals/build-rls-002-own-todos-client/EVAL.test.ts
new file mode 100644
index 00000000..35c05938
--- /dev/null
+++ b/evals/build-rls-002-own-todos-client/EVAL.test.ts
@@ -0,0 +1,29 @@
+import { expect, test } from 'vitest';
+import {
+  checksMessage,
+  scorerCtx,
+  seedPath,
+  withBackend,
+} from '../../apps/framework/harness/scorer-test-kit.js';
+import scorer from './EVAL.js';
+
+const EVAL_DIR = 'evals/build-rls-002-own-todos-client';
+
+test('passes once per-user RLS policies are in place', async () => {
+  await withBackend(
+    { projectSeedSql: seedPath(EVAL_DIR, 'project.sql') },
+    async (backend) => {
+      await backend.query(`
+ALTER TABLE todos ENABLE ROW LEVEL SECURITY;
+
+CREATE POLICY "users can read own todos" ON todos FOR SELECT TO authenticated USING (user_id = auth.uid());
+CREATE POLICY "users can insert own todos" ON todos FOR INSERT TO authenticated WITH CHECK (user_id = auth.uid());
+CREATE POLICY "users can update own todos" ON todos FOR UPDATE TO authenticated USING (user_id = auth.uid()) WITH CHECK (user_id = auth.uid());
+CREATE POLICY "users can delete own todos" ON todos FOR DELETE TO authenticated USING (user_id = auth.uid());
+      `);
+
+      const result = await scorer(scorerCtx(backend));
+      expect(result.passed, checksMessage(result)).toBe(true);
+    }
+  );
+});
diff --git a/evals/investigate-logs-001-top-error-function/EVAL.test.ts b/evals/investigate-logs-001-top-error-function/EVAL.test.ts
new file mode 100644
index 00000000..fc0d69b5
--- /dev/null
+++ b/evals/investigate-logs-001-top-error-function/EVAL.test.ts
@@ -0,0 +1,26 @@
+import { expect, test } from 'vitest';
+import {
+  checksMessage,
+  scorerCtx,
+  seedPath,
+  withBackend,
+} from '../../apps/framework/harness/scorer-test-kit.js';
+import scorer from './EVAL.js';
+
+const EVAL_DIR = 'evals/investigate-logs-001-top-error-function';
+
+test('passes for a report naming the top error function and its count', async () => {
+  await withBackend(
+    { logsSeedJsonl: seedPath(EVAL_DIR, 'logs.jsonl') },
+    async (backend) => {
+      const result = await scorer(
+        scorerCtx(backend, {
+          agentReport:
+            'stripe-webhook had the most errors with 9 errors out of 50 events.',
+        })
+      );
+
+      expect(result.passed, checksMessage(result)).toBe(true);
+    }
+  );
+});
diff --git a/evals/investigate-security-001-public-table/EVAL.test.ts b/evals/investigate-security-001-public-table/EVAL.test.ts
new file mode 100644
index 00000000..d5399fcc
--- /dev/null
+++ b/evals/investigate-security-001-public-table/EVAL.test.ts
@@ -0,0 +1,84 @@
+import { judge } from '@supabase-evals/core';
+import type * as Core from '@supabase-evals/core';
+import { expect, test, vi } from 'vitest';
+import {
+  checksMessage,
+  failedCheckNames,
+  scorerCtx,
+  seedPath,
+  withBackend,
+} from '../../apps/framework/harness/scorer-test-kit.js';
+import type { TranscriptPart } from '../../apps/framework/harness/types.js';
+import scorer from './EVAL.js';
+
+// The scorer's fourth check calls a real OpenAI judge, which needs
+// OPENAI_API_KEY and bills per run. Stub only `judge` so this suite stays
+// hermetic in CI; every other core export the scorer uses stays real, so the
+// deterministic checks are still exercised for real.
+vi.mock('@supabase-evals/core', async (importOriginal) => ({
+  ...(await importOriginal<typeof Core>()),
+  judge: vi.fn(async () => ({ passed: true, notes: 'stubbed verdict' })),
+}));
+
+const EVAL_DIR = 'evals/investigate-security-001-public-table';
+
+const GOOD_REPORT = [
+  'customer_payment_methods is exposed to anon.',
+  'Fix by REVOKE SELECT ON customer_payment_methods FROM anon and enable row level security.',
+].join(' ');
+
+const VAGUE_REPORT =
+  'Some tables look insecure, you should tighten permissions.';
+
+// The judge reads the transcript, so surface the report as assistant text.
+const asTranscript = (report: string): TranscriptPart[] => [
+  { type: 'message', role: 'assistant', content: report },
+];
+
+test('scores a report that names the exposure and the fix', async () => {
+  await withBackend(
+    {
+      projectSeedSql: seedPath(EVAL_DIR, 'project.sql'),
+      logsSeedJsonl: seedPath(EVAL_DIR, 'logs.jsonl'),
+    },
+    async (backend) => {
+      const { rows } = await backend.query(`
+SELECT grantee FROM information_schema.role_table_grants
+WHERE table_name = 'customer_payment_methods' AND privilege_type = 'SELECT'
+ORDER BY grantee;
+      `);
+      expect(
+        (rows as Array<{ grantee: string }>).some(
+          (row) => row.grantee === 'anon'
+        ),
+        'seed should expose customer_payment_methods to anon'
+      ).toBe(true);
+
+      const passing = await scorer(
+        scorerCtx(backend, {
+          agentReport: GOOD_REPORT,
+          transcript: asTranscript(GOOD_REPORT),
+        })
+      );
+      expect(passing.passed, checksMessage(passing)).toBe(true);
+
+      // The judge scores the transcript, not the report field.
+      expect(vi.mocked(judge).mock.calls[0]?.[0].input).toContain(
+        'customer_payment_methods'
+      );
+
+      const vague = await scorer(
+        scorerCtx(backend, {
+          agentReport: VAGUE_REPORT,
+          transcript: asTranscript(VAGUE_REPORT),
+        })
+      );
+      expect(vague.passed).toBe(false);
+      expect(failedCheckNames(vague)).toEqual([
+        'named the vulnerable table',
+        'mentioned the anon role',
+        'proposed a concrete fix',
+      ]);
+    }
+  );
+});
diff --git a/package.json b/package.json
index ab7f505e..6a3ac94f 100644
--- a/package.json
+++ b/package.json
@@ -30,7 +30,17 @@
     "@supabase-evals/sandbox": "workspace:*",
     "@types/common-tags": "^1.8.4",
     "@types/node": "catalog:",
-    "typescript": "catalog:"
+    "typescript": "catalog:",
+    "@supabase/lite": "catalog:",
+    "@supabase/supabase-js": "catalog:",
+    "@testing-library/jest-dom": "^6.9.1",
+    "@testing-library/react": "^16.3.2",
+    "@vitejs/plugin-react": "catalog:",
+    "happy-dom": "^20.9.0",
+    "react": "^19.2.5",
+    "react-dom": "^19.2.5",
+    "vite": "catalog:",
+    "vitest": "catalog:"
   },
   "engines": {
     "node": ">=22",
diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
index b7a51bfd..57b46d7b 100644
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
@@ -81,15 +81,45 @@ importers:
       '@supabase-evals/sandbox':
         specifier: workspace:*
         version: link:packages/sandbox
+      '@supabase/lite':
+        specifier: 'catalog:'
+        version: 0.7.1-next.3(@supabase/supabase-js@2.108.1)(hono@4.12.25)(vite@7.3.5(@types/node@22.19.20)(jiti@2.7.0)(lightningcss@1.32.0)(tsx@4.22.4))
+      '@supabase/supabase-js':
+        specifier: 'catalog:'
+        version: 2.108.1
+      '@testing-library/jest-dom':
+        specifier: ^6.9.1
+        version: 6.9.1
+      '@testing-library/react':
+        specifier: ^16.3.2
+        version: 16.3.2(@testing-library/dom@10.4.1)(@types/react-dom@19.2.3(@types/react@19.2.17))(@types/react@19.2.17)(react-dom@19.2.7(react@19.2.7))(react@19.2.7)
       '@types/common-tags':
         specifier: ^1.8.4
         version: 1.8.4
       '@types/node':
         specifier: 'catalog:'
         version: 22.19.20
+      '@vitejs/plugin-react':
+        specifier: 'catalog:'
+        version: 5.2.0(vite@7.3.5(@types/node@22.19.20)(jiti@2.7.0)(lightningcss@1.32.0)(tsx@4.22.4))
+      happy-dom:
+        specifier: ^20.9.0
+        version: 20.10.2
+      react:
+        specifier: ^19.2.5
+        version: 19.2.7
+      react-dom:
+        specifier: ^19.2.5
+        version: 19.2.7(react@19.2.7)
       typescript:
         specifier: 'catalog:'
         version: 5.9.3
+      vite:
+        specifier: 'catalog:'
+        version: 7.3.5(@types/node@22.19.20)(jiti@2.7.0)(lightningcss@1.32.0)(tsx@4.22.4)
+      vitest:
+        specifier: 'catalog:'
+        version: 4.1.8(@opentelemetry/api@1.9.1)(@types/node@22.19.20)(happy-dom@20.10.2)(vite@7.3.5(@types/node@22.19.20)(jiti@2.7.0)(lightningcss@1.32.0)(tsx@4.22.4))
 
   apps/framework:
     dependencies:
@@ -114,36 +144,9 @@ importers:
       '@supabase-evals/sandbox':
         specifier: workspace:*
         version: link:../../packages/sandbox
-      '@supabase/lite':
-        specifier: 'catalog:'
-        version: 0.7.1-next.3(@supabase/supabase-js@2.108.1)(hono@4.12.25)(vite@7.3.5(@types/node@22.19.20)(jiti@2.7.0)(lightningcss@1.32.0)(tsx@4.22.4))
-      '@supabase/supabase-js':
-        specifier: 'catalog:'
-        version: 2.108.1
-      '@testing-library/jest-dom':
-        specifier: ^6.9.1
-        version: 6.9.1
-      '@testing-library/react':
-        specifier: ^16.3.2
-        version: 16.3.2(@testing-library/dom@10.4.1)(@types/react-dom@19.2.3(@types/react@19.2.17))(@types/react@19.2.17)(react-dom@19.2.7(react@19.2.7))(react@19.2.7)
-      '@vitejs/plugin-react':
-        specifier: 'catalog:'
-        version: 5.2.0(vite@7.3.5(@types/node@22.19.20)(jiti@2.7.0)(lightningcss@1.32.0)(tsx@4.22.4))
       ai:
         specifier: 'catalog:'
         version: 6.0.199(zod@4.4.3)
-      happy-dom:
-        specifier: ^20.9.0
-        version: 20.10.2
-      react:
-        specifier: ^19.2.5
-        version: 19.2.7
-      react-dom:
-        specifier: ^19.2.5
-        version: 19.2.7(react@19.2.7)
-      vite:
-        specifier: 'catalog:'
-        version: 7.3.5(@types/node@22.19.20)(jiti@2.7.0)(lightningcss@1.32.0)(tsx@4.22.4)
       vitest:
         specifier: 'catalog:'
         version: 4.1.8(@opentelemetry/api@1.9.1)(@types/node@22.19.20)(happy-dom@20.10.2)(vite@7.3.5(@types/node@22.19.20)(jiti@2.7.0)(lightningcss@1.32.0)(tsx@4.22.4))

From 3d3bda6bdbdcc0688c33cb4fdba13269b41e92c6 Mon Sep 17 00:00:00 2001
From: Barry Roodt <barry.roodt@supabase.io>
Date: Tue, 28 Jul 2026 16:03:18 +0200
Subject: [PATCH 2/6] refactor(tests): move the scorer kit out of the framework
 internals

Review follow-ups on the Vitest migration.

Eval tests were importing `../../apps/framework/harness/{scorer-test-kit,types}.js`
while the EVAL.ts sitting next to them imported `@supabase-evals/core`: two
conventions for the same types, side by side, and six eval directories reaching
across a package boundary into an app's internals. The framework barrels they
reached for are re-exports anyway (platform-backend.ts is two lines).

The kit now lives at top-level `test-utils/`, not in core: it hardcodes the
monorepo layout (ROOT, seedPath) and is test-only, so a production package
should not own it. Canonical types come straight from core.

Also drops the `onConsoleLog` stderr filter. It was carried over reflexively
from the old script's global console.error stub, and never measured: Vitest
attributes console output per test, and the suite runs clean without it.
Suppressing output was the very thing that made the old script's failures
invisible, so it does not get reintroduced without evidence.

The frontend reference solution stops being an 80-line template string and
becomes evals/build-frontend-001-todos-app/reference/App.tsx, genuinely
typechecked via its own tsconfig extending the eval's jsx-capable one, plus
@types/react at the root. It stays outside local/ so the agent never receives
it. Note tsc cannot catch data-testid drift, only API misuse; the withheld
tests remain what proves the testids still line up.
---
 .github/workflows/biome.yml                   |   4 +-
 .../harness/platform-backend.test.ts          |   2 +-
 apps/framework/harness/project-runner.test.ts |  96 ++----------
 apps/framework/tsconfig.json                  |   3 +-
 apps/framework/vitest.config.ts               |   5 -
 .../reference/App.tsx                         | 138 ++++++++++++++++++
 .../reference/tsconfig.json                   |   7 +
 .../EVAL.test.ts                              |   2 +-
 .../EVAL.test.ts                              |   2 +-
 .../EVAL.test.ts                              |   4 +-
 .../EVAL.test.ts                              |   2 +-
 .../EVAL.test.ts                              |   2 +-
 .../EVAL.test.ts                              |   4 +-
 package.json                                  |   3 +-
 pnpm-lock.yaml                                |   3 +
 .../harness => test-utils}/scorer-test-kit.ts |  25 ++--
 16 files changed, 190 insertions(+), 112 deletions(-)
 create mode 100644 evals/build-frontend-001-todos-app/reference/App.tsx
 create mode 100644 evals/build-frontend-001-todos-app/reference/tsconfig.json
 rename {apps/framework/harness => test-utils}/scorer-test-kit.ts (81%)

diff --git a/.github/workflows/biome.yml b/.github/workflows/biome.yml
index e49bc2a7..af081863 100644
--- a/.github/workflows/biome.yml
+++ b/.github/workflows/biome.yml
@@ -52,5 +52,7 @@ jobs:
 
       # Typechecks plus the Vitest suites. No API keys: every test that would
       # reach a model stubs it, so this must never need credentials or spend.
+      # `typecheck` covers the web app and the frontend reference solution,
+      # which `check` does not reach.
       - name: Run checks
-        run: pnpm check
+        run: pnpm typecheck && pnpm check
diff --git a/apps/framework/harness/platform-backend.test.ts b/apps/framework/harness/platform-backend.test.ts
index 6b45fd9f..9a00ea40 100644
--- a/apps/framework/harness/platform-backend.test.ts
+++ b/apps/framework/harness/platform-backend.test.ts
@@ -1,6 +1,6 @@
 import { expect, test } from 'vitest';
 import { bootPlatformBackend } from './platform-backend.js';
-import { seedPath, withBackend } from './scorer-test-kit.js';
+import { seedPath, withBackend } from '../../../test-utils/scorer-test-kit.js';
 
 const LOGS_EVAL = 'evals/investigate-logs-001-top-error-function';
 
diff --git a/apps/framework/harness/project-runner.test.ts b/apps/framework/harness/project-runner.test.ts
index 7897806a..1568d1a3 100644
--- a/apps/framework/harness/project-runner.test.ts
+++ b/apps/framework/harness/project-runner.test.ts
@@ -1,91 +1,21 @@
-import { cpSync, existsSync, mkdirSync, rmSync, writeFileSync } from 'node:fs';
+import {
+  cpSync,
+  existsSync,
+  mkdirSync,
+  readFileSync,
+  rmSync,
+  writeFileSync,
+} from 'node:fs';
 import { dirname, join } from 'node:path';
 import { expect, test } from 'vitest';
 import { resolvePackageBin, viteBuild, vitestRun } from './project-runner.js';
-import { ROOT } from './scorer-test-kit.js';
+import { ROOT } from '../../../test-utils/scorer-test-kit.js';
 
 const FRONTEND_EVAL = 'evals/build-frontend-001-todos-app';
 
-const GOOD_FRONTEND_APP = `
-import { useState } from "react";
-import { createClient } from "@supabase/supabase-js";
-
-type Todo = { id: string; body: string; done: boolean };
-
-export const supabase = createClient(
-  import.meta.env.VITE_SUPABASE_URL,
-  import.meta.env.VITE_SUPABASE_ANON_KEY
-);
-
-export default function App() {
-  const [email, setEmail] = useState("");
-  const [password, setPassword] = useState("");
-  const [newTodo, setNewTodo] = useState("");
-  const [signedIn, setSignedIn] = useState(false);
-  const [todos, setTodos] = useState<Todo[]>([]);
-  const [error, setError] = useState("");
-
-  async function loadTodos() {
-    const { data, error } = await supabase.from("todos").select("id,body,done").order("created_at", { ascending: true });
-    if (error) throw error;
-    setTodos(data ?? []);
-  }
-
-  async function handleSignIn(event: React.FormEvent) {
-    event.preventDefault();
-    setError("");
-    const { error } = await supabase.auth.signInWithPassword({ email, password });
-    if (error) { setError(error.message); return; }
-    setSignedIn(true);
-    await loadTodos();
-  }
-
-  async function handleAddTodo(event: React.FormEvent) {
-    event.preventDefault();
-    const body = newTodo.trim();
-    if (!body) return;
-    setError("");
-    const { data, error } = await supabase.from("todos").insert({ body }).select("id,body,done").single();
-    if (error) { setError(error.message); return; }
-    setTodos((current) => [...current, data]);
-    setNewTodo("");
-  }
-
-  async function handleToggleTodo(todo: Todo) {
-    setError("");
-    const { data, error } = await supabase.from("todos").update({ done: !todo.done }).eq("id", todo.id).select("id,body,done").single();
-    if (error) { setError(error.message); return; }
-    setTodos((current) => current.map((item) => (item.id === todo.id ? data : item)));
-  }
-
-  return (
-    <main>
-      <h1>Todos</h1>
-      <form onSubmit={handleSignIn}>
-        <input data-testid="email-input" placeholder="Email" value={email} onChange={(e) => setEmail(e.target.value)} />
-        <input data-testid="password-input" placeholder="Password" type="password" value={password} onChange={(e) => setPassword(e.target.value)} />
-        <button data-testid="sign-in-button" type="submit">Sign in</button>
-      </form>
-      {signedIn ? <p data-testid="signed-in">Signed in</p> : null}
-      {error ? <p role="alert">{error}</p> : null}
-      <form onSubmit={handleAddTodo}>
-        <input data-testid="todo-input" placeholder="New todo" value={newTodo} onChange={(e) => setNewTodo(e.target.value)} />
-        <button data-testid="add-button" type="submit">Add</button>
-      </form>
-      <ul data-testid="todo-list">
-        {todos.map((todo) => (
-          <li key={todo.id}>
-            <label>
-              <input data-testid={\`todo-checkbox-\${todo.body}\`} type="checkbox" checked={todo.done} onChange={() => void handleToggleTodo(todo)} />
-              {todo.body}
-            </label>
-          </li>
-        ))}
-      </ul>
-    </main>
-  );
-}
-`;
+/** The eval's reference solution, kept as a real file so it typechecks. */
+const referenceApp = () =>
+  readFileSync(join(ROOT, FRONTEND_EVAL, 'reference', 'App.tsx'), 'utf8');
 
 // Regression guard for AI-975: pnpm's isolated layout has no hoisted
 // `<repo>/node_modules/<pkg>`, so a repo-root path missed every time and the
@@ -131,7 +61,7 @@ test('builds and tests a known-good frontend workspace', async () => {
       '',
     ].join('\n')
   );
-  writeFileSync(join(workspace, 'src', 'App.tsx'), GOOD_FRONTEND_APP);
+  writeFileSync(join(workspace, 'src', 'App.tsx'), referenceApp());
 
   const build = await viteBuild(workspace);
   expect(build.ok, build.stderr || build.stdout).toBe(true);
diff --git a/apps/framework/tsconfig.json b/apps/framework/tsconfig.json
index 4a852699..4a311453 100644
--- a/apps/framework/tsconfig.json
+++ b/apps/framework/tsconfig.json
@@ -6,6 +6,7 @@
     "shims",
     "scripts",
     "../../evals/**/EVAL.ts",
-    "../../evals/*/EVAL.test.ts"
+    "../../evals/*/EVAL.test.ts",
+    "../../test-utils"
   ]
 }
diff --git a/apps/framework/vitest.config.ts b/apps/framework/vitest.config.ts
index cf955168..28dfc644 100644
--- a/apps/framework/vitest.config.ts
+++ b/apps/framework/vitest.config.ts
@@ -13,10 +13,5 @@ export default defineConfig({
       // agent's workspace, and must never run as part of this suite.
       '../../evals/*/EVAL.test.ts',
     ],
-    // Scorers make expected-fail supabase-js calls that log through
-    // console.error. Vitest reports failures through its own reporter, so
-    // dropping this noise cannot hide a failing test.
-    onConsoleLog: (_log, type) =>
-      type === 'stderr' && !process.env.DEBUG_TESTS ? false : undefined,
   },
 });
diff --git a/evals/build-frontend-001-todos-app/reference/App.tsx b/evals/build-frontend-001-todos-app/reference/App.tsx
new file mode 100644
index 00000000..435b7820
--- /dev/null
+++ b/evals/build-frontend-001-todos-app/reference/App.tsx
@@ -0,0 +1,138 @@
+/**
+ * Reference solution for this eval, used by the harness's own test to prove
+ * `viteBuild` + `vitestRun` can build and pass a known-good workspace. It is
+ * NOT part of `local/` (that is the agent's starting point, and shipping this
+ * would hand over the answer) and it is never copied into a sandbox.
+ *
+ * It must keep satisfying the withheld tests in `../tests/`, which drive it by
+ * `data-testid`.
+ */
+import { useState } from 'react';
+import { createClient } from '@supabase/supabase-js';
+
+type Todo = { id: string; body: string; done: boolean };
+
+export const supabase = createClient(
+  import.meta.env.VITE_SUPABASE_URL,
+  import.meta.env.VITE_SUPABASE_ANON_KEY
+);
+
+export default function App() {
+  const [email, setEmail] = useState('');
+  const [password, setPassword] = useState('');
+  const [newTodo, setNewTodo] = useState('');
+  const [signedIn, setSignedIn] = useState(false);
+  const [todos, setTodos] = useState<Todo[]>([]);
+  const [error, setError] = useState('');
+
+  async function loadTodos() {
+    const { data, error } = await supabase
+      .from('todos')
+      .select('id,body,done')
+      .order('created_at', { ascending: true });
+    if (error) throw error;
+    setTodos(data ?? []);
+  }
+
+  async function handleSignIn(event: React.FormEvent) {
+    event.preventDefault();
+    setError('');
+    const { error } = await supabase.auth.signInWithPassword({
+      email,
+      password,
+    });
+    if (error) {
+      setError(error.message);
+      return;
+    }
+    setSignedIn(true);
+    await loadTodos();
+  }
+
+  async function handleAddTodo(event: React.FormEvent) {
+    event.preventDefault();
+    const body = newTodo.trim();
+    if (!body) return;
+    setError('');
+    const { data, error } = await supabase
+      .from('todos')
+      .insert({ body })
+      .select('id,body,done')
+      .single();
+    if (error) {
+      setError(error.message);
+      return;
+    }
+    setTodos((current) => [...current, data]);
+    setNewTodo('');
+  }
+
+  async function handleToggleTodo(todo: Todo) {
+    setError('');
+    const { data, error } = await supabase
+      .from('todos')
+      .update({ done: !todo.done })
+      .eq('id', todo.id)
+      .select('id,body,done')
+      .single();
+    if (error) {
+      setError(error.message);
+      return;
+    }
+    setTodos((current) =>
+      current.map((item) => (item.id === todo.id ? data : item))
+    );
+  }
+
+  return (
+    <main>
+      <h1>Todos</h1>
+      <form onSubmit={handleSignIn}>
+        <input
+          data-testid="email-input"
+          placeholder="Email"
+          value={email}
+          onChange={(e) => setEmail(e.target.value)}
+        />
+        <input
+          data-testid="password-input"
+          placeholder="Password"
+          type="password"
+          value={password}
+          onChange={(e) => setPassword(e.target.value)}
+        />
+        <button data-testid="sign-in-button" type="submit">
+          Sign in
+        </button>
+      </form>
+      {signedIn ? <p data-testid="signed-in">Signed in</p> : null}
+      {error ? <p role="alert">{error}</p> : null}
+      <form onSubmit={handleAddTodo}>
+        <input
+          data-testid="todo-input"
+          placeholder="New todo"
+          value={newTodo}
+          onChange={(e) => setNewTodo(e.target.value)}
+        />
+        <button data-testid="add-button" type="submit">
+          Add
+        </button>
+      </form>
+      <ul data-testid="todo-list">
+        {todos.map((todo) => (
+          <li key={todo.id}>
+            <label>
+              <input
+                data-testid={`todo-checkbox-${todo.body}`}
+                type="checkbox"
+                checked={todo.done}
+                onChange={() => void handleToggleTodo(todo)}
+              />
+              {todo.body}
+            </label>
+          </li>
+        ))}
+      </ul>
+    </main>
+  );
+}
diff --git a/evals/build-frontend-001-todos-app/reference/tsconfig.json b/evals/build-frontend-001-todos-app/reference/tsconfig.json
new file mode 100644
index 00000000..24e1a843
--- /dev/null
+++ b/evals/build-frontend-001-todos-app/reference/tsconfig.json
@@ -0,0 +1,7 @@
+{
+  "extends": "../local/tsconfig.json",
+  "compilerOptions": {
+    "types": ["vite/client"]
+  },
+  "include": ["."]
+}
diff --git a/evals/build-functions-001-order-total/EVAL.test.ts b/evals/build-functions-001-order-total/EVAL.test.ts
index 8362baa6..b3b64969 100644
--- a/evals/build-functions-001-order-total/EVAL.test.ts
+++ b/evals/build-functions-001-order-total/EVAL.test.ts
@@ -4,7 +4,7 @@ import {
   deployFunction,
   scorerCtx,
   withBackend,
-} from '../../apps/framework/harness/scorer-test-kit.js';
+} from '../../test-utils/scorer-test-kit.js';
 import scorer from './EVAL.js';
 
 const ORDER_TOTAL_SOURCE = `
diff --git a/evals/build-functions-002-edge-auth-db/EVAL.test.ts b/evals/build-functions-002-edge-auth-db/EVAL.test.ts
index e885cd73..59b30986 100644
--- a/evals/build-functions-002-edge-auth-db/EVAL.test.ts
+++ b/evals/build-functions-002-edge-auth-db/EVAL.test.ts
@@ -5,7 +5,7 @@ import {
   scorerCtx,
   seedPath,
   withBackend,
-} from '../../apps/framework/harness/scorer-test-kit.js';
+} from '../../test-utils/scorer-test-kit.js';
 import scorer from './EVAL.js';
 
 const EVAL_DIR = 'evals/build-functions-002-edge-auth-db';
diff --git a/evals/build-functions-004-service-role-bypass/EVAL.test.ts b/evals/build-functions-004-service-role-bypass/EVAL.test.ts
index 695cf7ca..19c2b04f 100644
--- a/evals/build-functions-004-service-role-bypass/EVAL.test.ts
+++ b/evals/build-functions-004-service-role-bypass/EVAL.test.ts
@@ -3,11 +3,11 @@ import {
   checksMessage,
   failedCheckNames,
   functionResponse,
-} from '../../apps/framework/harness/scorer-test-kit.js';
+} from '../../test-utils/scorer-test-kit.js';
 import type {
   EdgeFunctionsInvokeResult,
   ToolEvalContext,
-} from '../../apps/framework/harness/types.js';
+} from '@supabase-evals/core';
 import scorer from './EVAL.js';
 
 /**
diff --git a/evals/build-rls-002-own-todos-client/EVAL.test.ts b/evals/build-rls-002-own-todos-client/EVAL.test.ts
index 35c05938..8de4eadb 100644
--- a/evals/build-rls-002-own-todos-client/EVAL.test.ts
+++ b/evals/build-rls-002-own-todos-client/EVAL.test.ts
@@ -4,7 +4,7 @@ import {
   scorerCtx,
   seedPath,
   withBackend,
-} from '../../apps/framework/harness/scorer-test-kit.js';
+} from '../../test-utils/scorer-test-kit.js';
 import scorer from './EVAL.js';
 
 const EVAL_DIR = 'evals/build-rls-002-own-todos-client';
diff --git a/evals/investigate-logs-001-top-error-function/EVAL.test.ts b/evals/investigate-logs-001-top-error-function/EVAL.test.ts
index fc0d69b5..ebf60f36 100644
--- a/evals/investigate-logs-001-top-error-function/EVAL.test.ts
+++ b/evals/investigate-logs-001-top-error-function/EVAL.test.ts
@@ -4,7 +4,7 @@ import {
   scorerCtx,
   seedPath,
   withBackend,
-} from '../../apps/framework/harness/scorer-test-kit.js';
+} from '../../test-utils/scorer-test-kit.js';
 import scorer from './EVAL.js';
 
 const EVAL_DIR = 'evals/investigate-logs-001-top-error-function';
diff --git a/evals/investigate-security-001-public-table/EVAL.test.ts b/evals/investigate-security-001-public-table/EVAL.test.ts
index d5399fcc..e59f97f7 100644
--- a/evals/investigate-security-001-public-table/EVAL.test.ts
+++ b/evals/investigate-security-001-public-table/EVAL.test.ts
@@ -7,8 +7,8 @@ import {
   scorerCtx,
   seedPath,
   withBackend,
-} from '../../apps/framework/harness/scorer-test-kit.js';
-import type { TranscriptPart } from '../../apps/framework/harness/types.js';
+} from '../../test-utils/scorer-test-kit.js';
+import type { TranscriptPart } from '@supabase-evals/core';
 import scorer from './EVAL.js';
 
 // The scorer's fourth check calls a real OpenAI judge, which needs
diff --git a/package.json b/package.json
index 6a3ac94f..f089b339 100644
--- a/package.json
+++ b/package.json
@@ -11,7 +11,7 @@
     "eval:force": "pnpm --filter @supabase-evals/framework eval:force",
     "test:framework": "pnpm --filter @supabase-evals/framework test:framework",
     "export-results": "pnpm --filter @supabase-evals/framework export-results",
-    "typecheck": "pnpm --filter @supabase-evals/framework typecheck && pnpm --filter @supabase-evals/web typecheck",
+    "typecheck": "pnpm --filter @supabase-evals/framework typecheck && pnpm --filter @supabase-evals/web typecheck && tsc -p evals/build-frontend-001-todos-app/reference",
     "web": "pnpm --filter @supabase-evals/web dev",
     "web:build": "pnpm --filter @supabase-evals/web build",
     "demo:mcp": "pnpm --filter @supabase-evals/framework demo:mcp",
@@ -29,6 +29,7 @@
     "@supabase-evals/core": "workspace:*",
     "@supabase-evals/sandbox": "workspace:*",
     "@types/common-tags": "^1.8.4",
+    "@types/react": "^19.2.7",
     "@types/node": "catalog:",
     "typescript": "catalog:",
     "@supabase/lite": "catalog:",
diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
index 57b46d7b..4031581e 100644
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
@@ -99,6 +99,9 @@ importers:
       '@types/node':
         specifier: 'catalog:'
         version: 22.19.20
+      '@types/react':
+        specifier: ^19.2.7
+        version: 19.2.17
       '@vitejs/plugin-react':
         specifier: 'catalog:'
         version: 5.2.0(vite@7.3.5(@types/node@22.19.20)(jiti@2.7.0)(lightningcss@1.32.0)(tsx@4.22.4))
diff --git a/apps/framework/harness/scorer-test-kit.ts b/test-utils/scorer-test-kit.ts
similarity index 81%
rename from apps/framework/harness/scorer-test-kit.ts
rename to test-utils/scorer-test-kit.ts
index 5d7cc78e..d958d2f3 100644
--- a/apps/framework/harness/scorer-test-kit.ts
+++ b/test-utils/scorer-test-kit.ts
@@ -1,28 +1,29 @@
 /**
  * Helpers for testing eval scorers against a real platform-lite backend.
  *
+ * Lives at the repo root rather than inside a package on purpose: it hardcodes
+ * the monorepo layout (`ROOT`, `seedPath`) and is test-only, so shipping it
+ * from `@supabase-evals/core` would make a production package own repo-shaped
+ * test infrastructure. Canonical types come from core; nothing here reaches
+ * into an app's internals.
+ *
  * Scorer tests live next to the scorer they cover (`evals/<id>/EVAL.test.ts`)
  * and import it directly; harness tests live beside the harness module they
- * cover. Both import from here so a scorer test stays a description of the
- * scenario rather than a pile of backend plumbing.
+ * cover. Both import from here so a test stays a description of the scenario
+ * rather than a pile of backend plumbing.
  */
 import { dirname, join } from 'node:path';
 import { fileURLToPath } from 'node:url';
-import { bootPlatformBackend } from './platform-backend.js';
-import type { PlatformBackend } from './platform-backend.js';
+import { bootPlatformBackend } from '@supabase-evals/core';
 import type {
   EdgeFunctionsInvokeResult,
+  PlatformBackend,
   ToolEvalContext,
   TranscriptPart,
-} from './types.js';
+} from '@supabase-evals/core';
 
-/** Repo root, from `apps/framework/harness/`. */
-export const ROOT = join(
-  dirname(fileURLToPath(import.meta.url)),
-  '..',
-  '..',
-  '..'
-);
+/** Repo root, from `test-utils/`. */
+export const ROOT = join(dirname(fileURLToPath(import.meta.url)), '..');
 
 /** Path to a file in an eval's `remote/` seed directory. */
 export function seedPath(relDir: string, file: string): string {

From 5180fa3150a1752e3d397a1f8c0c36cc8b06bb5a Mon Sep 17 00:00:00 2001
From: Barry Roodt <barry.roodt@supabase.io>
Date: Tue, 28 Jul 2026 16:50:33 +0200
Subject: [PATCH 3/6] refactor(tests): derive eval seed paths from the test's
 own location

Each colocated scorer test hardcoded `EVAL_DIR = 'evals/<its own directory>'`,
restating what the file's path already says. Copying a test into a new eval and
missing that constant would seed a different eval's fixtures and score against
them, which does not throw and can pass: a wrong-but-valid path, in a suite
whose whole point is tests that do not lie.

seedPath now takes the caller's module URL, so a colocated test always seeds
its own scenario. platform-backend.test.ts genuinely borrows another eval's
logs fixture, so that one path stays spelled out to keep the exception visible.
---
 apps/framework/harness/platform-backend.test.ts      | 12 +++++++++---
 evals/build-functions-002-edge-auth-db/EVAL.test.ts  |  4 +---
 evals/build-rls-002-own-todos-client/EVAL.test.ts    |  4 +---
 .../EVAL.test.ts                                     |  4 +---
 .../EVAL.test.ts                                     |  6 ++----
 test-utils/scorer-test-kit.ts                        | 11 ++++++++---
 6 files changed, 22 insertions(+), 19 deletions(-)

diff --git a/apps/framework/harness/platform-backend.test.ts b/apps/framework/harness/platform-backend.test.ts
index 9a00ea40..22a33d14 100644
--- a/apps/framework/harness/platform-backend.test.ts
+++ b/apps/framework/harness/platform-backend.test.ts
@@ -1,8 +1,14 @@
+import { join } from 'node:path';
 import { expect, test } from 'vitest';
 import { bootPlatformBackend } from './platform-backend.js';
-import { seedPath, withBackend } from '../../../test-utils/scorer-test-kit.js';
+import { ROOT, withBackend } from '../../../test-utils/scorer-test-kit.js';
 
-const LOGS_EVAL = 'evals/investigate-logs-001-top-error-function';
+// Deliberately borrows another eval's fixture, so the path stays explicit
+// rather than going through the colocated `seedPath` helper.
+const LOGS_SEED = join(
+  ROOT,
+  'evals/investigate-logs-001-top-error-function/remote/logs.jsonl'
+);
 
 test('supalite auth issues a session supabase-js can write and read under RLS', async () => {
   await withBackend({}, async (backend) => {
@@ -54,7 +60,7 @@ test('close disposes the platform, and is idempotent', async () => {
 
 test('seeded logs are queryable over the analytics endpoint', async () => {
   await withBackend(
-    { logsSeedJsonl: seedPath(LOGS_EVAL, 'logs.jsonl') },
+    { logsSeedJsonl: LOGS_SEED },
     async ({ url, ref, accessToken }) => {
       const sql = 'SELECT count(*)::int AS n FROM edge_logs';
       const res = await fetch(
diff --git a/evals/build-functions-002-edge-auth-db/EVAL.test.ts b/evals/build-functions-002-edge-auth-db/EVAL.test.ts
index 59b30986..97d8937e 100644
--- a/evals/build-functions-002-edge-auth-db/EVAL.test.ts
+++ b/evals/build-functions-002-edge-auth-db/EVAL.test.ts
@@ -8,8 +8,6 @@ import {
 } from '../../test-utils/scorer-test-kit.js';
 import scorer from './EVAL.js';
 
-const EVAL_DIR = 'evals/build-functions-002-edge-auth-db';
-
 const TODO_CREATE_SOURCE = `
 import { createClient } from "@supabase/supabase-js";
 
@@ -48,7 +46,7 @@ Deno.serve(async (req) => {
 
 test('passes for a JWT-verifying function that inserts as the caller', async () => {
   await withBackend(
-    { projectSeedSql: seedPath(EVAL_DIR, 'project.sql') },
+    { projectSeedSql: seedPath(import.meta.url, 'project.sql') },
     async (backend) => {
       await deployFunction(backend, 'todo-create', TODO_CREATE_SOURCE, {
         verifyJwt: true,
diff --git a/evals/build-rls-002-own-todos-client/EVAL.test.ts b/evals/build-rls-002-own-todos-client/EVAL.test.ts
index 8de4eadb..7914e139 100644
--- a/evals/build-rls-002-own-todos-client/EVAL.test.ts
+++ b/evals/build-rls-002-own-todos-client/EVAL.test.ts
@@ -7,11 +7,9 @@ import {
 } from '../../test-utils/scorer-test-kit.js';
 import scorer from './EVAL.js';
 
-const EVAL_DIR = 'evals/build-rls-002-own-todos-client';
-
 test('passes once per-user RLS policies are in place', async () => {
   await withBackend(
-    { projectSeedSql: seedPath(EVAL_DIR, 'project.sql') },
+    { projectSeedSql: seedPath(import.meta.url, 'project.sql') },
     async (backend) => {
       await backend.query(`
 ALTER TABLE todos ENABLE ROW LEVEL SECURITY;
diff --git a/evals/investigate-logs-001-top-error-function/EVAL.test.ts b/evals/investigate-logs-001-top-error-function/EVAL.test.ts
index ebf60f36..86a0a7b0 100644
--- a/evals/investigate-logs-001-top-error-function/EVAL.test.ts
+++ b/evals/investigate-logs-001-top-error-function/EVAL.test.ts
@@ -7,11 +7,9 @@ import {
 } from '../../test-utils/scorer-test-kit.js';
 import scorer from './EVAL.js';
 
-const EVAL_DIR = 'evals/investigate-logs-001-top-error-function';
-
 test('passes for a report naming the top error function and its count', async () => {
   await withBackend(
-    { logsSeedJsonl: seedPath(EVAL_DIR, 'logs.jsonl') },
+    { logsSeedJsonl: seedPath(import.meta.url, 'logs.jsonl') },
     async (backend) => {
       const result = await scorer(
         scorerCtx(backend, {
diff --git a/evals/investigate-security-001-public-table/EVAL.test.ts b/evals/investigate-security-001-public-table/EVAL.test.ts
index e59f97f7..b458fdce 100644
--- a/evals/investigate-security-001-public-table/EVAL.test.ts
+++ b/evals/investigate-security-001-public-table/EVAL.test.ts
@@ -20,8 +20,6 @@ vi.mock('@supabase-evals/core', async (importOriginal) => ({
   judge: vi.fn(async () => ({ passed: true, notes: 'stubbed verdict' })),
 }));
 
-const EVAL_DIR = 'evals/investigate-security-001-public-table';
-
 const GOOD_REPORT = [
   'customer_payment_methods is exposed to anon.',
   'Fix by REVOKE SELECT ON customer_payment_methods FROM anon and enable row level security.',
@@ -38,8 +36,8 @@ const asTranscript = (report: string): TranscriptPart[] => [
 test('scores a report that names the exposure and the fix', async () => {
   await withBackend(
     {
-      projectSeedSql: seedPath(EVAL_DIR, 'project.sql'),
-      logsSeedJsonl: seedPath(EVAL_DIR, 'logs.jsonl'),
+      projectSeedSql: seedPath(import.meta.url, 'project.sql'),
+      logsSeedJsonl: seedPath(import.meta.url, 'logs.jsonl'),
     },
     async (backend) => {
       const { rows } = await backend.query(`
diff --git a/test-utils/scorer-test-kit.ts b/test-utils/scorer-test-kit.ts
index d958d2f3..0a93e3d7 100644
--- a/test-utils/scorer-test-kit.ts
+++ b/test-utils/scorer-test-kit.ts
@@ -25,9 +25,14 @@ import type {
 /** Repo root, from `test-utils/`. */
 export const ROOT = join(dirname(fileURLToPath(import.meta.url)), '..');
 
-/** Path to a file in an eval's `remote/` seed directory. */
-export function seedPath(relDir: string, file: string): string {
-  return join(ROOT, relDir, 'remote', file);
+/**
+ * Path to a file in an eval's `remote/` seed directory, resolved from the
+ * calling test's own location. Pass `import.meta.url`: a colocated
+ * `EVAL.test.ts` always seeds its own scenario, where a hand-written directory
+ * string can silently point at a different eval's fixtures and still pass.
+ */
+export function seedPath(testUrl: string, file: string): string {
+  return join(dirname(fileURLToPath(testUrl)), 'remote', file);
 }
 
 /** A `ToolEvalContext` backed by a live platform-lite project. */

From 7aa6d2a638b773379d4dd4e5efa2ed1939f5f579 Mon Sep 17 00:00:00 2001
From: Barry Roodt <barry.roodt@supabase.io>
Date: Wed, 29 Jul 2026 10:25:04 +0200
Subject: [PATCH 4/6] fix(harness): give scored frontend workspaces their
 VITE_SUPABASE_* env

Frontend evals instruct the agent to read import.meta.env.VITE_SUPABASE_URL
and _ANON_KEY (PROMPT.md:29-30), but nothing ever supplied them: no .env.local
in the workspace, none written by the runner or the sandbox. createClient threw
at module import, vitest collected zero tests, and the eval scored 1/2 no
matter what the agent produced.

The values already existed, but only as text inside the string setupSource()
returns, so the generated config could not reference them. Hoists them to real
module scope and interpolates them into both generated files, so the setup and
the app under test agree on one definition.

Also drops the .env.local write from project-runner.test.ts. A real agent
workspace has no such file, so writing it made the test green while the live
path failed, which is how this defect stayed hidden.

Verified end to end: claude-code-sonnet-5 x build-frontend-001-todos-app now
passes 2/2 with numTotalTests 2, up from 1/2 with zero tests collected. First
passing result this eval has ever produced.
---
 apps/framework/harness/project-runner.test.ts | 11 +++-------
 apps/framework/harness/project-runner.ts      | 21 ++++++++++++++++---
 2 files changed, 21 insertions(+), 11 deletions(-)

diff --git a/apps/framework/harness/project-runner.test.ts b/apps/framework/harness/project-runner.test.ts
index 1568d1a3..12345816 100644
--- a/apps/framework/harness/project-runner.test.ts
+++ b/apps/framework/harness/project-runner.test.ts
@@ -53,14 +53,9 @@ test('builds and tests a known-good frontend workspace', async () => {
   cpSync(join(ROOT, FRONTEND_EVAL, 'tests'), join(workspace, 'tests'), {
     recursive: true,
   });
-  writeFileSync(
-    join(workspace, '.env.local'),
-    [
-      'VITE_SUPABASE_URL=http://supabase-evals.local',
-      'VITE_SUPABASE_ANON_KEY=supabase-evals-anon-key',
-      '',
-    ].join('\n')
-  );
+  // Deliberately no `.env.local`. A real agent workspace does not have one, so
+  // writing it here would make this test pass while the live scoring path
+  // fails. `vitestRun` injects VITE_SUPABASE_* through the config it generates.
   writeFileSync(join(workspace, 'src', 'App.tsx'), referenceApp());
 
   const build = await viteBuild(workspace);
diff --git a/apps/framework/harness/project-runner.ts b/apps/framework/harness/project-runner.ts
index 500b7302..b2219859 100644
--- a/apps/framework/harness/project-runner.ts
+++ b/apps/framework/harness/project-runner.ts
@@ -16,6 +16,17 @@ export function resolvePackageBin(pkg: string, entry: string): string {
   return join(dirname(nodeRequire.resolve(`${pkg}/package.json`)), entry);
 }
 
+/**
+ * The mock project the generated vitest setup serves. Shared by that setup and
+ * by the generated config, which injects them as `VITE_*` so an app under test
+ * resolves the same project the setup boots. Evals instruct agents to read
+ * `import.meta.env.VITE_SUPABASE_URL` / `_ANON_KEY`, so the harness has to
+ * supply them or every correct solution throws at import.
+ */
+const PROJECT_DB_URL = 'http://supabase-evals.local';
+const PROJECT_DB_ANON_KEY = 'supabase-evals-anon-key';
+const PROJECT_DB_JWT_SECRET = 'supabase-evals-dev-secret';
+
 export async function viteBuild(workspace: string): Promise<CommandResult> {
   return runNodeBin(
     resolvePackageBin('vite', 'bin/vite.js'),
@@ -41,6 +52,10 @@ export async function vitestRun(workspace: string): Promise<VitestResult> {
       '    environment: "happy-dom",',
       `    setupFiles: [${JSON.stringify('./.evals/vitest-supalite-setup.ts')}],`,
       '    include: ["tests/**/*.test.{ts,tsx}"],',
+      '    env: {',
+      `      VITE_SUPABASE_URL: ${JSON.stringify(PROJECT_DB_URL)},`,
+      `      VITE_SUPABASE_ANON_KEY: ${JSON.stringify(PROJECT_DB_ANON_KEY)},`,
+      '    },',
       '  },',
       '});',
       '',
@@ -72,9 +87,9 @@ import { afterAll } from "vitest";
 import { App, getAuthSchemaSql, SUPABASE_AUTH_HELPERS_SQL } from "@supabase/lite";
 import { createPgliteConnection } from "@supabase/lite/pglite";
 
-const PROJECT_DB_URL = "http://supabase-evals.local";
-const PROJECT_DB_ANON_KEY = "supabase-evals-anon-key";
-const PROJECT_DB_JWT_SECRET = "supabase-evals-dev-secret";
+const PROJECT_DB_URL = ${JSON.stringify(PROJECT_DB_URL)};
+const PROJECT_DB_ANON_KEY = ${JSON.stringify(PROJECT_DB_ANON_KEY)};
+const PROJECT_DB_JWT_SECRET = ${JSON.stringify(PROJECT_DB_JWT_SECRET)};
 const AUTH_SQL = \`
 CREATE ROLE anon NOLOGIN;
 CREATE ROLE authenticated NOLOGIN;

From 3553356b6b9b2a79c85648be40156469b5f75149 Mon Sep 17 00:00:00 2001
From: Barry Roodt <barry.roodt@supabase.io>
Date: Wed, 29 Jul 2026 11:46:06 +0200
Subject: [PATCH 5/6] fix(harness): give vite build the same project env as
 vitest

The previous commit supplied VITE_SUPABASE_* to the generated vitest config
only. viteBuild invoked vite with no env, so `import.meta.env.VITE_SUPABASE_URL`
compiled to undefined and the bundle called createClient(void 0, void 0). The
build exited 0 and the eval scored 2/2 while shipping an artifact that throws
on load: the scorer was green, the app was not.

Both tools now read one PROJECT_ENV object, delivered the way each expects,
so a build and its tests cannot disagree about which project the app targets.

project-runner.test.ts asserts the built bundle actually contains the project
URL rather than just checking vite's exit code. Falsified by reverting the fix:
the assertion fails with "built bundle is missing the Supabase project URL".

Verified end to end: the real agent bundle from a sandboxed run now embeds
supabase-evals.local, where the previous passing run's bundle did not.
---
 apps/framework/harness/project-runner.test.ts | 16 +++++++++++-
 apps/framework/harness/project-runner.ts      | 25 +++++++++++++------
 2 files changed, 33 insertions(+), 8 deletions(-)

diff --git a/apps/framework/harness/project-runner.test.ts b/apps/framework/harness/project-runner.test.ts
index 12345816..34db36a5 100644
--- a/apps/framework/harness/project-runner.test.ts
+++ b/apps/framework/harness/project-runner.test.ts
@@ -2,6 +2,7 @@ import {
   cpSync,
   existsSync,
   mkdirSync,
+  readdirSync,
   readFileSync,
   rmSync,
   writeFileSync,
@@ -55,12 +56,25 @@ test('builds and tests a known-good frontend workspace', async () => {
   });
   // Deliberately no `.env.local`. A real agent workspace does not have one, so
   // writing it here would make this test pass while the live scoring path
-  // fails. `vitestRun` injects VITE_SUPABASE_* through the config it generates.
+  // fails. The harness supplies VITE_SUPABASE_* to both tools instead.
   writeFileSync(join(workspace, 'src', 'App.tsx'), referenceApp());
 
   const build = await viteBuild(workspace);
   expect(build.ok, build.stderr || build.stdout).toBe(true);
 
+  // An exit code of 0 only says Vite ran. Without the build env the bundle
+  // compiles `import.meta.env.VITE_SUPABASE_URL` to undefined and the app
+  // throws on load, so assert the config actually landed in the artifact.
+  const bundle = readdirSync(join(workspace, 'dist', 'assets'))
+    .filter((file) => file.endsWith('.js'))
+    .map((file) =>
+      readFileSync(join(workspace, 'dist', 'assets', file), 'utf8')
+    )
+    .join('');
+  expect(bundle, 'built bundle is missing the Supabase project URL').toContain(
+    'supabase-evals.local'
+  );
+
   const vitest = await vitestRun(workspace);
   expect(vitest.ok, vitest.stderr || vitest.stdout).toBe(true);
 });
diff --git a/apps/framework/harness/project-runner.ts b/apps/framework/harness/project-runner.ts
index b2219859..b17a5574 100644
--- a/apps/framework/harness/project-runner.ts
+++ b/apps/framework/harness/project-runner.ts
@@ -17,21 +17,31 @@ export function resolvePackageBin(pkg: string, entry: string): string {
 }
 
 /**
- * The mock project the generated vitest setup serves. Shared by that setup and
- * by the generated config, which injects them as `VITE_*` so an app under test
- * resolves the same project the setup boots. Evals instruct agents to read
- * `import.meta.env.VITE_SUPABASE_URL` / `_ANON_KEY`, so the harness has to
+ * The mock project the generated vitest setup serves. Evals instruct agents to
+ * read `import.meta.env.VITE_SUPABASE_URL` / `_ANON_KEY`, so the harness has to
  * supply them or every correct solution throws at import.
  */
 const PROJECT_DB_URL = 'http://supabase-evals.local';
 const PROJECT_DB_ANON_KEY = 'supabase-evals-anon-key';
 const PROJECT_DB_JWT_SECRET = 'supabase-evals-dev-secret';
 
+/**
+ * One definition, reached two ways: Vite reads `VITE_`-prefixed vars straight
+ * off the build process env, while Vitest takes them from the config it is
+ * handed. Both are generated from this object so a build and its tests can
+ * never disagree about which project the app points at.
+ */
+const PROJECT_ENV: Record<string, string> = {
+  VITE_SUPABASE_URL: PROJECT_DB_URL,
+  VITE_SUPABASE_ANON_KEY: PROJECT_DB_ANON_KEY,
+};
+
 export async function viteBuild(workspace: string): Promise<CommandResult> {
   return runNodeBin(
     resolvePackageBin('vite', 'bin/vite.js'),
     ['build'],
-    workspace
+    workspace,
+    PROJECT_ENV
   );
 }
 
@@ -53,8 +63,9 @@ export async function vitestRun(workspace: string): Promise<VitestResult> {
       `    setupFiles: [${JSON.stringify('./.evals/vitest-supalite-setup.ts')}],`,
       '    include: ["tests/**/*.test.{ts,tsx}"],',
       '    env: {',
-      `      VITE_SUPABASE_URL: ${JSON.stringify(PROJECT_DB_URL)},`,
-      `      VITE_SUPABASE_ANON_KEY: ${JSON.stringify(PROJECT_DB_ANON_KEY)},`,
+      ...Object.entries(PROJECT_ENV).map(
+        ([key, value]) => `      ${key}: ${JSON.stringify(value)},`
+      ),
       '    },',
       '  },',
       '});',

From 501f5e7a2205132746d8a198d6acb9127bd1a2ff Mon Sep 17 00:00:00 2001
From: Barry Roodt <barry.roodt@supabase.io>
Date: Wed, 29 Jul 2026 11:59:47 +0200
Subject: [PATCH 6/6] refactor(harness): hand both tools the project env the
 same way

The previous commit gave Vite the env via process env and Vitest via an `env:`
block interpolated into the generated config, then added a comment explaining
why there were two mechanisms. A comment justifying complexity is a good sign
the complexity is not needed.

Vite exposes VITE_-prefixed variables that already exist in the environment on
import.meta.env, and Vitest inherits that behaviour, so passing PROJECT_ENV
through runNodeBin covers both. Drops the generated env block and the
Object.entries interpolation.

Verified with the block removed: the smoke workspace's generated config has no
`env:` key and its withheld tests still run 2/2, and a full sandbox run of
build-frontend-001-todos-app passes 2/2 with the project URL still embedded in
the agent's bundle.
---
 apps/framework/harness/project-runner.ts | 15 +++++----------
 1 file changed, 5 insertions(+), 10 deletions(-)

diff --git a/apps/framework/harness/project-runner.ts b/apps/framework/harness/project-runner.ts
index b17a5574..93d0e6ee 100644
--- a/apps/framework/harness/project-runner.ts
+++ b/apps/framework/harness/project-runner.ts
@@ -26,10 +26,10 @@ const PROJECT_DB_ANON_KEY = 'supabase-evals-anon-key';
 const PROJECT_DB_JWT_SECRET = 'supabase-evals-dev-secret';
 
 /**
- * One definition, reached two ways: Vite reads `VITE_`-prefixed vars straight
- * off the build process env, while Vitest takes them from the config it is
- * handed. Both are generated from this object so a build and its tests can
- * never disagree about which project the app points at.
+ * Handed to both tools the same way, as process env. Vite exposes
+ * `VITE_`-prefixed variables that already exist in the environment on
+ * `import.meta.env`, and Vitest inherits that behaviour, so neither the build
+ * nor the test run needs this injected through generated config.
  */
 const PROJECT_ENV: Record<string, string> = {
   VITE_SUPABASE_URL: PROJECT_DB_URL,
@@ -62,11 +62,6 @@ export async function vitestRun(workspace: string): Promise<VitestResult> {
       '    environment: "happy-dom",',
       `    setupFiles: [${JSON.stringify('./.evals/vitest-supalite-setup.ts')}],`,
       '    include: ["tests/**/*.test.{ts,tsx}"],',
-      '    env: {',
-      ...Object.entries(PROJECT_ENV).map(
-        ([key, value]) => `      ${key}: ${JSON.stringify(value)},`
-      ),
-      '    },',
       '  },',
       '});',
       '',
@@ -82,7 +77,7 @@ export async function vitestRun(workspace: string): Promise<VitestResult> {
       `--outputFile=${reportPath}`,
     ],
     workspace,
-    { SUPABASE_EVALS_WORKSPACE: workspace }
+    { ...PROJECT_ENV, SUPABASE_EVALS_WORKSPACE: workspace }
   );
   const parsed = existsSync(reportPath)
     ? parseVitestReport(reportPath)