refactor(framework): run framework tests through Vitest

.github/workflows/biome.yml

@@ -29,3 +29,30 @@ jobs:
 
       - name: Check formatting
         run: pnpm format:check
+
+  check:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@9f698171ed81b15d1823a05fc7211befd50c8ae0 # v6.0.3
+        with:
+          persist-credentials: false
+
+      - name: Install pnpm
+        uses: pnpm/action-setup@fc06bc1257f339d1d5d8b3a19a8cae5388b55320 # v5.0.0
+
+      - name: Setup Node.js
+        uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
+        with:
+          node-version-file: .node-version
+          cache: pnpm
+
+      - name: Install dependencies
+        run: pnpm install --frozen-lockfile
+
+      # Typechecks plus the Vitest suites. No API keys: every test that would
+      # reach a model stubs it, so this must never need credentials or spend.
+      # `typecheck` covers the web app and the frontend reference solution,
+      # which `check` does not reach.
+      - name: Run checks
+        run: pnpm typecheck && pnpm check

apps/framework/harness/platform-backend.test.ts

@@ -0,0 +1,75 @@
+import { join } from 'node:path';
+import { expect, test } from 'vitest';
+import { bootPlatformBackend } from './platform-backend.js';
+import { ROOT, withBackend } from '../../../test-utils/scorer-test-kit.js';
+
+// Deliberately borrows another eval's fixture, so the path stays explicit
+// rather than going through the colocated `seedPath` helper.
+const LOGS_SEED = join(
+  ROOT,
+  'evals/investigate-logs-001-top-error-function/remote/logs.jsonl'
+);
+
+test('supalite auth issues a session supabase-js can write and read under RLS', async () => {
+  await withBackend({}, async (backend) => {
+    await backend.query(`
+CREATE TABLE todos (
+  id uuid PRIMARY KEY DEFAULT gen_random_uuid(),
+  user_id uuid NOT NULL,
+  body text NOT NULL
+);
+
+ALTER TABLE todos ENABLE ROW LEVEL SECURITY;
+GRANT SELECT, INSERT ON todos TO authenticated;
+
+CREATE POLICY "users can insert their own todos" ON todos FOR INSERT TO authenticated WITH CHECK (user_id = auth.uid());
+CREATE POLICY "users can read their own todos" ON todos FOR SELECT TO authenticated USING (user_id = auth.uid());
+    `);
+
+    const client = backend.client;
+    const { data: signup, error: signupError } = await client.auth.signUp({
+      email: `smoke-${Date.now()}@example.com`,
+      password: 'secret123',
+    });
+    expect(signupError).toBeNull();
+    expect(signup.user?.id).toBeTruthy();
+
+    const { error: insertError } = await client.from('todos').insert({
+      user_id: signup.user?.id,
+      body: 'verify supabase-js path',
+    });
+    expect(insertError).toBeNull();
+
+    const { data: rows, error: selectError } = await client
+      .from('todos')
+      .select('body')
+      .eq('user_id', signup.user?.id);
+    expect(selectError).toBeNull();
+    expect(rows).toEqual([{ body: 'verify supabase-js path' }]);
+  });
+});
+
+test('close disposes the platform, and is idempotent', async () => {
+  const backend = await bootPlatformBackend({});
+  await backend.query('select 1 as n');
+  await backend.close();
+
+  await expect(backend.query('select 1 as n')).rejects.toThrow();
+  await expect(backend.close()).resolves.not.toThrow();
+});
+
+test('seeded logs are queryable over the analytics endpoint', async () => {
+  await withBackend(
+    { logsSeedJsonl: LOGS_SEED },
+    async ({ url, ref, accessToken }) => {
+      const sql = 'SELECT count(*)::int AS n FROM edge_logs';
+      const res = await fetch(
+        `${url}/v1/projects/${ref}/analytics/endpoints/logs.all?sql=${encodeURIComponent(sql)}`,
+        { headers: { Authorization: `Bearer ${accessToken}` } }
+      );
+      const body = (await res.json()) as { result: Array<{ n: number }> };
+
+      expect(body.result[0]?.n).toBeGreaterThan(0);
+    }
+  );
+});

apps/framework/harness/project-runner.test.ts

@@ -0,0 +1,80 @@
+import {
+  cpSync,
+  existsSync,
+  mkdirSync,
+  readdirSync,
+  readFileSync,
+  rmSync,
+  writeFileSync,
+} from 'node:fs';
+import { dirname, join } from 'node:path';
+import { expect, test } from 'vitest';
+import { resolvePackageBin, viteBuild, vitestRun } from './project-runner.js';
+import { ROOT } from '../../../test-utils/scorer-test-kit.js';
+
+const FRONTEND_EVAL = 'evals/build-frontend-001-todos-app';
+
+/** The eval's reference solution, kept as a real file so it typechecks. */
+const referenceApp = () =>
+  readFileSync(join(ROOT, FRONTEND_EVAL, 'reference', 'App.tsx'), 'utf8');
+
+// Regression guard for AI-975: pnpm's isolated layout has no hoisted
+// `<repo>/node_modules/<pkg>`, so a repo-root path missed every time and the
+// bin never even launched.
+test.each([
+  ['vite', 'bin/vite.js'],
+  ['vitest', 'vitest.mjs'],
+])('resolves the %s binary that actually exists on disk', (pkg, entry) => {
+  const resolved = resolvePackageBin(pkg, entry);
+
+  expect(existsSync(resolved), `${pkg} bin missing at ${resolved}`).toBe(true);
+});
+
+// AI-975, second layer: the workspace's own `vite.config.ts` imports `vite`,
+// and vite compiles that config into `<repo>/node_modules/.vite-temp/`, so
+// resolution anchors at the repo root. The scored workspace lives under
+// `results/` and can only walk up to the repo root too. That is the documented
+// contract (README, and the `copyToHost` doc comment: score with repo-root
+// vite/vitest so the toolchain need not exist in the sandbox), so the root
+// manifest owns the frontend toolchain. This fixture copies `local/` with no
+// `node_modules`, exactly as that contract assumes.
+test('builds and tests a known-good frontend workspace', async () => {
+  const workspace = join(
+    ROOT,
+    'results',
+    '_smoke',
+    'build-frontend-001-todos-app'
+  );
+  rmSync(workspace, { recursive: true, force: true });
+  mkdirSync(dirname(workspace), { recursive: true });
+  cpSync(join(ROOT, FRONTEND_EVAL, 'local'), workspace, {
+    recursive: true,
+    filter: (src) => !src.endsWith('/EVAL.ts'),
+  });
+  cpSync(join(ROOT, FRONTEND_EVAL, 'tests'), join(workspace, 'tests'), {
+    recursive: true,
+  });
+  // Deliberately no `.env.local`. A real agent workspace does not have one, so
+  // writing it here would make this test pass while the live scoring path
+  // fails. The harness supplies VITE_SUPABASE_* to both tools instead.
+  writeFileSync(join(workspace, 'src', 'App.tsx'), referenceApp());
+
+  const build = await viteBuild(workspace);
+  expect(build.ok, build.stderr || build.stdout).toBe(true);
+
+  // An exit code of 0 only says Vite ran. Without the build env the bundle
+  // compiles `import.meta.env.VITE_SUPABASE_URL` to undefined and the app
+  // throws on load, so assert the config actually landed in the artifact.
+  const bundle = readdirSync(join(workspace, 'dist', 'assets'))
+    .filter((file) => file.endsWith('.js'))
+    .map((file) =>
+      readFileSync(join(workspace, 'dist', 'assets', file), 'utf8')
+    )
+    .join('');
+  expect(bundle, 'built bundle is missing the Supabase project URL').toContain(
+    'supabase-evals.local'
+  );
+
+  const vitest = await vitestRun(workspace);
+  expect(vitest.ok, vitest.stderr || vitest.stdout).toBe(true);
+});

apps/framework/harness/project-runner.ts

@@ -1,17 +1,47 @@
 import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs';
 import { dirname, join } from 'node:path';
-import { fileURLToPath } from 'node:url';
+import { createRequire } from 'node:module';
 import { spawn } from 'node:child_process';
 import type { CommandResult, VitestResult } from '@supabase-evals/core';
 
-const __dirname = dirname(fileURLToPath(import.meta.url));
-const ROOT = join(__dirname, '..', '..', '..');
+const nodeRequire = createRequire(import.meta.url);
+
+/**
+ * Resolve a dependency's bundled entry script. pnpm's isolated layout never
+ * creates a hoisted `<repo>/node_modules/<pkg>`, so anchoring on the repo root
+ * misses every time. Anchor on the `package.json` each package exports instead
+ * of requesting a deep subpath, which `exports` may stop publishing on a bump.
+ */
+export function resolvePackageBin(pkg: string, entry: string): string {
+  return join(dirname(nodeRequire.resolve(`${pkg}/package.json`)), entry);
+}
+
+/**
+ * The mock project the generated vitest setup serves. Evals instruct agents to
+ * read `import.meta.env.VITE_SUPABASE_URL` / `_ANON_KEY`, so the harness has to
+ * supply them or every correct solution throws at import.
+ */
+const PROJECT_DB_URL = 'http://supabase-evals.local';
+const PROJECT_DB_ANON_KEY = 'supabase-evals-anon-key';
+const PROJECT_DB_JWT_SECRET = 'supabase-evals-dev-secret';
+
+/**
+ * Handed to both tools the same way, as process env. Vite exposes
+ * `VITE_`-prefixed variables that already exist in the environment on
+ * `import.meta.env`, and Vitest inherits that behaviour, so neither the build
+ * nor the test run needs this injected through generated config.
+ */
+const PROJECT_ENV: Record<string, string> = {
+  VITE_SUPABASE_URL: PROJECT_DB_URL,
+  VITE_SUPABASE_ANON_KEY: PROJECT_DB_ANON_KEY,
+};
 
 export async function viteBuild(workspace: string): Promise<CommandResult> {
   return runNodeBin(
-    join(ROOT, 'node_modules', 'vite', 'bin', 'vite.js'),
+    resolvePackageBin('vite', 'bin/vite.js'),
     ['build'],
-    workspace
+    workspace,
+    PROJECT_ENV
   );
 }
 
@@ -38,7 +68,7 @@ export async function vitestRun(workspace: string): Promise<VitestResult> {
     ].join('\n')
   );
   const result = await runNodeBin(
-    join(ROOT, 'node_modules', 'vitest', 'vitest.mjs'),
+    resolvePackageBin('vitest', 'vitest.mjs'),
     [
       'run',
       '--config',
@@ -47,7 +77,7 @@ export async function vitestRun(workspace: string): Promise<VitestResult> {
       `--outputFile=${reportPath}`,
     ],
     workspace,
-    { SUPABASE_EVALS_WORKSPACE: workspace }
+    { ...PROJECT_ENV, SUPABASE_EVALS_WORKSPACE: workspace }
   );
   const parsed = existsSync(reportPath)
     ? parseVitestReport(reportPath)
@@ -63,9 +93,9 @@ import { afterAll } from "vitest";
 import { App, getAuthSchemaSql, SUPABASE_AUTH_HELPERS_SQL } from "@supabase/lite";
 import { createPgliteConnection } from "@supabase/lite/pglite";
 
-const PROJECT_DB_URL = "http://supabase-evals.local";
-const PROJECT_DB_ANON_KEY = "supabase-evals-anon-key";
-const PROJECT_DB_JWT_SECRET = "supabase-evals-dev-secret";
+const PROJECT_DB_URL = ${JSON.stringify(PROJECT_DB_URL)};
+const PROJECT_DB_ANON_KEY = ${JSON.stringify(PROJECT_DB_ANON_KEY)};
+const PROJECT_DB_JWT_SECRET = ${JSON.stringify(PROJECT_DB_JWT_SECRET)};
 const AUTH_SQL = \`
 CREATE ROLE anon NOLOGIN;
 CREATE ROLE authenticated NOLOGIN;

apps/framework/package.json

@@ -9,7 +9,7 @@
     "eval:dry": "node --env-file=../../.env --import tsx/esm harness/run-eval.ts --dry",
     "eval:smoke": "node --env-file=../../.env --import tsx/esm harness/run-eval.ts --smoke",
     "typecheck": "tsc --noEmit",
-    "test:framework": "node --env-file-if-exists=../../.env --import tsx/esm scripts/smoke-framework.ts",
+    "test:framework": "vitest run",
     "export-results": "node --import tsx/esm scripts/export-results.ts",
     "demo:mcp": "node --env-file=../../.env --import tsx/esm scripts/mcp-demo.ts",
     "demo:executor": "node --env-file=../../.env --import tsx/esm scripts/executor-demo.ts"
@@ -22,16 +22,7 @@
     "@electric-sql/pglite": "catalog:",
     "@supabase-evals/core": "workspace:*",
     "@supabase-evals/sandbox": "workspace:*",
-    "@supabase/supabase-js": "catalog:",
-    "@testing-library/jest-dom": "^6.9.1",
-    "@testing-library/react": "^16.3.2",
-    "@vitejs/plugin-react": "catalog:",
     "ai": "catalog:",
-    "happy-dom": "^20.9.0",
-    "@supabase/lite": "catalog:",
-    "react": "^19.2.5",
-    "react-dom": "^19.2.5",
-    "vite": "catalog:",
     "vitest": "catalog:"
   },
   "devDependencies": {