diff --git a/.github/workflows/biome.yml b/.github/workflows/biome.yml index fc635388..af081863 100644 --- a/.github/workflows/biome.yml +++ b/.github/workflows/biome.yml @@ -29,3 +29,30 @@ jobs: - name: Check formatting run: pnpm format:check + + check: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@9f698171ed81b15d1823a05fc7211befd50c8ae0 # v6.0.3 + with: + persist-credentials: false + + - name: Install pnpm + uses: pnpm/action-setup@fc06bc1257f339d1d5d8b3a19a8cae5388b55320 # v5.0.0 + + - name: Setup Node.js + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 + with: + node-version-file: .node-version + cache: pnpm + + - name: Install dependencies + run: pnpm install --frozen-lockfile + + # Typechecks plus the Vitest suites. No API keys: every test that would + # reach a model stubs it, so this must never need credentials or spend. + # `typecheck` covers the web app and the frontend reference solution, + # which `check` does not reach. + - name: Run checks + run: pnpm typecheck && pnpm check diff --git a/apps/framework/harness/platform-backend.test.ts b/apps/framework/harness/platform-backend.test.ts new file mode 100644 index 00000000..22a33d14 --- /dev/null +++ b/apps/framework/harness/platform-backend.test.ts @@ -0,0 +1,75 @@ +import { join } from 'node:path'; +import { expect, test } from 'vitest'; +import { bootPlatformBackend } from './platform-backend.js'; +import { ROOT, withBackend } from '../../../test-utils/scorer-test-kit.js'; + +// Deliberately borrows another eval's fixture, so the path stays explicit +// rather than going through the colocated `seedPath` helper. +const LOGS_SEED = join( + ROOT, + 'evals/investigate-logs-001-top-error-function/remote/logs.jsonl' +); + +test('supalite auth issues a session supabase-js can write and read under RLS', async () => { + await withBackend({}, async (backend) => { + await backend.query(` +CREATE TABLE todos ( + id uuid PRIMARY KEY DEFAULT gen_random_uuid(), + user_id uuid NOT NULL, + body text NOT NULL +); + +ALTER TABLE todos ENABLE ROW LEVEL SECURITY; +GRANT SELECT, INSERT ON todos TO authenticated; + +CREATE POLICY "users can insert their own todos" ON todos FOR INSERT TO authenticated WITH CHECK (user_id = auth.uid()); +CREATE POLICY "users can read their own todos" ON todos FOR SELECT TO authenticated USING (user_id = auth.uid()); + `); + + const client = backend.client; + const { data: signup, error: signupError } = await client.auth.signUp({ + email: `smoke-${Date.now()}@example.com`, + password: 'secret123', + }); + expect(signupError).toBeNull(); + expect(signup.user?.id).toBeTruthy(); + + const { error: insertError } = await client.from('todos').insert({ + user_id: signup.user?.id, + body: 'verify supabase-js path', + }); + expect(insertError).toBeNull(); + + const { data: rows, error: selectError } = await client + .from('todos') + .select('body') + .eq('user_id', signup.user?.id); + expect(selectError).toBeNull(); + expect(rows).toEqual([{ body: 'verify supabase-js path' }]); + }); +}); + +test('close disposes the platform, and is idempotent', async () => { + const backend = await bootPlatformBackend({}); + await backend.query('select 1 as n'); + await backend.close(); + + await expect(backend.query('select 1 as n')).rejects.toThrow(); + await expect(backend.close()).resolves.not.toThrow(); +}); + +test('seeded logs are queryable over the analytics endpoint', async () => { + await withBackend( + { logsSeedJsonl: LOGS_SEED }, + async ({ url, ref, accessToken }) => { + const sql = 'SELECT count(*)::int AS n FROM edge_logs'; + const res = await fetch( + `${url}/v1/projects/${ref}/analytics/endpoints/logs.all?sql=${encodeURIComponent(sql)}`, + { headers: { Authorization: `Bearer ${accessToken}` } } + ); + const body = (await res.json()) as { result: Array<{ n: number }> }; + + expect(body.result[0]?.n).toBeGreaterThan(0); + } + ); +}); diff --git a/apps/framework/harness/project-runner.test.ts b/apps/framework/harness/project-runner.test.ts new file mode 100644 index 00000000..34db36a5 --- /dev/null +++ b/apps/framework/harness/project-runner.test.ts @@ -0,0 +1,80 @@ +import { + cpSync, + existsSync, + mkdirSync, + readdirSync, + readFileSync, + rmSync, + writeFileSync, +} from 'node:fs'; +import { dirname, join } from 'node:path'; +import { expect, test } from 'vitest'; +import { resolvePackageBin, viteBuild, vitestRun } from './project-runner.js'; +import { ROOT } from '../../../test-utils/scorer-test-kit.js'; + +const FRONTEND_EVAL = 'evals/build-frontend-001-todos-app'; + +/** The eval's reference solution, kept as a real file so it typechecks. */ +const referenceApp = () => + readFileSync(join(ROOT, FRONTEND_EVAL, 'reference', 'App.tsx'), 'utf8'); + +// Regression guard for AI-975: pnpm's isolated layout has no hoisted +// `/node_modules/`, so a repo-root path missed every time and the +// bin never even launched. +test.each([ + ['vite', 'bin/vite.js'], + ['vitest', 'vitest.mjs'], +])('resolves the %s binary that actually exists on disk', (pkg, entry) => { + const resolved = resolvePackageBin(pkg, entry); + + expect(existsSync(resolved), `${pkg} bin missing at ${resolved}`).toBe(true); +}); + +// AI-975, second layer: the workspace's own `vite.config.ts` imports `vite`, +// and vite compiles that config into `/node_modules/.vite-temp/`, so +// resolution anchors at the repo root. The scored workspace lives under +// `results/` and can only walk up to the repo root too. That is the documented +// contract (README, and the `copyToHost` doc comment: score with repo-root +// vite/vitest so the toolchain need not exist in the sandbox), so the root +// manifest owns the frontend toolchain. This fixture copies `local/` with no +// `node_modules`, exactly as that contract assumes. +test('builds and tests a known-good frontend workspace', async () => { + const workspace = join( + ROOT, + 'results', + '_smoke', + 'build-frontend-001-todos-app' + ); + rmSync(workspace, { recursive: true, force: true }); + mkdirSync(dirname(workspace), { recursive: true }); + cpSync(join(ROOT, FRONTEND_EVAL, 'local'), workspace, { + recursive: true, + filter: (src) => !src.endsWith('/EVAL.ts'), + }); + cpSync(join(ROOT, FRONTEND_EVAL, 'tests'), join(workspace, 'tests'), { + recursive: true, + }); + // Deliberately no `.env.local`. A real agent workspace does not have one, so + // writing it here would make this test pass while the live scoring path + // fails. The harness supplies VITE_SUPABASE_* to both tools instead. + writeFileSync(join(workspace, 'src', 'App.tsx'), referenceApp()); + + const build = await viteBuild(workspace); + expect(build.ok, build.stderr || build.stdout).toBe(true); + + // An exit code of 0 only says Vite ran. Without the build env the bundle + // compiles `import.meta.env.VITE_SUPABASE_URL` to undefined and the app + // throws on load, so assert the config actually landed in the artifact. + const bundle = readdirSync(join(workspace, 'dist', 'assets')) + .filter((file) => file.endsWith('.js')) + .map((file) => + readFileSync(join(workspace, 'dist', 'assets', file), 'utf8') + ) + .join(''); + expect(bundle, 'built bundle is missing the Supabase project URL').toContain( + 'supabase-evals.local' + ); + + const vitest = await vitestRun(workspace); + expect(vitest.ok, vitest.stderr || vitest.stdout).toBe(true); +}); diff --git a/apps/framework/harness/project-runner.ts b/apps/framework/harness/project-runner.ts index 3b17d13a..93d0e6ee 100644 --- a/apps/framework/harness/project-runner.ts +++ b/apps/framework/harness/project-runner.ts @@ -1,37 +1,51 @@ -import { - existsSync, - mkdirSync, - readFileSync, - symlinkSync, - writeFileSync, -} from 'node:fs'; +import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs'; import { dirname, join } from 'node:path'; -import { fileURLToPath } from 'node:url'; +import { createRequire } from 'node:module'; import { spawn } from 'node:child_process'; import type { CommandResult, VitestResult } from '@supabase-evals/core'; -const __dirname = dirname(fileURLToPath(import.meta.url)); -const ROOT = join(__dirname, '..'); +const nodeRequire = createRequire(import.meta.url); -// Vite/vitest resolve their own package (and the workspace's deps, e.g. react) -// by walking up from the workspace looking for a node_modules dir. Workspaces -// live under results/, outside ROOT, so link ROOT's node_modules in directly. -function linkNodeModules(workspace: string) { - const link = join(workspace, 'node_modules'); - if (!existsSync(link)) symlinkSync(join(ROOT, 'node_modules'), link, 'dir'); +/** + * Resolve a dependency's bundled entry script. pnpm's isolated layout never + * creates a hoisted `/node_modules/`, so anchoring on the repo root + * misses every time. Anchor on the `package.json` each package exports instead + * of requesting a deep subpath, which `exports` may stop publishing on a bump. + */ +export function resolvePackageBin(pkg: string, entry: string): string { + return join(dirname(nodeRequire.resolve(`${pkg}/package.json`)), entry); } +/** + * The mock project the generated vitest setup serves. Evals instruct agents to + * read `import.meta.env.VITE_SUPABASE_URL` / `_ANON_KEY`, so the harness has to + * supply them or every correct solution throws at import. + */ +const PROJECT_DB_URL = 'http://supabase-evals.local'; +const PROJECT_DB_ANON_KEY = 'supabase-evals-anon-key'; +const PROJECT_DB_JWT_SECRET = 'supabase-evals-dev-secret'; + +/** + * Handed to both tools the same way, as process env. Vite exposes + * `VITE_`-prefixed variables that already exist in the environment on + * `import.meta.env`, and Vitest inherits that behaviour, so neither the build + * nor the test run needs this injected through generated config. + */ +const PROJECT_ENV: Record = { + VITE_SUPABASE_URL: PROJECT_DB_URL, + VITE_SUPABASE_ANON_KEY: PROJECT_DB_ANON_KEY, +}; + export async function viteBuild(workspace: string): Promise { - linkNodeModules(workspace); return runNodeBin( - join(ROOT, 'node_modules', 'vite', 'bin', 'vite.js'), + resolvePackageBin('vite', 'bin/vite.js'), ['build'], - workspace + workspace, + PROJECT_ENV ); } export async function vitestRun(workspace: string): Promise { - linkNodeModules(workspace); const reportPath = join(workspace, 'vitest-report.json'); const configPath = join(workspace, 'vitest.evals.config.ts'); const setupDir = join(workspace, '.evals'); @@ -54,7 +68,7 @@ export async function vitestRun(workspace: string): Promise { ].join('\n') ); const result = await runNodeBin( - join(ROOT, 'node_modules', 'vitest', 'vitest.mjs'), + resolvePackageBin('vitest', 'vitest.mjs'), [ 'run', '--config', @@ -63,7 +77,7 @@ export async function vitestRun(workspace: string): Promise { `--outputFile=${reportPath}`, ], workspace, - { SUPABASE_EVALS_WORKSPACE: workspace } + { ...PROJECT_ENV, SUPABASE_EVALS_WORKSPACE: workspace } ); const parsed = existsSync(reportPath) ? parseVitestReport(reportPath) @@ -79,9 +93,9 @@ import { afterAll } from "vitest"; import { App, getAuthSchemaSql, SUPABASE_AUTH_HELPERS_SQL } from "@supabase/lite"; import { createPgliteConnection } from "@supabase/lite/pglite"; -const PROJECT_DB_URL = "http://supabase-evals.local"; -const PROJECT_DB_ANON_KEY = "supabase-evals-anon-key"; -const PROJECT_DB_JWT_SECRET = "supabase-evals-dev-secret"; +const PROJECT_DB_URL = ${JSON.stringify(PROJECT_DB_URL)}; +const PROJECT_DB_ANON_KEY = ${JSON.stringify(PROJECT_DB_ANON_KEY)}; +const PROJECT_DB_JWT_SECRET = ${JSON.stringify(PROJECT_DB_JWT_SECRET)}; const AUTH_SQL = \` CREATE ROLE anon NOLOGIN; CREATE ROLE authenticated NOLOGIN; diff --git a/apps/framework/package.json b/apps/framework/package.json index c726bc37..710155d9 100644 --- a/apps/framework/package.json +++ b/apps/framework/package.json @@ -9,7 +9,7 @@ "eval:dry": "node --env-file=../../.env --import tsx/esm harness/run-eval.ts --dry", "eval:smoke": "node --env-file=../../.env --import tsx/esm harness/run-eval.ts --smoke", "typecheck": "tsc --noEmit", - "test:framework": "node --env-file-if-exists=../../.env --import tsx/esm scripts/smoke-framework.ts", + "test:framework": "vitest run", "export-results": "node --import tsx/esm scripts/export-results.ts", "demo:mcp": "node --env-file=../../.env --import tsx/esm scripts/mcp-demo.ts", "demo:executor": "node --env-file=../../.env --import tsx/esm scripts/executor-demo.ts" @@ -22,16 +22,7 @@ "@electric-sql/pglite": "catalog:", "@supabase-evals/core": "workspace:*", "@supabase-evals/sandbox": "workspace:*", - "@supabase/supabase-js": "catalog:", - "@testing-library/jest-dom": "^6.9.1", - "@testing-library/react": "^16.3.2", - "@vitejs/plugin-react": "catalog:", "ai": "catalog:", - "happy-dom": "^20.9.0", - "@supabase/lite": "catalog:", - "react": "^19.2.5", - "react-dom": "^19.2.5", - "vite": "catalog:", "vitest": "catalog:" }, "devDependencies": { diff --git a/apps/framework/scripts/smoke-framework.ts b/apps/framework/scripts/smoke-framework.ts deleted file mode 100644 index 6709e3be..00000000 --- a/apps/framework/scripts/smoke-framework.ts +++ /dev/null @@ -1,659 +0,0 @@ -import assert from 'node:assert/strict'; -import { cpSync, mkdirSync, rmSync, writeFileSync } from 'node:fs'; -import { dirname, join } from 'node:path'; -import { fileURLToPath, pathToFileURL } from 'node:url'; -import { bootPlatformBackend } from '../harness/platform-backend.js'; -import { viteBuild, vitestRun } from '../harness/project-runner.js'; -import type { - EdgeFunctionsInvokeResult, - ToolEvalContext, - ToolScorer, - TranscriptPart, -} from '../harness/types.js'; -import type { PlatformBackend } from '../harness/platform-backend.js'; -const __dirname = dirname(fileURLToPath(import.meta.url)); -const ROOT = join(__dirname, '..', '..', '..'); - -const DEBUG = process.argv.includes('--debug'); - -// Scorers trigger expected-fail supabase-js calls that log via console.error. -// Silence it globally so smoke runs stay quiet on those. -const stderr = console.error; -if (!DEBUG) console.error = () => undefined; - -const CLIENT_RLS_EVAL = 'evals/build-rls-002-own-todos-client'; -const FUNCTIONS_EVAL = 'evals/build-functions-001-order-total'; -const EDGE_AUTH_DB_EVAL = 'evals/build-functions-002-edge-auth-db'; -const SERVICE_ROLE_BYPASS_EVAL = - 'evals/build-functions-004-service-role-bypass'; -const INVESTIGATE_LOGS_EVAL = 'evals/investigate-logs-001-top-error-function'; -const INVESTIGATE_SECURITY_EVAL = 'evals/investigate-security-001-public-table'; -const FRONTEND_EVAL = 'evals/build-frontend-001-todos-app'; - -async function loadScorer(relDir: string): Promise { - const mod = await import(pathToFileURL(join(ROOT, relDir, 'EVAL.ts')).href); - return mod.default as ToolScorer; -} - -function scorerCtx( - backend: PlatformBackend, - extra?: { agentReport?: string; transcript?: TranscriptPart[] } -) { - return { - mgmt: backend.mgmt, - ref: backend.ref, - client: backend.client, - getClient: backend.getClient, - query: backend.query, - invokeFunction: backend.invokeFunction, - toolCalls: [], - transcript: extra?.transcript ?? [], - agentReport: extra?.agentReport, - }; -} - -function checksMessage(result: { checks?: unknown[] }) { - return JSON.stringify(result.checks ?? []); -} - -function failedCheckNames(result: { - checks?: { name: string; passed: boolean }[]; -}) { - return ( - result.checks - ?.filter((check) => !check.passed) - .map((check) => check.name) ?? [] - ); -} - -async function withBackend( - opts: { projectSeedSql?: string; logsSeedJsonl?: string }, - fn: (backend: PlatformBackend) => Promise -): Promise { - const backend = await bootPlatformBackend(opts); - try { - return await fn(backend); - } finally { - await backend.close(); - } -} - -function seedPath(relDir: string, file: string): string { - return join(ROOT, relDir, 'remote', file); -} - -async function smokeClientRlsEval() { - const scorer = await loadScorer(CLIENT_RLS_EVAL); - - await withBackend( - { projectSeedSql: seedPath(CLIENT_RLS_EVAL, 'project.sql') }, - async (backend) => { - await backend.query(` -ALTER TABLE todos ENABLE ROW LEVEL SECURITY; - -CREATE POLICY "users can read own todos" ON todos FOR SELECT TO authenticated USING (user_id = auth.uid()); -CREATE POLICY "users can insert own todos" ON todos FOR INSERT TO authenticated WITH CHECK (user_id = auth.uid()); -CREATE POLICY "users can update own todos" ON todos FOR UPDATE TO authenticated USING (user_id = auth.uid()) WITH CHECK (user_id = auth.uid()); -CREATE POLICY "users can delete own todos" ON todos FOR DELETE TO authenticated USING (user_id = auth.uid()); - `); - - const result = await scorer(scorerCtx(backend)); - assert.equal(result.passed, true, checksMessage(result)); - } - ); - - console.log('PASS client-scored RLS scorer + supabase-js'); -} - -async function smokeFunctionsEval() { - const scorer = await loadScorer(FUNCTIONS_EVAL); - - await withBackend({}, async (backend) => { - const before = await scorer(scorerCtx(backend)); - assert.equal(before.passed, false); - assert.match(checksMessage(before), /function not found/i); - - const deployUrl = `${backend.url}/v1/projects/${backend.ref}/functions/deploy?slug=order-total`; - const form = new FormData(); - form.append( - 'metadata', - JSON.stringify({ - name: 'order-total', - verify_jwt: false, - entrypoint_path: 'index.ts', - }) - ); - form.append( - 'file', - new File([ORDER_TOTAL_SOURCE], 'index.ts', { - type: 'application/typescript', - }) - ); - - const deployRes = await fetch(deployUrl, { - method: 'POST', - headers: { Authorization: `Bearer ${backend.accessToken}` }, - body: form, - }); - assert.equal( - deployRes.status, - 201, - `deploy failed: ${await deployRes.text()}` - ); - - const after = await scorer(scorerCtx(backend)); - assert.equal(after.passed, true, checksMessage(after)); - }); - - console.log('PASS functions scorer + edge-functions dispatcher'); -} - -function functionResponse( - status: number, - body = '', - outboundBearerTokens: string[] = [] -): EdgeFunctionsInvokeResult { - return { type: 'response', status, headers: {}, body, outboundBearerTokens }; -} - -// Unlike the other smokes, this one fakes the ToolEvalContext instead of -// booting a real backend: the scorer's *decision logic* (which statuses and -// bodies pass) is what we want to pin, and driving that through a real stack -// would mean deploying six edge-function variants. `responses` are consumed in -// the scorer's invocation order (missingAuth, ownNotes, aRequestsB, bRequestsA); -// `serviceRoleResponses` builds them in that same order. -function serviceRoleBypassCtx( - responses: EdgeFunctionsInvokeResult[] -): ToolEvalContext { - const authResult = (id: string, accessToken: string) => ({ - data: { user: { id }, session: { access_token: accessToken } }, - error: null, - }); - const clientA = { - auth: { signUp: async () => authResult('user-a', 'token-a') }, - } as unknown as ToolEvalContext['client']; - const clientB = { - auth: { signUp: async () => authResult('user-b', 'token-b') }, - } as unknown as ToolEvalContext['client']; - - return { - mgmt: {} as ToolEvalContext['mgmt'], - ref: 'test-ref', - client: clientA, - getClient: () => clientB, - query: async () => ({ rows: [] }), - invokeFunction: async () => { - const response = responses.shift(); - if (!response) throw new Error('missing fake function response'); - return response; - }, - toolCalls: [], - transcript: [], - }; -} - -// Order matches the scorer's invocation sequence; overrides are keyed by role. -function serviceRoleResponses( - overrides: Partial< - Record< - 'missingAuth' | 'ownNotes' | 'aRequestsB' | 'bRequestsA', - EdgeFunctionsInvokeResult - > - > = {} -): EdgeFunctionsInvokeResult[] { - return [ - overrides.missingAuth ?? functionResponse(401), - overrides.ownNotes ?? - functionResponse(200, 'user A private note', ['token-a']), - overrides.aRequestsB ?? functionResponse(401, 'unauthorized'), - overrides.bRequestsA ?? functionResponse(403, 'forbidden'), - ]; -} - -async function smokeServiceRoleBypassEval() { - const scorer = await loadScorer(SERVICE_ROLE_BYPASS_EVAL); - const runScorer = (overrides?: Parameters[0]) => - scorer(serviceRoleBypassCtx(serviceRoleResponses(overrides))); - - // The recommended secure fix: reject anonymous access, serve the caller their - // own note over their JWT, and deny forced cross-user reads. - const secure = await runScorer(); - assert.equal(secure.passed, true, checksMessage(secure)); - assert.match(checksMessage(secure), /"notes":"status=401"/); - assert.match(checksMessage(secure), /"notes":"status=200"/); - assert.match(checksMessage(secure), /"notes":"status=403"/); - assert.match( - checksMessage(secure), - /"notes":"bearer_tokens=1, all_match=true"/ - ); - - // Ignoring the spoofed user_id and returning the caller's own note is the - // other secure shape the scorer must accept. - const callerScoped = await runScorer({ - aRequestsB: functionResponse(200, 'user A private note', ['token-a']), - bRequestsA: functionResponse(200, 'user B private note', ['token-b']), - }); - assert.equal(callerScoped.passed, true, checksMessage(callerScoped)); - - // An RLS-scoped read of another user's note returns no rows, which a function - // may surface as 404. That is still a non-leaking denial. - const notFound = await runScorer({ - aRequestsB: functionResponse(404, 'not found'), - bRequestsA: functionResponse(404, 'not found'), - }); - assert.equal(notFound.passed, true, checksMessage(notFound)); - - // A leaked note fails regardless of status (200 or 403). - const leaky = await runScorer({ - aRequestsB: functionResponse(200, 'user B private note'), - bRequestsA: functionResponse(403, 'user A private note'), - }); - assert.equal(leaky.passed, false, checksMessage(leaky)); - assert.deepEqual(failedCheckNames(leaky), [ - 'user A cannot force-read user B note', - 'user B cannot force-read user A note', - ]); - assert.match(checksMessage(leaky), /"notes":"status=200"/); - assert.match(checksMessage(leaky), /"notes":"status=403"/); - - // Serving data to an unauthenticated caller fails, even with no leaked note. - const ignoredAuth = await runScorer({ missingAuth: functionResponse(200) }); - assert.equal(ignoredAuth.passed, false, checksMessage(ignoredAuth)); - assert.deepEqual(failedCheckNames(ignoredAuth), ['rejects missing auth']); - - // A 401 that still echoes a note is a leak, not a denial. - const unauthenticatedLeak = await runScorer({ - missingAuth: functionResponse(401, 'user A private note'), - }); - assert.equal( - unauthenticatedLeak.passed, - false, - checksMessage(unauthenticatedLeak) - ); - assert.deepEqual(failedCheckNames(unauthenticatedLeak), [ - 'rejects missing auth', - ]); - - // Reading via the service-role key instead of the caller's JWT fails. - const serviceRoleRead = await runScorer({ - ownNotes: functionResponse(200, 'user A private note', [ - 'service-role-key', - ]), - }); - assert.equal(serviceRoleRead.passed, false, checksMessage(serviceRoleRead)); - assert.deepEqual(failedCheckNames(serviceRoleRead), [ - "reads only with the caller's JWT", - ]); - - console.log('PASS service-role bypass scorer security contract'); -} - -async function smokeSupaliteClient() { - await withBackend({}, async (backend) => { - await backend.query(` -CREATE TABLE todos ( - id uuid PRIMARY KEY DEFAULT gen_random_uuid(), - user_id uuid NOT NULL, - body text NOT NULL -); - -ALTER TABLE todos ENABLE ROW LEVEL SECURITY; -GRANT SELECT, INSERT ON todos TO authenticated; - -CREATE POLICY "users can insert their own todos" ON todos FOR INSERT TO authenticated WITH CHECK (user_id = auth.uid()); -CREATE POLICY "users can read their own todos" ON todos FOR SELECT TO authenticated USING (user_id = auth.uid()); - `); - - const client = backend.client; - const email = `smoke-${Date.now()}@example.com`; - const { data: signup, error: signupError } = await client.auth.signUp({ - email, - password: 'secret123', - }); - assert.equal(signupError, null); - assert(signup.user?.id); - - const { error: insertError } = await client.from('todos').insert({ - user_id: signup.user.id, - body: 'verify supabase-js path', - }); - assert.equal(insertError, null); - - const { data: rows, error: selectError } = await client - .from('todos') - .select('body') - .eq('user_id', signup.user.id); - assert.equal(selectError, null); - assert.deepEqual(rows, [{ body: 'verify supabase-js path' }]); - }); - - console.log('PASS supalite auth + supabase-js client'); -} - -async function smokePlatformBackendClose() { - const backend = await bootPlatformBackend({}); - await backend.query('select 1 as n'); - await backend.close(); - await assert.rejects(() => backend.query('select 1 as n')); - await backend.close(); - - console.log('PASS platform backend close disposes platform'); -} - -async function smokeEdgeAuthDbEval() { - const scorer = await loadScorer(EDGE_AUTH_DB_EVAL); - - await withBackend( - { projectSeedSql: seedPath(EDGE_AUTH_DB_EVAL, 'project.sql') }, - async (backend) => { - // Deploy the function via platform-lite's HTTP management API - const deployUrl = `${backend.url}/v1/projects/${backend.ref}/functions/deploy?slug=todo-create`; - const form = new FormData(); - form.append( - 'metadata', - JSON.stringify({ - name: 'todo-create', - verify_jwt: true, - entrypoint_path: 'index.ts', - }) - ); - form.append( - 'file', - new File([TODO_CREATE_SOURCE], 'index.ts', { - type: 'application/typescript', - }) - ); - - const deployRes = await fetch(deployUrl, { - method: 'POST', - headers: { Authorization: `Bearer ${backend.accessToken}` }, - body: form, - }); - assert.equal( - deployRes.status, - 201, - `deploy failed: ${await deployRes.text()}` - ); - - const result = await scorer(scorerCtx(backend)); - assert.equal(result.passed, true, checksMessage(result)); - } - ); - - console.log('PASS edge function auth + supabase-js DB scorer'); -} - -async function smokeLogsSeeding() { - await withBackend( - { logsSeedJsonl: seedPath(INVESTIGATE_LOGS_EVAL, 'logs.jsonl') }, - async ({ url, ref, accessToken }) => { - const logsUrl = `${url}/v1/projects/${ref}/analytics/endpoints/logs.all?sql=${encodeURIComponent('SELECT count(*)::int AS n FROM edge_logs')}`; - const res = await fetch(logsUrl, { - headers: { Authorization: `Bearer ${accessToken}` }, - }); - const body = (await res.json()) as { result: Array<{ n: number }> }; - assert(body.result[0] && body.result[0].n > 0, 'expected seeded logs'); - } - ); - - console.log('PASS logs seeding via platform-lite'); -} - -async function smokeInvestigateLogsEval() { - const scorer = await loadScorer(INVESTIGATE_LOGS_EVAL); - - await withBackend( - { logsSeedJsonl: seedPath(INVESTIGATE_LOGS_EVAL, 'logs.jsonl') }, - async (backend) => { - const result = await scorer( - scorerCtx(backend, { - agentReport: - 'stripe-webhook had the most errors with 9 errors out of 50 events.', - }) - ); - assert.equal(result.passed, true, checksMessage(result)); - } - ); - - console.log('PASS investigate logs scorer'); -} - -async function smokeInvestigateSecurityEval() { - const scorer = await loadScorer(INVESTIGATE_SECURITY_EVAL); - - await withBackend( - { - projectSeedSql: seedPath(INVESTIGATE_SECURITY_EVAL, 'project.sql'), - logsSeedJsonl: seedPath(INVESTIGATE_SECURITY_EVAL, 'logs.jsonl'), - }, - async (backend) => { - const { rows } = await backend.query(` -SELECT grantee FROM information_schema.role_table_grants -WHERE table_name = 'customer_payment_methods' AND privilege_type = 'SELECT' -ORDER BY grantee; - `); - assert( - (rows as Array<{ grantee: string }>).some( - (row) => row.grantee === 'anon' - ) - ); - - const report = [ - 'customer_payment_methods is exposed to anon.', - 'Fix by REVOKE SELECT ON customer_payment_methods FROM anon and enable row level security.', - ].join(' '); - - // The judge reads the transcript, so surface the report as assistant text. - const transcript: TranscriptPart[] = [ - { type: 'message', role: 'assistant', content: report }, - ]; - - const result = await scorer( - scorerCtx(backend, { agentReport: report, transcript }) - ); - assert.equal(result.passed, true, checksMessage(result)); - } - ); - - console.log('PASS investigate security scorer + database shim'); -} - -async function smokeFrontendBuildTooling() { - const source = join(ROOT, FRONTEND_EVAL, 'local'); - const workspace = join( - ROOT, - 'results', - '_smoke', - 'build-frontend-001-todos-app' - ); - rmSync(workspace, { recursive: true, force: true }); - mkdirSync(dirname(workspace), { recursive: true }); - cpSync(source, workspace, { - recursive: true, - filter: (src) => !src.endsWith('/EVAL.ts'), - }); - cpSync(join(ROOT, FRONTEND_EVAL, 'tests'), join(workspace, 'tests'), { - recursive: true, - }); - writeFileSync( - join(workspace, '.env.local'), - [ - 'VITE_SUPABASE_URL=http://supabase-evals.local', - 'VITE_SUPABASE_ANON_KEY=supabase-evals-anon-key', - '', - ].join('\n') - ); - writeFileSync(join(workspace, 'src', 'App.tsx'), GOOD_FRONTEND_APP); - - const build = await viteBuild(workspace); - assert.equal(build.ok, true, build.stderr || build.stdout); - const vitest = await vitestRun(workspace); - assert.equal(vitest.ok, true, vitest.stderr || vitest.stdout); - - console.log('PASS frontend vite/react/supalite build + test tooling'); -} - -async function main() { - await smokeClientRlsEval(); - await smokeFunctionsEval(); - await smokeServiceRoleBypassEval(); - await smokeSupaliteClient(); - await smokePlatformBackendClose(); - await smokeEdgeAuthDbEval(); - await smokeLogsSeeding(); - await smokeInvestigateLogsEval(); - await smokeInvestigateSecurityEval(); - await smokeFrontendBuildTooling(); - console.log('PASS framework smoke'); -} - -main().catch((error) => { - console.error(error); - process.exit(1); -}); - -const ORDER_TOTAL_SOURCE = ` -Deno.serve(async (req) => { - const json = (payload, status = 200) => - new Response(JSON.stringify(payload), { status, headers: { "content-type": "application/json" } }); - - if (req.method !== "POST") return json({ error: "method not allowed" }, 405); - - let body; - try { body = await req.json(); } catch { return json({ error: "invalid json" }, 400); } - if (!Array.isArray(body.items) || body.items.length === 0) return json({ error: "items are required" }, 400); - - let subtotal = 0; - for (const item of body.items) { - if (!Number.isFinite(item.unit_price_cents) || !Number.isFinite(item.quantity) || item.unit_price_cents <= 0 || item.quantity <= 0) { - return json({ error: "invalid item" }, 400); - } - subtotal += item.unit_price_cents * item.quantity; - } - - const couponDiscount = body.coupon === "WELCOME10" ? Math.min(Math.round(subtotal * 0.1), 2000) : 0; - const enterpriseDiscount = body.customer_tier === "enterprise" ? Math.round(subtotal * 0.15) : 0; - const discount = Math.max(couponDiscount, enterpriseDiscount); - const taxable = subtotal - discount; - const tax = Math.round(taxable * 0.0725); - - return json({ subtotal_cents: subtotal, discount_cents: discount, tax_cents: tax, total_cents: taxable + tax }); -}); -`; - -const TODO_CREATE_SOURCE = ` -import { createClient } from "@supabase/supabase-js"; - -Deno.serve(async (req) => { - const json = (payload, status = 200) => - new Response(JSON.stringify(payload), { - status, - headers: { "content-type": "application/json" }, - }); - - if (req.method !== "POST") return json({ error: "method not allowed" }, 405); - - const authorization = req.headers.get("authorization"); - if (!authorization) return json({ error: "missing authorization" }, 401); - - let body; - try { body = await req.json(); } catch { return json({ error: "invalid json" }, 400); } - if (typeof body.body !== "string" || body.body.trim() === "") return json({ error: "body is required" }, 400); - - const supabase = createClient( - Deno.env.get("SUPABASE_URL"), - Deno.env.get("SUPABASE_ANON_KEY"), - { global: { headers: { Authorization: authorization } } } - ); - - const { data, error } = await supabase - .from("todos") - .insert({ body: body.body }) - .select("id,body,user_id") - .single(); - - if (error) return json({ error: error.message }, 400); - return json(data, 201); -}); -`; - -const GOOD_FRONTEND_APP = ` -import { useState } from "react"; -import { createClient } from "@supabase/supabase-js"; - -type Todo = { id: string; body: string; done: boolean }; - -export const supabase = createClient( - import.meta.env.VITE_SUPABASE_URL, - import.meta.env.VITE_SUPABASE_ANON_KEY -); - -export default function App() { - const [email, setEmail] = useState(""); - const [password, setPassword] = useState(""); - const [newTodo, setNewTodo] = useState(""); - const [signedIn, setSignedIn] = useState(false); - const [todos, setTodos] = useState([]); - const [error, setError] = useState(""); - - async function loadTodos() { - const { data, error } = await supabase.from("todos").select("id,body,done").order("created_at", { ascending: true }); - if (error) throw error; - setTodos(data ?? []); - } - - async function handleSignIn(event: React.FormEvent) { - event.preventDefault(); - setError(""); - const { error } = await supabase.auth.signInWithPassword({ email, password }); - if (error) { setError(error.message); return; } - setSignedIn(true); - await loadTodos(); - } - - async function handleAddTodo(event: React.FormEvent) { - event.preventDefault(); - const body = newTodo.trim(); - if (!body) return; - setError(""); - const { data, error } = await supabase.from("todos").insert({ body }).select("id,body,done").single(); - if (error) { setError(error.message); return; } - setTodos((current) => [...current, data]); - setNewTodo(""); - } - - async function handleToggleTodo(todo: Todo) { - setError(""); - const { data, error } = await supabase.from("todos").update({ done: !todo.done }).eq("id", todo.id).select("id,body,done").single(); - if (error) { setError(error.message); return; } - setTodos((current) => current.map((item) => (item.id === todo.id ? data : item))); - } - - return ( -
-

Todos

-
- setEmail(e.target.value)} /> - setPassword(e.target.value)} /> - -
- {signedIn ?

Signed in

: null} - {error ?

{error}

: null} -
- setNewTodo(e.target.value)} /> - -
-
    - {todos.map((todo) => ( -
  • - -
    • - ))} -
-
- ); -} -`; diff --git a/apps/framework/tsconfig.json b/apps/framework/tsconfig.json index 0b21c008..4a311453 100644 --- a/apps/framework/tsconfig.json +++ b/apps/framework/tsconfig.json @@ -5,6 +5,8 @@ "harness", "shims", "scripts", - "../../evals/**/EVAL.ts" + "../../evals/**/EVAL.ts", + "../../evals/*/EVAL.test.ts", + "../../test-utils" ] } diff --git a/apps/framework/vitest.config.ts b/apps/framework/vitest.config.ts new file mode 100644 index 00000000..28dfc644 --- /dev/null +++ b/apps/framework/vitest.config.ts @@ -0,0 +1,17 @@ +import { defineConfig } from 'vitest/config'; + +export default defineConfig({ + test: { + environment: 'node', + // Booting platform-lite and shelling out to vite/vitest is slow. + testTimeout: 120000, + hookTimeout: 120000, + include: [ + 'harness/**/*.test.ts', + // Scorer tests only. Deliberately not `evals/**/*.test.ts`: several evals + // ship withheld tests under `evals//tests/` that exist to score an + // agent's workspace, and must never run as part of this suite. + '../../evals/*/EVAL.test.ts', + ], + }, +}); diff --git a/evals/build-frontend-001-todos-app/reference/App.tsx b/evals/build-frontend-001-todos-app/reference/App.tsx new file mode 100644 index 00000000..435b7820 --- /dev/null +++ b/evals/build-frontend-001-todos-app/reference/App.tsx @@ -0,0 +1,138 @@ +/** + * Reference solution for this eval, used by the harness's own test to prove + * `viteBuild` + `vitestRun` can build and pass a known-good workspace. It is + * NOT part of `local/` (that is the agent's starting point, and shipping this + * would hand over the answer) and it is never copied into a sandbox. + * + * It must keep satisfying the withheld tests in `../tests/`, which drive it by + * `data-testid`. + */ +import { useState } from 'react'; +import { createClient } from '@supabase/supabase-js'; + +type Todo = { id: string; body: string; done: boolean }; + +export const supabase = createClient( + import.meta.env.VITE_SUPABASE_URL, + import.meta.env.VITE_SUPABASE_ANON_KEY +); + +export default function App() { + const [email, setEmail] = useState(''); + const [password, setPassword] = useState(''); + const [newTodo, setNewTodo] = useState(''); + const [signedIn, setSignedIn] = useState(false); + const [todos, setTodos] = useState([]); + const [error, setError] = useState(''); + + async function loadTodos() { + const { data, error } = await supabase + .from('todos') + .select('id,body,done') + .order('created_at', { ascending: true }); + if (error) throw error; + setTodos(data ?? []); + } + + async function handleSignIn(event: React.FormEvent) { + event.preventDefault(); + setError(''); + const { error } = await supabase.auth.signInWithPassword({ + email, + password, + }); + if (error) { + setError(error.message); + return; + } + setSignedIn(true); + await loadTodos(); + } + + async function handleAddTodo(event: React.FormEvent) { + event.preventDefault(); + const body = newTodo.trim(); + if (!body) return; + setError(''); + const { data, error } = await supabase + .from('todos') + .insert({ body }) + .select('id,body,done') + .single(); + if (error) { + setError(error.message); + return; + } + setTodos((current) => [...current, data]); + setNewTodo(''); + } + + async function handleToggleTodo(todo: Todo) { + setError(''); + const { data, error } = await supabase + .from('todos') + .update({ done: !todo.done }) + .eq('id', todo.id) + .select('id,body,done') + .single(); + if (error) { + setError(error.message); + return; + } + setTodos((current) => + current.map((item) => (item.id === todo.id ? data : item)) + ); + } + + return ( +
+

Todos

+
+ setEmail(e.target.value)} + /> + setPassword(e.target.value)} + /> + +
+ {signedIn ?

Signed in

: null} + {error ?

{error}

: null} +
+ setNewTodo(e.target.value)} + /> + +
+
    + {todos.map((todo) => ( +
  • + +
    • + ))} +
+
+ ); +} diff --git a/evals/build-frontend-001-todos-app/reference/tsconfig.json b/evals/build-frontend-001-todos-app/reference/tsconfig.json new file mode 100644 index 00000000..24e1a843 --- /dev/null +++ b/evals/build-frontend-001-todos-app/reference/tsconfig.json @@ -0,0 +1,7 @@ +{ + "extends": "../local/tsconfig.json", + "compilerOptions": { + "types": ["vite/client"] + }, + "include": ["."] +} diff --git a/evals/build-functions-001-order-total/EVAL.test.ts b/evals/build-functions-001-order-total/EVAL.test.ts new file mode 100644 index 00000000..b3b64969 --- /dev/null +++ b/evals/build-functions-001-order-total/EVAL.test.ts @@ -0,0 +1,52 @@ +import { expect, test } from 'vitest'; +import { + checksMessage, + deployFunction, + scorerCtx, + withBackend, +} from '../../test-utils/scorer-test-kit.js'; +import scorer from './EVAL.js'; + +const ORDER_TOTAL_SOURCE = ` +Deno.serve(async (req) => { + const json = (payload, status = 200) => + new Response(JSON.stringify(payload), { status, headers: { "content-type": "application/json" } }); + + if (req.method !== "POST") return json({ error: "method not allowed" }, 405); + + let body; + try { body = await req.json(); } catch { return json({ error: "invalid json" }, 400); } + if (!Array.isArray(body.items) || body.items.length === 0) return json({ error: "items are required" }, 400); + + let subtotal = 0; + for (const item of body.items) { + if (!Number.isFinite(item.unit_price_cents) || !Number.isFinite(item.quantity) || item.unit_price_cents <= 0 || item.quantity <= 0) { + return json({ error: "invalid item" }, 400); + } + subtotal += item.unit_price_cents * item.quantity; + } + + const couponDiscount = body.coupon === "WELCOME10" ? Math.min(Math.round(subtotal * 0.1), 2000) : 0; + const enterpriseDiscount = body.customer_tier === "enterprise" ? Math.round(subtotal * 0.15) : 0; + const discount = Math.max(couponDiscount, enterpriseDiscount); + const taxable = subtotal - discount; + const tax = Math.round(taxable * 0.0725); + + return json({ subtotal_cents: subtotal, discount_cents: discount, tax_cents: tax, total_cents: taxable + tax }); +}); +`; + +test('fails before the function exists, passes once it is deployed', async () => { + await withBackend({}, async (backend) => { + const before = await scorer(scorerCtx(backend)); + expect(before.passed).toBe(false); + expect(checksMessage(before)).toMatch(/function not found/i); + + await deployFunction(backend, 'order-total', ORDER_TOTAL_SOURCE, { + verifyJwt: false, + }); + + const after = await scorer(scorerCtx(backend)); + expect(after.passed, checksMessage(after)).toBe(true); + }); +}); diff --git a/evals/build-functions-002-edge-auth-db/EVAL.test.ts b/evals/build-functions-002-edge-auth-db/EVAL.test.ts new file mode 100644 index 00000000..97d8937e --- /dev/null +++ b/evals/build-functions-002-edge-auth-db/EVAL.test.ts @@ -0,0 +1,59 @@ +import { expect, test } from 'vitest'; +import { + checksMessage, + deployFunction, + scorerCtx, + seedPath, + withBackend, +} from '../../test-utils/scorer-test-kit.js'; +import scorer from './EVAL.js'; + +const TODO_CREATE_SOURCE = ` +import { createClient } from "@supabase/supabase-js"; + +Deno.serve(async (req) => { + const json = (payload, status = 200) => + new Response(JSON.stringify(payload), { + status, + headers: { "content-type": "application/json" }, + }); + + if (req.method !== "POST") return json({ error: "method not allowed" }, 405); + + const authorization = req.headers.get("authorization"); + if (!authorization) return json({ error: "missing authorization" }, 401); + + let body; + try { body = await req.json(); } catch { return json({ error: "invalid json" }, 400); } + if (typeof body.body !== "string" || body.body.trim() === "") return json({ error: "body is required" }, 400); + + const supabase = createClient( + Deno.env.get("SUPABASE_URL"), + Deno.env.get("SUPABASE_ANON_KEY"), + { global: { headers: { Authorization: authorization } } } + ); + + const { data, error } = await supabase + .from("todos") + .insert({ body: body.body }) + .select("id,body,user_id") + .single(); + + if (error) return json({ error: error.message }, 400); + return json(data, 201); +}); +`; + +test('passes for a JWT-verifying function that inserts as the caller', async () => { + await withBackend( + { projectSeedSql: seedPath(import.meta.url, 'project.sql') }, + async (backend) => { + await deployFunction(backend, 'todo-create', TODO_CREATE_SOURCE, { + verifyJwt: true, + }); + + const result = await scorer(scorerCtx(backend)); + expect(result.passed, checksMessage(result)).toBe(true); + } + ); +}); diff --git a/evals/build-functions-004-service-role-bypass/EVAL.test.ts b/evals/build-functions-004-service-role-bypass/EVAL.test.ts new file mode 100644 index 00000000..19c2b04f --- /dev/null +++ b/evals/build-functions-004-service-role-bypass/EVAL.test.ts @@ -0,0 +1,141 @@ +import { describe, expect, test } from 'vitest'; +import { + checksMessage, + failedCheckNames, + functionResponse, +} from '../../test-utils/scorer-test-kit.js'; +import type { + EdgeFunctionsInvokeResult, + ToolEvalContext, +} from '@supabase-evals/core'; +import scorer from './EVAL.js'; + +/** + * Unlike the other scorer tests, this one fakes the `ToolEvalContext` instead + * of booting a real backend: the scorer's *decision logic* (which statuses and + * bodies pass) is what we want to pin, and driving that through a real stack + * would mean deploying six edge-function variants. + */ +function fakeCtx(responses: EdgeFunctionsInvokeResult[]): ToolEvalContext { + const authResult = (id: string, accessToken: string) => ({ + data: { user: { id }, session: { access_token: accessToken } }, + error: null, + }); + const clientA = { + auth: { signUp: async () => authResult('user-a', 'token-a') }, + } as unknown as ToolEvalContext['client']; + const clientB = { + auth: { signUp: async () => authResult('user-b', 'token-b') }, + } as unknown as ToolEvalContext['client']; + + return { + mgmt: {} as ToolEvalContext['mgmt'], + ref: 'test-ref', + client: clientA, + getClient: () => clientB, + query: async () => ({ rows: [] }), + invokeFunction: async () => { + const response = responses.shift(); + if (!response) throw new Error('missing fake function response'); + return response; + }, + toolCalls: [], + transcript: [], + }; +} + +type Role = 'missingAuth' | 'ownNotes' | 'aRequestsB' | 'bRequestsA'; + +/** + * Responses are consumed in the scorer's invocation order (missingAuth, + * ownNotes, aRequestsB, bRequestsA); overrides are keyed by role. + */ +function runScorer( + overrides: Partial> = {} +) { + return scorer( + fakeCtx([ + overrides.missingAuth ?? functionResponse(401), + overrides.ownNotes ?? + functionResponse(200, 'user A private note', ['token-a']), + overrides.aRequestsB ?? functionResponse(401, 'unauthorized'), + overrides.bRequestsA ?? functionResponse(403, 'forbidden'), + ]) + ); +} + +describe('secure shapes the scorer must accept', () => { + test('rejects anonymous access, serves own note, denies cross-user reads', async () => { + const result = await runScorer(); + + expect(result.passed, checksMessage(result)).toBe(true); + expect(checksMessage(result)).toMatch(/"notes":"status=401"/); + expect(checksMessage(result)).toMatch(/"notes":"status=200"/); + expect(checksMessage(result)).toMatch(/"notes":"status=403"/); + expect(checksMessage(result)).toMatch( + /"notes":"bearer_tokens=1, all_match=true"/ + ); + }); + + test('ignoring the spoofed user_id and returning the caller their own note', async () => { + const result = await runScorer({ + aRequestsB: functionResponse(200, 'user A private note', ['token-a']), + bRequestsA: functionResponse(200, 'user B private note', ['token-b']), + }); + + expect(result.passed, checksMessage(result)).toBe(true); + }); + + test('404 from an RLS-scoped read that returns no rows', async () => { + const result = await runScorer({ + aRequestsB: functionResponse(404, 'not found'), + bRequestsA: functionResponse(404, 'not found'), + }); + + expect(result.passed, checksMessage(result)).toBe(true); + }); +}); + +describe('leaks and bypasses the scorer must reject', () => { + test('a leaked note fails regardless of status', async () => { + const result = await runScorer({ + aRequestsB: functionResponse(200, 'user B private note'), + bRequestsA: functionResponse(403, 'user A private note'), + }); + + expect(result.passed, checksMessage(result)).toBe(false); + expect(failedCheckNames(result)).toEqual([ + 'user A cannot force-read user B note', + 'user B cannot force-read user A note', + ]); + }); + + test('serving data to an unauthenticated caller fails without any leak', async () => { + const result = await runScorer({ missingAuth: functionResponse(200) }); + + expect(result.passed, checksMessage(result)).toBe(false); + expect(failedCheckNames(result)).toEqual(['rejects missing auth']); + }); + + test('a 401 that still echoes a note is a leak, not a denial', async () => { + const result = await runScorer({ + missingAuth: functionResponse(401, 'user A private note'), + }); + + expect(result.passed, checksMessage(result)).toBe(false); + expect(failedCheckNames(result)).toEqual(['rejects missing auth']); + }); + + test('reading via the service-role key instead of the caller JWT fails', async () => { + const result = await runScorer({ + ownNotes: functionResponse(200, 'user A private note', [ + 'service-role-key', + ]), + }); + + expect(result.passed, checksMessage(result)).toBe(false); + expect(failedCheckNames(result)).toEqual([ + "reads only with the caller's JWT", + ]); + }); +}); diff --git a/evals/build-rls-002-own-todos-client/EVAL.test.ts b/evals/build-rls-002-own-todos-client/EVAL.test.ts new file mode 100644 index 00000000..7914e139 --- /dev/null +++ b/evals/build-rls-002-own-todos-client/EVAL.test.ts @@ -0,0 +1,27 @@ +import { expect, test } from 'vitest'; +import { + checksMessage, + scorerCtx, + seedPath, + withBackend, +} from '../../test-utils/scorer-test-kit.js'; +import scorer from './EVAL.js'; + +test('passes once per-user RLS policies are in place', async () => { + await withBackend( + { projectSeedSql: seedPath(import.meta.url, 'project.sql') }, + async (backend) => { + await backend.query(` +ALTER TABLE todos ENABLE ROW LEVEL SECURITY; + +CREATE POLICY "users can read own todos" ON todos FOR SELECT TO authenticated USING (user_id = auth.uid()); +CREATE POLICY "users can insert own todos" ON todos FOR INSERT TO authenticated WITH CHECK (user_id = auth.uid()); +CREATE POLICY "users can update own todos" ON todos FOR UPDATE TO authenticated USING (user_id = auth.uid()) WITH CHECK (user_id = auth.uid()); +CREATE POLICY "users can delete own todos" ON todos FOR DELETE TO authenticated USING (user_id = auth.uid()); + `); + + const result = await scorer(scorerCtx(backend)); + expect(result.passed, checksMessage(result)).toBe(true); + } + ); +}); diff --git a/evals/investigate-logs-001-top-error-function/EVAL.test.ts b/evals/investigate-logs-001-top-error-function/EVAL.test.ts new file mode 100644 index 00000000..86a0a7b0 --- /dev/null +++ b/evals/investigate-logs-001-top-error-function/EVAL.test.ts @@ -0,0 +1,24 @@ +import { expect, test } from 'vitest'; +import { + checksMessage, + scorerCtx, + seedPath, + withBackend, +} from '../../test-utils/scorer-test-kit.js'; +import scorer from './EVAL.js'; + +test('passes for a report naming the top error function and its count', async () => { + await withBackend( + { logsSeedJsonl: seedPath(import.meta.url, 'logs.jsonl') }, + async (backend) => { + const result = await scorer( + scorerCtx(backend, { + agentReport: + 'stripe-webhook had the most errors with 9 errors out of 50 events.', + }) + ); + + expect(result.passed, checksMessage(result)).toBe(true); + } + ); +}); diff --git a/evals/investigate-security-001-public-table/EVAL.test.ts b/evals/investigate-security-001-public-table/EVAL.test.ts new file mode 100644 index 00000000..b458fdce --- /dev/null +++ b/evals/investigate-security-001-public-table/EVAL.test.ts @@ -0,0 +1,82 @@ +import { judge } from '@supabase-evals/core'; +import type * as Core from '@supabase-evals/core'; +import { expect, test, vi } from 'vitest'; +import { + checksMessage, + failedCheckNames, + scorerCtx, + seedPath, + withBackend, +} from '../../test-utils/scorer-test-kit.js'; +import type { TranscriptPart } from '@supabase-evals/core'; +import scorer from './EVAL.js'; + +// The scorer's fourth check calls a real OpenAI judge, which needs +// OPENAI_API_KEY and bills per run. Stub only `judge` so this suite stays +// hermetic in CI; every other core export the scorer uses stays real, so the +// deterministic checks are still exercised for real. +vi.mock('@supabase-evals/core', async (importOriginal) => ({ + ...(await importOriginal()), + judge: vi.fn(async () => ({ passed: true, notes: 'stubbed verdict' })), +})); + +const GOOD_REPORT = [ + 'customer_payment_methods is exposed to anon.', + 'Fix by REVOKE SELECT ON customer_payment_methods FROM anon and enable row level security.', +].join(' '); + +const VAGUE_REPORT = + 'Some tables look insecure, you should tighten permissions.'; + +// The judge reads the transcript, so surface the report as assistant text. +const asTranscript = (report: string): TranscriptPart[] => [ + { type: 'message', role: 'assistant', content: report }, +]; + +test('scores a report that names the exposure and the fix', async () => { + await withBackend( + { + projectSeedSql: seedPath(import.meta.url, 'project.sql'), + logsSeedJsonl: seedPath(import.meta.url, 'logs.jsonl'), + }, + async (backend) => { + const { rows } = await backend.query(` +SELECT grantee FROM information_schema.role_table_grants +WHERE table_name = 'customer_payment_methods' AND privilege_type = 'SELECT' +ORDER BY grantee; + `); + expect( + (rows as Array<{ grantee: string }>).some( + (row) => row.grantee === 'anon' + ), + 'seed should expose customer_payment_methods to anon' + ).toBe(true); + + const passing = await scorer( + scorerCtx(backend, { + agentReport: GOOD_REPORT, + transcript: asTranscript(GOOD_REPORT), + }) + ); + expect(passing.passed, checksMessage(passing)).toBe(true); + + // The judge scores the transcript, not the report field. + expect(vi.mocked(judge).mock.calls[0]?.[0].input).toContain( + 'customer_payment_methods' + ); + + const vague = await scorer( + scorerCtx(backend, { + agentReport: VAGUE_REPORT, + transcript: asTranscript(VAGUE_REPORT), + }) + ); + expect(vague.passed).toBe(false); + expect(failedCheckNames(vague)).toEqual([ + 'named the vulnerable table', + 'mentioned the anon role', + 'proposed a concrete fix', + ]); + } + ); +}); diff --git a/package.json b/package.json index ab7f505e..f089b339 100644 --- a/package.json +++ b/package.json @@ -11,7 +11,7 @@ "eval:force": "pnpm --filter @supabase-evals/framework eval:force", "test:framework": "pnpm --filter @supabase-evals/framework test:framework", "export-results": "pnpm --filter @supabase-evals/framework export-results", - "typecheck": "pnpm --filter @supabase-evals/framework typecheck && pnpm --filter @supabase-evals/web typecheck", + "typecheck": "pnpm --filter @supabase-evals/framework typecheck && pnpm --filter @supabase-evals/web typecheck && tsc -p evals/build-frontend-001-todos-app/reference", "web": "pnpm --filter @supabase-evals/web dev", "web:build": "pnpm --filter @supabase-evals/web build", "demo:mcp": "pnpm --filter @supabase-evals/framework demo:mcp", @@ -29,8 +29,19 @@ "@supabase-evals/core": "workspace:*", "@supabase-evals/sandbox": "workspace:*", "@types/common-tags": "^1.8.4", + "@types/react": "^19.2.7", "@types/node": "catalog:", - "typescript": "catalog:" + "typescript": "catalog:", + "@supabase/lite": "catalog:", + "@supabase/supabase-js": "catalog:", + "@testing-library/jest-dom": "^6.9.1", + "@testing-library/react": "^16.3.2", + "@vitejs/plugin-react": "catalog:", + "happy-dom": "^20.9.0", + "react": "^19.2.5", + "react-dom": "^19.2.5", + "vite": "catalog:", + "vitest": "catalog:" }, "engines": { "node": ">=22", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index de9e3fa6..8fc546ae 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -81,15 +81,48 @@ importers: '@supabase-evals/sandbox': specifier: workspace:* version: link:packages/sandbox + '@supabase/lite': + specifier: 'catalog:' + version: 0.7.1-next.3(@supabase/supabase-js@2.108.1)(hono@4.12.25)(vite@7.3.5(@types/node@22.19.20)(jiti@2.7.0)(lightningcss@1.32.0)(tsx@4.22.4)) + '@supabase/supabase-js': + specifier: 'catalog:' + version: 2.108.1 + '@testing-library/jest-dom': + specifier: ^6.9.1 + version: 6.9.1 + '@testing-library/react': + specifier: ^16.3.2 + version: 16.3.2(@testing-library/dom@10.4.1)(@types/react-dom@19.2.3(@types/react@19.2.17))(@types/react@19.2.17)(react-dom@19.2.7(react@19.2.7))(react@19.2.7) '@types/common-tags': specifier: ^1.8.4 version: 1.8.4 '@types/node': specifier: 'catalog:' version: 22.19.20 + '@types/react': + specifier: ^19.2.7 + version: 19.2.17 + '@vitejs/plugin-react': + specifier: 'catalog:' + version: 5.2.0(vite@7.3.5(@types/node@22.19.20)(jiti@2.7.0)(lightningcss@1.32.0)(tsx@4.22.4)) + happy-dom: + specifier: ^20.9.0 + version: 20.10.2 + react: + specifier: ^19.2.5 + version: 19.2.7 + react-dom: + specifier: ^19.2.5 + version: 19.2.7(react@19.2.7) typescript: specifier: 'catalog:' version: 5.9.3 + vite: + specifier: 'catalog:' + version: 7.3.5(@types/node@22.19.20)(jiti@2.7.0)(lightningcss@1.32.0)(tsx@4.22.4) + vitest: + specifier: 'catalog:' + version: 4.1.8(@opentelemetry/api@1.9.1)(@types/node@22.19.20)(happy-dom@20.10.2)(vite@7.3.5(@types/node@22.19.20)(jiti@2.7.0)(lightningcss@1.32.0)(tsx@4.22.4)) apps/framework: dependencies: @@ -114,36 +147,9 @@ importers: '@supabase-evals/sandbox': specifier: workspace:* version: link:../../packages/sandbox - '@supabase/lite': - specifier: 'catalog:' - version: 0.7.1-next.3(@supabase/supabase-js@2.108.1)(hono@4.12.25)(vite@7.3.5(@types/node@22.19.20)(jiti@2.7.0)(lightningcss@1.32.0)(tsx@4.22.4)) - '@supabase/supabase-js': - specifier: 'catalog:' - version: 2.108.1 - '@testing-library/jest-dom': - specifier: ^6.9.1 - version: 6.9.1 - '@testing-library/react': - specifier: ^16.3.2 - version: 16.3.2(@testing-library/dom@10.4.1)(@types/react-dom@19.2.3(@types/react@19.2.17))(@types/react@19.2.17)(react-dom@19.2.7(react@19.2.7))(react@19.2.7) - '@vitejs/plugin-react': - specifier: 'catalog:' - version: 5.2.0(vite@7.3.5(@types/node@22.19.20)(jiti@2.7.0)(lightningcss@1.32.0)(tsx@4.22.4)) ai: specifier: 'catalog:' version: 6.0.199(zod@4.4.3) - happy-dom: - specifier: ^20.9.0 - version: 20.10.2 - react: - specifier: ^19.2.5 - version: 19.2.7 - react-dom: - specifier: ^19.2.5 - version: 19.2.7(react@19.2.7) - vite: - specifier: 'catalog:' - version: 7.3.5(@types/node@22.19.20)(jiti@2.7.0)(lightningcss@1.32.0)(tsx@4.22.4) vitest: specifier: 'catalog:' version: 4.1.8(@opentelemetry/api@1.9.1)(@types/node@22.19.20)(happy-dom@20.10.2)(vite@7.3.5(@types/node@22.19.20)(jiti@2.7.0)(lightningcss@1.32.0)(tsx@4.22.4)) diff --git a/test-utils/scorer-test-kit.ts b/test-utils/scorer-test-kit.ts new file mode 100644 index 00000000..0a93e3d7 --- /dev/null +++ b/test-utils/scorer-test-kit.ts @@ -0,0 +1,130 @@ +/** + * Helpers for testing eval scorers against a real platform-lite backend. + * + * Lives at the repo root rather than inside a package on purpose: it hardcodes + * the monorepo layout (`ROOT`, `seedPath`) and is test-only, so shipping it + * from `@supabase-evals/core` would make a production package own repo-shaped + * test infrastructure. Canonical types come from core; nothing here reaches + * into an app's internals. + * + * Scorer tests live next to the scorer they cover (`evals//EVAL.test.ts`) + * and import it directly; harness tests live beside the harness module they + * cover. Both import from here so a test stays a description of the scenario + * rather than a pile of backend plumbing. + */ +import { dirname, join } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { bootPlatformBackend } from '@supabase-evals/core'; +import type { + EdgeFunctionsInvokeResult, + PlatformBackend, + ToolEvalContext, + TranscriptPart, +} from '@supabase-evals/core'; + +/** Repo root, from `test-utils/`. */ +export const ROOT = join(dirname(fileURLToPath(import.meta.url)), '..'); + +/** + * Path to a file in an eval's `remote/` seed directory, resolved from the + * calling test's own location. Pass `import.meta.url`: a colocated + * `EVAL.test.ts` always seeds its own scenario, where a hand-written directory + * string can silently point at a different eval's fixtures and still pass. + */ +export function seedPath(testUrl: string, file: string): string { + return join(dirname(fileURLToPath(testUrl)), 'remote', file); +} + +/** A `ToolEvalContext` backed by a live platform-lite project. */ +export function scorerCtx( + backend: PlatformBackend, + extra?: { agentReport?: string; transcript?: TranscriptPart[] } +): ToolEvalContext { + return { + mgmt: backend.mgmt, + ref: backend.ref, + client: backend.client, + getClient: backend.getClient, + query: backend.query, + invokeFunction: backend.invokeFunction, + toolCalls: [], + transcript: extra?.transcript ?? [], + agentReport: extra?.agentReport, + }; +} + +/** Boot a backend, run `fn` against it, and always close it. */ +export async function withBackend( + opts: { projectSeedSql?: string; logsSeedJsonl?: string }, + fn: (backend: PlatformBackend) => Promise +): Promise { + const backend = await bootPlatformBackend(opts); + try { + return await fn(backend); + } finally { + await backend.close(); + } +} + +/** Serialized checks, for use as an assertion failure message. */ +export function checksMessage(result: { checks?: unknown[] }) { + return JSON.stringify(result.checks ?? []); +} + +/** Names of the checks that failed, for asserting on *which* check broke. */ +export function failedCheckNames(result: { + checks?: { name: string; passed: boolean }[]; +}) { + return ( + result.checks + ?.filter((check) => !check.passed) + .map((check) => check.name) ?? [] + ); +} + +/** An edge function response, as the scorer's `invokeFunction` would see it. */ +export function functionResponse( + status: number, + body = '', + outboundBearerTokens: string[] = [] +): EdgeFunctionsInvokeResult { + return { type: 'response', status, headers: {}, body, outboundBearerTokens }; +} + +/** + * Deploy an edge function through platform-lite's management API, the same + * path the real harness uses. + */ +export async function deployFunction( + backend: PlatformBackend, + slug: string, + source: string, + opts: { verifyJwt: boolean } +): Promise { + const form = new FormData(); + form.append( + 'metadata', + JSON.stringify({ + name: slug, + verify_jwt: opts.verifyJwt, + entrypoint_path: 'index.ts', + }) + ); + form.append( + 'file', + new File([source], 'index.ts', { type: 'application/typescript' }) + ); + + const res = await fetch( + `${backend.url}/v1/projects/${backend.ref}/functions/deploy?slug=${slug}`, + { + method: 'POST', + headers: { Authorization: `Bearer ${backend.accessToken}` }, + body: form, + } + ); + + if (res.status !== 201) { + throw new Error(`deploy of ${slug} failed: ${await res.text()}`); + } +}