From 216ca870255f586ee737507c00a7c5a60a197e7a Mon Sep 17 00:00:00 2001 From: Katerina Skroumpelou Date: Mon, 27 Jul 2026 18:20:18 +0300 Subject: [PATCH 1/7] feat: supabase-js SDK evals (auth flow, relational report, key migration) --- .gitignore | 2 + .../EVAL.ts | 238 ++++++++++++++++++ .../PROMPT.md | 31 +++ .../README.md | 19 ++ .../local/app/package.json | 5 + .../local/app/src/auth.mjs | 36 +++ .../local/supabase/config.toml | 165 ++++++++++++ .../migrations/0000_profiles_schema.sql | 53 ++++ .../EVAL.ts | 214 ++++++++++++++++ .../PROMPT.md | 27 ++ .../local/app/package.json | 5 + .../local/app/report.mjs | 22 ++ .../local/supabase/config.toml | 165 ++++++++++++ .../migrations/0000_orders_schema.sql | 69 +++++ .../EVAL.ts | 230 +++++++++++++++++ .../PROMPT.md | 26 ++ .../README.md | 24 ++ .../local/app/.env | 4 + .../local/app/package.json | 12 + .../local/app/posts.mjs | 19 ++ .../local/app/stats.mjs | 19 ++ .../local/supabase/config.toml | 165 ++++++++++++ .../supabase/migrations/0000_posts_schema.sql | 27 ++ 23 files changed, 1577 insertions(+) create mode 100644 evals/build-auth-001-email-password-flow/EVAL.ts create mode 100644 evals/build-auth-001-email-password-flow/PROMPT.md create mode 100644 evals/build-auth-001-email-password-flow/README.md create mode 100644 evals/build-auth-001-email-password-flow/local/app/package.json create mode 100644 evals/build-auth-001-email-password-flow/local/app/src/auth.mjs create mode 100644 evals/build-auth-001-email-password-flow/local/supabase/config.toml create mode 100644 evals/build-auth-001-email-password-flow/local/supabase/migrations/0000_profiles_schema.sql create mode 100644 evals/build-dataapi-001-relational-report/EVAL.ts create mode 100644 evals/build-dataapi-001-relational-report/PROMPT.md create mode 100644 evals/build-dataapi-001-relational-report/local/app/package.json create mode 100644 evals/build-dataapi-001-relational-report/local/app/report.mjs create mode 100644 evals/build-dataapi-001-relational-report/local/supabase/config.toml create mode 100644 evals/build-dataapi-001-relational-report/local/supabase/migrations/0000_orders_schema.sql create mode 100644 evals/resolve-sdk-001-legacy-key-migration/EVAL.ts create mode 100644 evals/resolve-sdk-001-legacy-key-migration/PROMPT.md create mode 100644 evals/resolve-sdk-001-legacy-key-migration/README.md create mode 100644 evals/resolve-sdk-001-legacy-key-migration/local/app/.env create mode 100644 evals/resolve-sdk-001-legacy-key-migration/local/app/package.json create mode 100644 evals/resolve-sdk-001-legacy-key-migration/local/app/posts.mjs create mode 100644 evals/resolve-sdk-001-legacy-key-migration/local/app/stats.mjs create mode 100644 evals/resolve-sdk-001-legacy-key-migration/local/supabase/config.toml create mode 100644 evals/resolve-sdk-001-legacy-key-migration/local/supabase/migrations/0000_posts_schema.sql diff --git a/.gitignore b/.gitignore index 932a6aea..23951d46 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,8 @@ node_modules/ dist/ .env .env*.local +# eval seed data may include a .env with well-known local demo keys +!evals/*/local/**/.env .DS_Store results/*/ .sync-tmp/ diff --git a/evals/build-auth-001-email-password-flow/EVAL.ts b/evals/build-auth-001-email-password-flow/EVAL.ts new file mode 100644 index 00000000..f658d512 --- /dev/null +++ b/evals/build-auth-001-email-password-flow/EVAL.ts @@ -0,0 +1,238 @@ +import { + type CheckResult, + type LocalStackEvalContext, + type LocalStackScorer, +} from '@supabase-evals/core'; + +// Email+password auth benchmark: the prompt asks the agent to finish the +// app's auth layer (app/src/auth.mjs) against the running local stack and +// never names supabase-js — the "uses @supabase/supabase-js" check is +// GATING. On top of SDK discovery it measures driving auth correctly: +// passing the display name as signup user metadata (the seeded profiles +// trigger reads raw_user_meta_data), handling bad credentials gracefully, +// and reading the RLS-scoped profile with the session actually attached. + +const APP_DIR = 'app'; +const DRIVER = 'eval-driver.mjs'; +const DRIVER_MARKER = '___EVAL_DRIVER___'; + +// Runs inside the sandbox, in one process, exactly like the app would use +// the module: sign up, fail a sign-in, sign in, read the profile. +const DRIVER_SOURCE = ` +import { signUp, signIn, getMyProfile } from './src/auth.mjs'; + +const [email, password, displayName, wrongPassword] = process.argv.slice(2); +const out = {}; +const step = async (name, fn) => { + try { + out[name] = await fn(); + } catch (error) { + out[name] = { + threw: String(error instanceof Error ? error.message : error), + }; + } +}; +await step('signUp', () => signUp(email, password, displayName)); +await step('signInWrong', () => signIn(email, wrongPassword)); +await step('signIn', () => signIn(email, password)); +await step('profile', () => getMyProfile()); +console.log('${DRIVER_MARKER}' + JSON.stringify(out)); +process.exit(0); +`; + +interface DriverStep { + userId?: unknown; + displayName?: unknown; + plan?: unknown; + error?: unknown; + threw?: unknown; +} + +const scorer: LocalStackScorer = async (ctx) => { + const checks: CheckResult[] = []; + try { + // Unique suffix keeps signup emails collision-free across attempts. + const suffix = Date.now().toString(36); + const email = `alex-${suffix}@example.com`; + const password = 'correct-horse-battery'; + const wrongPassword = 'wrong-horse-battery'; + const displayName = 'Alex Doe'; + + const status = await readStatus(ctx); + const apiUrl = str(status.API_URL); + const publishableKey = str(status.PUBLISHABLE_KEY); + if (!apiUrl || !publishableKey) { + return fail( + 'read stack config from `supabase status`', + `missing API_URL/PUBLISHABLE_KEY — is the stack running on a new-enough CLI? got keys: ${Object.keys(status).join(', ')}` + ); + } + + const written = await writeDriver(ctx); + if (!written.ok) { + return fail( + 'installed the eval driver', + written.stderr.trim() || written.stdout.trim() + ); + } + const run = await ctx.exec( + `cd ${APP_DIR} && SUPABASE_URL="${apiUrl}" SUPABASE_PUBLISHABLE_KEY="${publishableKey}" ` + + `node ${DRIVER} "${email}" "${password}" "${displayName}" "${wrongPassword}"`, + { timeoutMs: 60_000 } + ); + const out = parseDriverOutput(run.stdout); + checks.push({ + name: 'auth module loads and the driver completes', + passed: out !== undefined, + notes: out + ? 'driver produced a result' + : `no driver output — ${preview(run.stderr || run.stdout)}`, + }); + + const signUp = (out?.signUp ?? {}) as DriverStep; + const signInWrong = (out?.signInWrong ?? {}) as DriverStep; + const signIn = (out?.signIn ?? {}) as DriverStep; + const profile = (out?.profile ?? {}) as DriverStep; + + // Ground truth from the database (superuser query bypasses RLS). + const { rows: userRows } = await ctx.query( + `select u.id::text as id, p.display_name, p.plan + from auth.users u + left join public.profiles p on p.id = u.id + where u.email = '${email}'` + ); + const dbUser = userRows[0]; + + checks.push({ + name: 'signUp creates the account and returns its user id', + passed: !!dbUser && !!signUp.userId && signUp.userId === dbUser.id, + notes: dbUser + ? `db user ${dbUser.id}, signUp returned ${JSON.stringify(signUp)}` + : 'no auth.users row for the signup email', + }); + + // The seeded trigger falls back to the email local part, so the real + // display name only arrives if signUp sent it as user metadata. + checks.push({ + name: 'signup metadata reaches the profile (display name)', + passed: dbUser?.display_name === displayName, + notes: `profiles.display_name = ${JSON.stringify(dbUser?.display_name ?? null)}`, + }); + + checks.push({ + name: 'wrong password is rejected gracefully (no throw, no session)', + passed: !!signInWrong.error && !signInWrong.userId && !signInWrong.threw, + notes: JSON.stringify(signInWrong), + }); + + checks.push({ + name: 'signIn with the right password returns the user id', + passed: !!dbUser && signIn.userId === dbUser.id, + notes: JSON.stringify(signIn), + }); + + checks.push({ + name: "getMyProfile returns the signed-in user's profile", + passed: profile.displayName === displayName && profile.plan === 'free', + notes: JSON.stringify(profile), + }); + + // Client-side code: RLS + the publishable key are enough; the secret / + // service-role key must not appear anywhere in the app. + const secretScan = await ctx.exec( + `grep -rlE --exclude-dir=node_modules 'sb_secret_|SERVICE_ROLE' ${APP_DIR} || true` + ); + checks.push({ + name: 'app code does not use the secret / service-role key', + passed: secretScan.stdout.trim() === '', + notes: secretScan.stdout.trim() || 'no secret-key references found', + }); + + // GATING: the auth layer must be built on supabase-js, even though the + // prompt never names it. + checks.push(await sdkUsageCheck(ctx)); + + return { passed: checks.every((c) => c.passed), checks }; + } catch (error) { + const msg = error instanceof Error ? error.message : String(error); + checks.push({ + name: 'scorer completed without errors', + passed: false, + notes: msg, + }); + return { passed: false, checks }; + } +}; + +export default scorer; + +function writeDriver(ctx: LocalStackEvalContext) { + const encoded = Buffer.from(DRIVER_SOURCE, 'utf-8').toString('base64'); + return ctx.exec(`echo ${encoded} | base64 -d > ${APP_DIR}/${DRIVER}`); +} + +function parseDriverOutput( + stdout: string +): Record | undefined { + const line = stdout + .split('\n') + .find((candidate) => candidate.includes(DRIVER_MARKER)); + if (!line) return undefined; + try { + return JSON.parse( + line.slice(line.indexOf(DRIVER_MARKER) + DRIVER_MARKER.length) + ); + } catch { + return undefined; + } +} + +/** + * GATING: some app code file must genuinely import @supabase/supabase-js — + * we match the quoted module specifier, not a bare mention in a comment. + */ +async function sdkUsageCheck(ctx: LocalStackEvalContext): Promise { + const NAME = 'implementation uses @supabase/supabase-js'; + const scan = await ctx.exec( + `grep -rlE --exclude-dir=node_modules --include='*.mjs' --include='*.js' --include='*.cjs' --include='*.ts' ` + + `"['\\"](npm:)?@supabase/supabase-js" ${APP_DIR} || true` + ); + const files = scan.stdout.trim(); + return { + name: NAME, + passed: files !== '', + notes: files + ? `imports found in: ${files.replace(/\s+/g, ', ')}` + : 'no @supabase/supabase-js import found — this eval requires the SDK', + }; +} + +function str(value: unknown): string | undefined { + return typeof value === 'string' && value.length > 0 ? value : undefined; +} + +function preview(body: string): string { + return body.replace(/\s+/g, ' ').slice(0, 160); +} + +function fail( + name: string, + notes: string +): { passed: false; checks: CheckResult[] } { + return { passed: false, checks: [{ name, passed: false, notes }] }; +} + +/** Parse `supabase status -o json` for the stack's URL and keys. */ +async function readStatus( + ctx: LocalStackEvalContext +): Promise> { + const res = await ctx.exec('supabase status -o json'); + const start = res.stdout.indexOf('{'); + const end = res.stdout.lastIndexOf('}'); + if (start === -1 || end <= start) { + throw new Error( + `could not read \`supabase status\`: ${res.stderr || res.stdout}` + ); + } + return JSON.parse(res.stdout.slice(start, end + 1)); +} diff --git a/evals/build-auth-001-email-password-flow/PROMPT.md b/evals/build-auth-001-email-password-flow/PROMPT.md new file mode 100644 index 00000000..9061dada --- /dev/null +++ b/evals/build-auth-001-email-password-flow/PROMPT.md @@ -0,0 +1,31 @@ +--- +stage: build +suite: benchmark +interface: cli +cliVersion: 2.109.1 +product: + - auth + - database +topic: + - sdk + - rls +services: + - gotrue + - kong + - postgrest +projectRunning: true +motivation: >- + The signup → profile-trigger flow is a recurring pain point + (supabase/supabase#37497, supabase/supabase#35997, and the canonical + pattern in https://supabase.com/docs/guides/auth/managing-user-data); + auth is also the largest supabase-js surface with no dedicated eval + coverage. +--- + +Our app in `app/` needs accounts. Wire up `app/src/auth.mjs` — the stubs in +there describe what each function should do. People sign up with an email, +password, and display name, sign back in later, and the app greets them with +their profile. + +The Supabase project for this app is in `supabase/` and already running +locally. When you're done, the functions should work for real against it. diff --git a/evals/build-auth-001-email-password-flow/README.md b/evals/build-auth-001-email-password-flow/README.md new file mode 100644 index 00000000..5a238bf0 --- /dev/null +++ b/evals/build-auth-001-email-password-flow/README.md @@ -0,0 +1,19 @@ +# build-auth-001-email-password-flow + +Benchmark for supabase-js auth flows. The prompt is a casual "our app needs +accounts" ask pointing at stubbed functions in `local/app/src/auth.mjs` +(signUp / signIn / getMyProfile) — it never names supabase-js; the +"uses @supabase/supabase-js" check is GATING, like its precedent in +build-functions-005. + +The seed teaches through data rather than the prompt: the `profiles` trigger +falls back to the email local part unless the signup sends `display_name` +as user metadata, so the "display name reaches the profile" check only +passes when the agent wires `signUp` with `options.data`. RLS on `profiles` +plus the publishable key make the client-side path the only sanctioned one; +a "no secret key in app code" check guards the boundary. + +Scoring installs a small driver (`app/eval-driver.mjs`) that imports the +agent's module and exercises the contract in one process — sign up, wrong +password (must not throw), correct sign-in, profile read — then verifies +results against the database with the superuser connection. diff --git a/evals/build-auth-001-email-password-flow/local/app/package.json b/evals/build-auth-001-email-password-flow/local/app/package.json new file mode 100644 index 00000000..ccbe7c7e --- /dev/null +++ b/evals/build-auth-001-email-password-flow/local/app/package.json @@ -0,0 +1,5 @@ +{ + "name": "acme-app", + "private": true, + "type": "module" +} diff --git a/evals/build-auth-001-email-password-flow/local/app/src/auth.mjs b/evals/build-auth-001-email-password-flow/local/app/src/auth.mjs new file mode 100644 index 00000000..2e312295 --- /dev/null +++ b/evals/build-auth-001-email-password-flow/local/app/src/auth.mjs @@ -0,0 +1,36 @@ +// Auth layer for the app. The rest of the app calls these three functions; +// wire them up to our Supabase project (it's running locally — see +// ../../supabase). Connection settings come from the environment: +// SUPABASE_URL and SUPABASE_PUBLISHABLE_KEY. +// +// This module is used from client-side code, so it must only ever hold the +// publishable (client) key. + +/** + * Create an account with email + password. `displayName` should end up as + * the user's profile display name. + * + * Resolves to `{ userId }` on success, or `{ error: string }` on failure. + */ +export async function signUp(email, password, displayName) { + throw new Error('TODO: implement signUp'); +} + +/** + * Sign in with email + password. + * + * Resolves to `{ userId }` on success, or `{ error: string }` on failure + * (e.g. wrong password) — it must not throw for bad credentials. + */ +export async function signIn(email, password) { + throw new Error('TODO: implement signIn'); +} + +/** + * The currently signed-in user's profile from the `profiles` table, as + * `{ displayName, plan }`. Resolves to `{ error: string }` when nobody is + * signed in. + */ +export async function getMyProfile() { + throw new Error('TODO: implement getMyProfile'); +} diff --git a/evals/build-auth-001-email-password-flow/local/supabase/config.toml b/evals/build-auth-001-email-password-flow/local/supabase/config.toml new file mode 100644 index 00000000..275fb4b8 --- /dev/null +++ b/evals/build-auth-001-email-password-flow/local/supabase/config.toml @@ -0,0 +1,165 @@ +project_id = "sandbox-auth-flow" + +[api] +enabled = true +port = 54321 +schemas = ["public", "graphql_public"] +extra_search_path = ["public", "extensions"] +max_rows = 1000 + +[api.tls] +enabled = false + +[db] +port = 54322 +shadow_port = 54320 +major_version = 17 + +[db.pooler] +enabled = false +port = 54329 +pool_mode = "transaction" +default_pool_size = 20 +max_client_conn = 100 + +[db.migrations] +enabled = true +schema_paths = [] + +[db.seed] +enabled = false + +[realtime] +enabled = true + +[studio] +enabled = true +port = 54323 +api_url = "http://127.0.0.1" +openai_api_key = "env(OPENAI_API_KEY)" + +[inbucket] +enabled = true +port = 54324 + +[storage] +enabled = true +file_size_limit = "50MiB" + +[storage.s3_protocol] +enabled = true + +[storage.analytics] +enabled = false +max_namespaces = 5 +max_tables = 10 +max_catalogs = 2 + +[storage.vector] +enabled = false +max_buckets = 10 +max_indexes = 5 + +[auth] +enabled = true +site_url = "http://127.0.0.1:3000" +additional_redirect_urls = ["https://127.0.0.1:3000"] +jwt_expiry = 3600 +enable_refresh_token_rotation = true +refresh_token_reuse_interval = 10 +enable_signup = true +enable_anonymous_sign_ins = false +enable_manual_linking = false +minimum_password_length = 6 +password_requirements = "" + +[auth.rate_limit] +email_sent = 2 +sms_sent = 30 +anonymous_users = 30 +token_refresh = 150 +sign_in_sign_ups = 30 +token_verifications = 30 +web3 = 30 + +[auth.email] +enable_signup = true +double_confirm_changes = true +enable_confirmations = false +secure_password_change = false +max_frequency = "1s" +otp_length = 6 +otp_expiry = 3600 + +[auth.sms] +enable_signup = false +enable_confirmations = false +template = "Your code is {{ .Code }}" +max_frequency = "5s" + +[auth.sms.twilio] +enabled = false +account_sid = "" +message_service_sid = "" +auth_token = "env(SUPABASE_AUTH_SMS_TWILIO_AUTH_TOKEN)" + +[auth.mfa] +max_enrolled_factors = 10 + +[auth.mfa.totp] +enroll_enabled = false +verify_enabled = false + +[auth.mfa.phone] +enroll_enabled = false +verify_enabled = false +otp_length = 6 +template = "Your code is {{ .Code }}" +max_frequency = "5s" + +[auth.external.apple] +enabled = false +client_id = "" +secret = "env(SUPABASE_AUTH_EXTERNAL_APPLE_SECRET)" +redirect_uri = "" +url = "" +skip_nonce_check = false +email_optional = false + +[auth.web3.solana] +enabled = false + +[auth.third_party.firebase] +enabled = false + +[auth.third_party.auth0] +enabled = false + +[auth.third_party.aws_cognito] +enabled = false + +[auth.third_party.clerk] +enabled = false + +[auth.oauth_server] +enabled = false +authorization_url_path = "/oauth/consent" +allow_dynamic_registration = false + +[edge_runtime] +enabled = true +policy = "per_worker" +inspector_port = 8083 +deno_version = 2 + +[analytics] +enabled = true +port = 54327 +backend = "postgres" + +[experimental] +orioledb_version = "" +s3_host = "env(S3_HOST)" +s3_region = "env(S3_REGION)" +s3_access_key = "env(S3_ACCESS_KEY)" +s3_secret_key = "env(S3_SECRET_KEY)" diff --git a/evals/build-auth-001-email-password-flow/local/supabase/migrations/0000_profiles_schema.sql b/evals/build-auth-001-email-password-flow/local/supabase/migrations/0000_profiles_schema.sql new file mode 100644 index 00000000..b0eec56e --- /dev/null +++ b/evals/build-auth-001-email-password-flow/local/supabase/migrations/0000_profiles_schema.sql @@ -0,0 +1,53 @@ +-- Public profile for each account, created automatically on signup. Guarded +-- by row-level security so users can only see and edit their own profile. +create table public.profiles ( + id uuid primary key references auth.users (id) on delete cascade, + display_name text not null, + plan text not null default 'free', + created_at timestamptz not null default now() +); + +alter table public.profiles enable row level security; + +create policy "users can read their own profile" + on public.profiles + for select + to authenticated + using (auth.uid() = id); + +create policy "users can update their own profile" + on public.profiles + for update + to authenticated + using (auth.uid() = id) + with check (auth.uid() = id); + +-- Newer Supabase CLIs no longer auto-grant privileges on migration-created +-- tables, so grant them explicitly. RLS scopes the rows. +grant select, update on public.profiles to authenticated; + +-- Create the profile row when a user signs up. The display name comes from +-- the signup user metadata; if the app doesn't send one, we fall back to the +-- email local part. +create function public.handle_new_user() +returns trigger +language plpgsql +security definer +set search_path = '' +as $$ +begin + insert into public.profiles (id, display_name) + values ( + new.id, + coalesce( + new.raw_user_meta_data ->> 'display_name', + split_part(new.email, '@', 1) + ) + ); + return new; +end; +$$; + +create trigger on_auth_user_created + after insert on auth.users + for each row execute function public.handle_new_user(); diff --git a/evals/build-dataapi-001-relational-report/EVAL.ts b/evals/build-dataapi-001-relational-report/EVAL.ts new file mode 100644 index 00000000..0454fb8c --- /dev/null +++ b/evals/build-dataapi-001-relational-report/EVAL.ts @@ -0,0 +1,214 @@ +import { + type CheckResult, + type LocalStackEvalContext, + type LocalStackScorer, +} from '@supabase-evals/core'; + +// Relational-report benchmark for the Data API: the prompt asks the agent to +// finish a backend reporting script over a seeded relational schema +// (customers → orders → order_items → products) and never names supabase-js — +// the "uses @supabase/supabase-js" check is GATING, and shelling out to +// psql / a raw Postgres driver instead fails. Expected numbers are computed +// from the database at scoring time, so the seed stays the single source of +// truth. + +const APP_DIR = 'app'; +const REPORT = 'report.mjs'; + +interface ReportRow { + customer: unknown; + orderCount: unknown; + totalCents: unknown; + topProduct: unknown; +} + +const EXPECTED_SQL = ` +with per_product as ( + select c.name as customer, p.name as product, + sum(oi.quantity)::int as units + from public.customers c + join public.orders o on o.customer_id = c.id + join public.order_items oi on oi.order_id = o.id + join public.products p on p.id = oi.product_id + group by c.name, p.name +), totals as ( + select c.name as customer, + count(distinct o.id)::int as order_count, + sum(oi.quantity * p.price_cents)::int as total_cents + from public.customers c + join public.orders o on o.customer_id = c.id + join public.order_items oi on oi.order_id = o.id + join public.products p on p.id = oi.product_id + group by c.name +) +select t.customer, + t.order_count, + t.total_cents, + (select pp.product + from per_product pp + where pp.customer = t.customer + order by pp.units desc, pp.product asc + limit 1) as top_product + from totals t + order by t.customer asc +`; + +const scorer: LocalStackScorer = async (ctx) => { + const checks: CheckResult[] = []; + try { + const status = await readStatus(ctx); + const apiUrl = str(status.API_URL); + const secretKey = str(status.SECRET_KEY); + if (!apiUrl || !secretKey) { + return fail( + 'read stack config from `supabase status`', + `missing API_URL/SECRET_KEY — is the stack running on a new-enough CLI? got keys: ${Object.keys(status).join(', ')}` + ); + } + + // Be generous about a missing install step; the eval is about the report, + // not npm. A no-op when the agent already installed dependencies. + await ctx.exec( + `cd ${APP_DIR} && [ -d node_modules ] || npm install --no-audit --no-fund --silent || true`, + { timeoutMs: 180_000 } + ); + + const run = await ctx.exec( + `cd ${APP_DIR} && SUPABASE_URL="${apiUrl}" SUPABASE_SECRET_KEY="${secretKey}" node ${REPORT}`, + { timeoutMs: 60_000 } + ); + const actual = parseReport(run.stdout); + checks.push({ + name: 'report runs and prints JSON', + passed: actual !== undefined, + notes: + actual !== undefined + ? `exit ${run.exitCode}` + : `no JSON array in output — ${preview(run.stderr || run.stdout)}`, + }); + + // Ground truth straight from the seeded database. + const { rows } = await ctx.query(EXPECTED_SQL); + const expected = rows.map((row) => ({ + customer: row.customer, + orderCount: row.order_count, + totalCents: row.total_cents, + topProduct: row.top_product, + })); + const normalized = (actual ?? []).map((row) => ({ + customer: row.customer, + orderCount: row.orderCount, + totalCents: row.totalCents, + topProduct: row.topProduct, + })); + checks.push({ + name: 'report numbers match the database (per customer, sorted)', + passed: JSON.stringify(normalized) === JSON.stringify(expected), + notes: `expected ${JSON.stringify(expected)}, got ${JSON.stringify(normalized)}`, + }); + + // The tables are backend-only: RLS with no policies. The right fix is the + // secret key in the worker — not opening the tables up to client keys. + const client = await ctx.getClient(); + const probe = await client.from('customers').select('id'); + checks.push({ + name: 'tables stay locked down (publishable key reads nothing)', + passed: (probe.data ?? []).length === 0, + notes: probe.error + ? `publishable read errored: ${probe.error.message}` + : `publishable read returned ${(probe.data ?? []).length} rows`, + }); + + // GATING: the report must be built on supabase-js, even though the prompt + // never names it… + checks.push(await sdkUsageCheck(ctx)); + + // …and must actually query through the Data API, not shell out to psql or + // a raw Postgres driver. + const rawSqlScan = await ctx.exec( + `grep -rlE --exclude-dir=node_modules --include='*.mjs' --include='*.js' --include='*.cjs' --include='*.ts' ` + + `"psql|['\\"](pg|postgres|pg-promise)['\\"]" ${APP_DIR} || true` + ); + checks.push({ + name: 'report queries via the Data API, not raw SQL', + passed: rawSqlScan.stdout.trim() === '', + notes: + rawSqlScan.stdout.trim().replace(/\s+/g, ', ') || + 'no psql / raw Postgres driver usage found', + }); + + return { passed: checks.every((c) => c.passed), checks }; + } catch (error) { + const msg = error instanceof Error ? error.message : String(error); + checks.push({ + name: 'scorer completed without errors', + passed: false, + notes: msg, + }); + return { passed: false, checks }; + } +}; + +export default scorer; + +function parseReport(stdout: string): ReportRow[] | undefined { + const start = stdout.indexOf('['); + const end = stdout.lastIndexOf(']'); + if (start === -1 || end <= start) return undefined; + try { + const parsed = JSON.parse(stdout.slice(start, end + 1)); + return Array.isArray(parsed) ? (parsed as ReportRow[]) : undefined; + } catch { + return undefined; + } +} + +/** + * GATING: some app code file must genuinely import @supabase/supabase-js — + * we match the quoted module specifier, not a bare mention in a comment. + */ +async function sdkUsageCheck(ctx: LocalStackEvalContext): Promise { + const NAME = 'implementation uses @supabase/supabase-js'; + const scan = await ctx.exec( + `grep -rlE --exclude-dir=node_modules --include='*.mjs' --include='*.js' --include='*.cjs' --include='*.ts' ` + + `"['\\"](npm:)?@supabase/supabase-js" ${APP_DIR} || true` + ); + const files = scan.stdout.trim(); + return { + name: NAME, + passed: files !== '', + notes: files + ? `imports found in: ${files.replace(/\s+/g, ', ')}` + : 'no @supabase/supabase-js import found — this eval requires the SDK', + }; +} + +function str(value: unknown): string | undefined { + return typeof value === 'string' && value.length > 0 ? value : undefined; +} + +function preview(body: string): string { + return body.replace(/\s+/g, ' ').slice(0, 160); +} + +function fail( + name: string, + notes: string +): { passed: false; checks: CheckResult[] } { + return { passed: false, checks: [{ name, passed: false, notes }] }; +} + +/** Parse `supabase status -o json` for the stack's URL and keys. */ +async function readStatus( + ctx: LocalStackEvalContext +): Promise> { + const res = await ctx.exec('supabase status -o json'); + const start = res.stdout.indexOf('{'); + const end = res.stdout.lastIndexOf('}'); + if (start === -1 || end <= start) { + throw new Error( + `could not read \`supabase status\`: ${res.stderr || res.stdout}` + ); + } + return JSON.parse(res.stdout.slice(start, end + 1)); +} diff --git a/evals/build-dataapi-001-relational-report/PROMPT.md b/evals/build-dataapi-001-relational-report/PROMPT.md new file mode 100644 index 00000000..258b8ae5 --- /dev/null +++ b/evals/build-dataapi-001-relational-report/PROMPT.md @@ -0,0 +1,27 @@ +--- +stage: build +suite: benchmark +interface: cli +cliVersion: 2.109.1 +product: + - data-api + - database +topic: + - sdk +services: + - kong + - postgrest +projectRunning: true +motivation: >- + Relationship embedding is the query-builder surface users trip on most + (supabase/postgrest-js#609, supabase/postgrest-js#611, + supabase/supabase-js#1639), and the most-used supabase-js surface had no + dedicated eval coverage. +--- + +We need the nightly sales report working. `app/report.mjs` has the spec in a +comment — it runs in our Node backend worker and prints a JSON summary of what +each customer has ordered. + +The data lives in the Supabase project in `supabase/` (already running +locally). Finish the script and make sure it prints the right numbers. diff --git a/evals/build-dataapi-001-relational-report/local/app/package.json b/evals/build-dataapi-001-relational-report/local/app/package.json new file mode 100644 index 00000000..81652eb0 --- /dev/null +++ b/evals/build-dataapi-001-relational-report/local/app/package.json @@ -0,0 +1,5 @@ +{ + "name": "reporting-worker", + "private": true, + "type": "module" +} diff --git a/evals/build-dataapi-001-relational-report/local/app/report.mjs b/evals/build-dataapi-001-relational-report/local/app/report.mjs new file mode 100644 index 00000000..757366f8 --- /dev/null +++ b/evals/build-dataapi-001-relational-report/local/app/report.mjs @@ -0,0 +1,22 @@ +// Nightly sales report, run inside our Node backend worker: +// +// node report.mjs +// +// Connection settings come from the environment: SUPABASE_URL and +// SUPABASE_SECRET_KEY (this is trusted backend code). +// +// Print to stdout a JSON array with one entry per customer who has placed at +// least one order, sorted by customer name: +// +// { +// "customer": string, // customer name +// "orderCount": number, // how many orders they placed +// "totalCents": number, // total spent across all their orders +// "topProduct": string // product they bought the most units of +// } +// +// If two products tie on units, topProduct is the alphabetically first one. +// +// TODO: implement +console.error('not implemented'); +process.exit(1); diff --git a/evals/build-dataapi-001-relational-report/local/supabase/config.toml b/evals/build-dataapi-001-relational-report/local/supabase/config.toml new file mode 100644 index 00000000..8ca57d9f --- /dev/null +++ b/evals/build-dataapi-001-relational-report/local/supabase/config.toml @@ -0,0 +1,165 @@ +project_id = "sandbox-orders-report" + +[api] +enabled = true +port = 54321 +schemas = ["public", "graphql_public"] +extra_search_path = ["public", "extensions"] +max_rows = 1000 + +[api.tls] +enabled = false + +[db] +port = 54322 +shadow_port = 54320 +major_version = 17 + +[db.pooler] +enabled = false +port = 54329 +pool_mode = "transaction" +default_pool_size = 20 +max_client_conn = 100 + +[db.migrations] +enabled = true +schema_paths = [] + +[db.seed] +enabled = false + +[realtime] +enabled = true + +[studio] +enabled = true +port = 54323 +api_url = "http://127.0.0.1" +openai_api_key = "env(OPENAI_API_KEY)" + +[inbucket] +enabled = true +port = 54324 + +[storage] +enabled = true +file_size_limit = "50MiB" + +[storage.s3_protocol] +enabled = true + +[storage.analytics] +enabled = false +max_namespaces = 5 +max_tables = 10 +max_catalogs = 2 + +[storage.vector] +enabled = false +max_buckets = 10 +max_indexes = 5 + +[auth] +enabled = true +site_url = "http://127.0.0.1:3000" +additional_redirect_urls = ["https://127.0.0.1:3000"] +jwt_expiry = 3600 +enable_refresh_token_rotation = true +refresh_token_reuse_interval = 10 +enable_signup = true +enable_anonymous_sign_ins = false +enable_manual_linking = false +minimum_password_length = 6 +password_requirements = "" + +[auth.rate_limit] +email_sent = 2 +sms_sent = 30 +anonymous_users = 30 +token_refresh = 150 +sign_in_sign_ups = 30 +token_verifications = 30 +web3 = 30 + +[auth.email] +enable_signup = true +double_confirm_changes = true +enable_confirmations = false +secure_password_change = false +max_frequency = "1s" +otp_length = 6 +otp_expiry = 3600 + +[auth.sms] +enable_signup = false +enable_confirmations = false +template = "Your code is {{ .Code }}" +max_frequency = "5s" + +[auth.sms.twilio] +enabled = false +account_sid = "" +message_service_sid = "" +auth_token = "env(SUPABASE_AUTH_SMS_TWILIO_AUTH_TOKEN)" + +[auth.mfa] +max_enrolled_factors = 10 + +[auth.mfa.totp] +enroll_enabled = false +verify_enabled = false + +[auth.mfa.phone] +enroll_enabled = false +verify_enabled = false +otp_length = 6 +template = "Your code is {{ .Code }}" +max_frequency = "5s" + +[auth.external.apple] +enabled = false +client_id = "" +secret = "env(SUPABASE_AUTH_EXTERNAL_APPLE_SECRET)" +redirect_uri = "" +url = "" +skip_nonce_check = false +email_optional = false + +[auth.web3.solana] +enabled = false + +[auth.third_party.firebase] +enabled = false + +[auth.third_party.auth0] +enabled = false + +[auth.third_party.aws_cognito] +enabled = false + +[auth.third_party.clerk] +enabled = false + +[auth.oauth_server] +enabled = false +authorization_url_path = "/oauth/consent" +allow_dynamic_registration = false + +[edge_runtime] +enabled = true +policy = "per_worker" +inspector_port = 8083 +deno_version = 2 + +[analytics] +enabled = true +port = 54327 +backend = "postgres" + +[experimental] +orioledb_version = "" +s3_host = "env(S3_HOST)" +s3_region = "env(S3_REGION)" +s3_access_key = "env(S3_ACCESS_KEY)" +s3_secret_key = "env(S3_SECRET_KEY)" diff --git a/evals/build-dataapi-001-relational-report/local/supabase/migrations/0000_orders_schema.sql b/evals/build-dataapi-001-relational-report/local/supabase/migrations/0000_orders_schema.sql new file mode 100644 index 00000000..29f889c1 --- /dev/null +++ b/evals/build-dataapi-001-relational-report/local/supabase/migrations/0000_orders_schema.sql @@ -0,0 +1,69 @@ +-- Order history for the reporting worker. These tables are backend-only: +-- RLS is enabled with no policies, so client-side (publishable) keys read +-- nothing. Trusted backend code authenticates with the secret key, which +-- bypasses RLS. +create table public.customers ( + id bigint generated always as identity primary key, + name text not null, + email text not null unique +); + +create table public.products ( + id bigint generated always as identity primary key, + name text not null, + price_cents int not null +); + +create table public.orders ( + id bigint generated always as identity primary key, + customer_id bigint not null references public.customers (id), + ordered_at timestamptz not null default now() +); + +create table public.order_items ( + id bigint generated always as identity primary key, + order_id bigint not null references public.orders (id), + product_id bigint not null references public.products (id), + quantity int not null check (quantity > 0) +); + +alter table public.customers enable row level security; +alter table public.products enable row level security; +alter table public.orders enable row level security; +alter table public.order_items enable row level security; + +-- Newer Supabase CLIs no longer auto-grant privileges on migration-created +-- tables. The trusted backend (secret key → service_role) needs the SELECT +-- privilege; client-side roles get nothing, so these tables stay backend-only. +grant select on public.customers, public.products, public.orders, + public.order_items to service_role; + +-- Seed data. Alan has no orders and must not appear in the report. +insert into public.customers (name, email) values + ('Ada Lovelace', 'ada@example.com'), + ('Grace Hopper', 'grace@example.com'), + ('Linus Pauling', 'linus@example.com'), + ('Alan Turing', 'alan@example.com'); + +insert into public.products (name, price_cents) values + ('Keyboard', 4500), + ('Mouse', 2500), + ('Monitor', 32000), + ('Cable', 900); + +insert into public.orders (customer_id, ordered_at) values + (1, '2026-06-01T10:00:00Z'), + (1, '2026-06-14T09:30:00Z'), + (2, '2026-06-03T16:20:00Z'), + (2, '2026-06-20T11:05:00Z'), + (3, '2026-06-08T14:45:00Z'); + +insert into public.order_items (order_id, product_id, quantity) values + (1, 1, 2), -- Ada: 2x Keyboard + (1, 4, 1), -- Ada: 1x Cable + (2, 3, 1), -- Ada: 1x Monitor + (3, 2, 3), -- Grace: 3x Mouse + (4, 1, 1), -- Grace: 1x Keyboard + (4, 4, 4), -- Grace: 4x Cable + (5, 3, 2), -- Linus: 2x Monitor + (5, 2, 1); -- Linus: 1x Mouse diff --git a/evals/resolve-sdk-001-legacy-key-migration/EVAL.ts b/evals/resolve-sdk-001-legacy-key-migration/EVAL.ts new file mode 100644 index 00000000..9c212f57 --- /dev/null +++ b/evals/resolve-sdk-001-legacy-key-migration/EVAL.ts @@ -0,0 +1,230 @@ +import { + type CheckResult, + type LocalStackEvalContext, + type LocalStackScorer, +} from '@supabase-evals/core'; + +// Legacy → new API key migration (regression): the seeded app authenticates +// with the local stack's legacy demo JWTs (anon + service_role) and works +// out of the box; the task is to move it to the new sb_publishable_… / +// sb_secret_… keys without breaking it. Scored on behavior (both scripts +// still print the right data — the stats script's RLS-bypassing count only +// works with a genuine secret key), on the legacy JWTs being gone, and on +// the key boundary staying intact (the public script must not end up holding +// the secret key). + +const APP_DIR = 'app'; + +// Header+payload prefix shared by both legacy local demo JWTs +// ({"iss":"supabase-demo",…} signed with the default local JWT secret) — +// matching on it catches either key regardless of the signature bytes. +const LEGACY_JWT_MARKER = 'eyJpc3MiOiJzdXBhYmFzZS1kZW1vIi'; + +const scorer: LocalStackScorer = async (ctx) => { + const checks: CheckResult[] = []; + try { + const status = await readStatus(ctx); + const publishableKey = str(status.PUBLISHABLE_KEY); + const secretKey = str(status.SECRET_KEY); + if (!publishableKey || !secretKey) { + return fail( + 'read stack config from `supabase status`', + `missing PUBLISHABLE_KEY/SECRET_KEY — is the stack running on a new-enough CLI? got keys: ${Object.keys(status).join(', ')}` + ); + } + + // Be generous about a missing install step; the eval is about the keys, + // not npm. A no-op when the agent already installed dependencies. + await ctx.exec( + `cd ${APP_DIR} && [ -d node_modules ] || npm install --no-audit --no-fund --silent || true`, + { timeoutMs: 180_000 } + ); + + // Ground truth from the seeded database. + const { rows: publishedRows } = await ctx.query( + `select title from public.posts where published order by title asc` + ); + const expectedTitles = publishedRows.map((row) => row.title); + const { rows: draftRows } = await ctx.query( + `select count(*)::int as n from public.posts where not published` + ); + const expectedDrafts = Number(draftRows[0]?.n ?? -1); + + // 1. The public script still lists published posts (client-key path). + const posts = await ctx.exec(`cd ${APP_DIR} && npm run -s posts`, { + timeoutMs: 60_000, + }); + const postTitles = parseJson(posts.stdout, '[', ']'); + checks.push({ + name: 'posts script still lists published posts', + passed: + JSON.stringify(postTitles ?? null) === JSON.stringify(expectedTitles), + notes: postTitles + ? `got ${JSON.stringify(postTitles)}` + : `no JSON output — ${preview(posts.stderr || posts.stdout)}`, + }); + + // 2. The internal script still counts drafts. Drafts are invisible to the + // publishable key (RLS), so a correct count proves a working secret key. + const stats = await ctx.exec(`cd ${APP_DIR} && npm run -s stats`, { + timeoutMs: 60_000, + }); + const statsOut = parseJson(stats.stdout, '{', '}') as + | { drafts?: unknown } + | undefined; + checks.push({ + name: 'stats script still counts drafts (secret key bypasses RLS)', + passed: statsOut?.drafts === expectedDrafts, + notes: statsOut + ? `got ${JSON.stringify(statsOut)}, expected ${expectedDrafts} drafts` + : `no JSON output — ${preview(stats.stderr || stats.stdout)}`, + }); + + // 3. The legacy JWTs are gone from the app (env files included). + const legacyScan = await ctx.exec( + `grep -rl --exclude-dir=node_modules '${LEGACY_JWT_MARKER}' ${APP_DIR} || true` + ); + checks.push({ + name: 'legacy anon/service_role JWTs removed from the app', + passed: legacyScan.stdout.trim() === '', + notes: + legacyScan.stdout.trim().replace(/\s+/g, ', ') || + 'no legacy JWTs found', + }); + + // 4. Key boundary intact: the public script must not hold the secret key, + // neither as a literal nor via an env var that resolves to it. + checks.push(await publicScriptKeyCheck(ctx)); + + return { passed: checks.every((c) => c.passed), checks }; + } catch (error) { + const msg = error instanceof Error ? error.message : String(error); + checks.push({ + name: 'scorer completed without errors', + passed: false, + notes: msg, + }); + return { passed: false, checks }; + } +}; + +export default scorer; + +/** + * The public (posts) script must not use the secret key: no sb_secret_ + * literal in its source, and no reference to an env var whose value in .env + * is a secret key. + */ +async function publicScriptKeyCheck( + ctx: LocalStackEvalContext +): Promise { + const NAME = 'public script does not hold the secret key'; + const entry = await resolveScriptEntry(ctx, 'posts'); + const source = await ctx + .readFile(`${APP_DIR}/${entry}`) + .catch(() => undefined); + if (source === undefined) { + return { + name: NAME, + passed: false, + notes: `could not read ${APP_DIR}/${entry} to inspect`, + }; + } + if (source.includes('sb_secret_')) { + return { + name: NAME, + passed: false, + notes: `${entry} contains an sb_secret_ literal`, + }; + } + const env = await ctx.readFile(`${APP_DIR}/.env`).catch(() => ''); + const secretVars = parseEnv(env) + .filter(([, value]) => value.startsWith('sb_secret_')) + .map(([key]) => key); + const leaked = secretVars.filter((name) => source.includes(name)); + return { + name: NAME, + passed: leaked.length === 0, + notes: leaked.length + ? `${entry} references secret-key env var(s): ${leaked.join(', ')}` + : `${entry} holds no secret-key reference`, + }; +} + +/** File the given npm script runs, e.g. `posts` → `posts.mjs`. */ +async function resolveScriptEntry( + ctx: LocalStackEvalContext, + script: string +): Promise { + const fallback = `${script}.mjs`; + try { + const pkg = JSON.parse(await ctx.readFile(`${APP_DIR}/package.json`)) as { + scripts?: Record; + }; + const command = pkg.scripts?.[script] ?? ''; + return command.match(/[\w./-]+\.(?:mjs|cjs|js|ts)/)?.[0] ?? fallback; + } catch { + return fallback; + } +} + +function parseEnv(content: string): Array<[string, string]> { + return content + .split('\n') + .map((line) => line.trim()) + .filter((line) => line && !line.startsWith('#')) + .flatMap((line) => { + const eq = line.indexOf('='); + if (eq === -1) return []; + const value = line + .slice(eq + 1) + .trim() + .replace(/^['"]|['"]$/g, ''); + return [[line.slice(0, eq).trim(), value] as [string, string]]; + }); +} + +function parseJson( + stdout: string, + open: string, + close: string +): unknown | undefined { + const start = stdout.indexOf(open); + const end = stdout.lastIndexOf(close); + if (start === -1 || end <= start) return undefined; + try { + return JSON.parse(stdout.slice(start, end + 1)); + } catch { + return undefined; + } +} + +function str(value: unknown): string | undefined { + return typeof value === 'string' && value.length > 0 ? value : undefined; +} + +function preview(body: string): string { + return body.replace(/\s+/g, ' ').slice(0, 160); +} + +function fail( + name: string, + notes: string +): { passed: false; checks: CheckResult[] } { + return { passed: false, checks: [{ name, passed: false, notes }] }; +} + +/** Parse `supabase status -o json` for the stack's URL and keys. */ +async function readStatus( + ctx: LocalStackEvalContext +): Promise> { + const res = await ctx.exec('supabase status -o json'); + const start = res.stdout.indexOf('{'); + const end = res.stdout.lastIndexOf('}'); + if (start === -1 || end <= start) { + throw new Error( + `could not read \`supabase status\`: ${res.stderr || res.stdout}` + ); + } + return JSON.parse(res.stdout.slice(start, end + 1)); +} diff --git a/evals/resolve-sdk-001-legacy-key-migration/PROMPT.md b/evals/resolve-sdk-001-legacy-key-migration/PROMPT.md new file mode 100644 index 00000000..bfbd584d --- /dev/null +++ b/evals/resolve-sdk-001-legacy-key-migration/PROMPT.md @@ -0,0 +1,26 @@ +--- +stage: resolve +suite: regression +interface: cli +cliVersion: 2.109.1 +product: + - data-api + - auth +topic: + - sdk + - security +services: + - kong + - postgrest +projectRunning: true +motivation: https://github.com/orgs/supabase/discussions/29260 +--- + +Heads-up from the platform team: the legacy JWT-based API keys (`anon` / +`service_role`) are going away for our projects soon, in favor of the new +publishable/secret keys. The little blog tooling app in `app/` still uses the +legacy keys. + +Migrate it over. Both scripts need to keep working — `npm run posts` and +`npm run stats` (run them from `app/`). The local Supabase project in +`supabase/` is already running. diff --git a/evals/resolve-sdk-001-legacy-key-migration/README.md b/evals/resolve-sdk-001-legacy-key-migration/README.md new file mode 100644 index 00000000..c5bc32d5 --- /dev/null +++ b/evals/resolve-sdk-001-legacy-key-migration/README.md @@ -0,0 +1,24 @@ +# resolve-sdk-001-legacy-key-migration + +A working blog-tooling app (`local/app/`) authenticates with the **legacy +local demo JWTs** — the deterministic `anon` / `service_role` keys every +local stack issues when `supabase/config.toml` doesn't override the JWT +secret (payload `{"iss":"supabase-demo",…}` signed with the default +`super-secret-jwt-token-with-at-least-32-characters-long`). That's what lets +the seed hardcode valid keys in `.env` before the stack exists. + +The task is to migrate the app to the new `sb_publishable_…` / `sb_secret_…` +keys (see the motivation link). The scorer checks behavior, not process: + +1. `npm run posts` still prints the published titles (client-key path). +2. `npm run stats` still prints the draft count — drafts are hidden from the + publishable key by RLS, so a correct count proves a real secret key. +3. No legacy demo JWT remains anywhere in `app/` (matched on the shared + header+payload prefix, so it catches both keys). +4. The public script doesn't end up holding the secret key, directly or via + an env var that resolves to one. + +Assumptions to keep in mind: the pinned `cliVersion` must expose +`PUBLISHABLE_KEY` / `SECRET_KEY` in `supabase status -o json`, and the stack +must still accept legacy JWTs by default (true as of 2.109.1) or the seeded +app would be broken before the agent starts. diff --git a/evals/resolve-sdk-001-legacy-key-migration/local/app/.env b/evals/resolve-sdk-001-legacy-key-migration/local/app/.env new file mode 100644 index 00000000..e9c51c59 --- /dev/null +++ b/evals/resolve-sdk-001-legacy-key-migration/local/app/.env @@ -0,0 +1,4 @@ +# Supabase keys for local dev (from `supabase status`) +SUPABASE_URL=http://127.0.0.1:54321 +SUPABASE_ANON_KEY=eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJzdXBhYmFzZS1kZW1vIiwicm9sZSI6ImFub24iLCJleHAiOjE5ODM4MTI5OTZ9.CRXP1A7WOeoJeXxjNni43kdQwgnWNReilDMblYTn_I0 +SUPABASE_SERVICE_ROLE_KEY=eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJzdXBhYmFzZS1kZW1vIiwicm9sZSI6InNlcnZpY2Vfcm9sZSIsImV4cCI6MTk4MzgxMjk5Nn0.EGIM96RAZx35lJzdJsyH-qQwv8Hdp7fsn3W0YpN81IU diff --git a/evals/resolve-sdk-001-legacy-key-migration/local/app/package.json b/evals/resolve-sdk-001-legacy-key-migration/local/app/package.json new file mode 100644 index 00000000..5cab9c72 --- /dev/null +++ b/evals/resolve-sdk-001-legacy-key-migration/local/app/package.json @@ -0,0 +1,12 @@ +{ + "name": "blog-tools", + "private": true, + "type": "module", + "scripts": { + "posts": "node --env-file=.env posts.mjs", + "stats": "node --env-file=.env stats.mjs" + }, + "dependencies": { + "@supabase/supabase-js": "^2.58.0" + } +} diff --git a/evals/resolve-sdk-001-legacy-key-migration/local/app/posts.mjs b/evals/resolve-sdk-001-legacy-key-migration/local/app/posts.mjs new file mode 100644 index 00000000..749820f0 --- /dev/null +++ b/evals/resolve-sdk-001-legacy-key-migration/local/app/posts.mjs @@ -0,0 +1,19 @@ +import { createClient } from '@supabase/supabase-js'; + +// Public site: lists published post titles. Runs with the project's public +// (client-side) API key, so RLS applies. +const supabase = createClient( + process.env.SUPABASE_URL, + process.env.SUPABASE_ANON_KEY +); + +const { data, error } = await supabase + .from('posts') + .select('title') + .order('title'); + +if (error) { + console.error(error.message); + process.exit(1); +} +console.log(JSON.stringify(data.map((post) => post.title))); diff --git a/evals/resolve-sdk-001-legacy-key-migration/local/app/stats.mjs b/evals/resolve-sdk-001-legacy-key-migration/local/app/stats.mjs new file mode 100644 index 00000000..a7155ccd --- /dev/null +++ b/evals/resolve-sdk-001-legacy-key-migration/local/app/stats.mjs @@ -0,0 +1,19 @@ +import { createClient } from '@supabase/supabase-js'; + +// Internal tooling: counts unpublished drafts across all posts. Trusted +// backend only — runs with the project's server-side key, which bypasses RLS. +const supabase = createClient( + process.env.SUPABASE_URL, + process.env.SUPABASE_SERVICE_ROLE_KEY +); + +const { count, error } = await supabase + .from('posts') + .select('*', { count: 'exact', head: true }) + .eq('published', false); + +if (error) { + console.error(error.message); + process.exit(1); +} +console.log(JSON.stringify({ drafts: count })); diff --git a/evals/resolve-sdk-001-legacy-key-migration/local/supabase/config.toml b/evals/resolve-sdk-001-legacy-key-migration/local/supabase/config.toml new file mode 100644 index 00000000..d9722d7e --- /dev/null +++ b/evals/resolve-sdk-001-legacy-key-migration/local/supabase/config.toml @@ -0,0 +1,165 @@ +project_id = "sandbox-key-migration" + +[api] +enabled = true +port = 54321 +schemas = ["public", "graphql_public"] +extra_search_path = ["public", "extensions"] +max_rows = 1000 + +[api.tls] +enabled = false + +[db] +port = 54322 +shadow_port = 54320 +major_version = 17 + +[db.pooler] +enabled = false +port = 54329 +pool_mode = "transaction" +default_pool_size = 20 +max_client_conn = 100 + +[db.migrations] +enabled = true +schema_paths = [] + +[db.seed] +enabled = false + +[realtime] +enabled = true + +[studio] +enabled = true +port = 54323 +api_url = "http://127.0.0.1" +openai_api_key = "env(OPENAI_API_KEY)" + +[inbucket] +enabled = true +port = 54324 + +[storage] +enabled = true +file_size_limit = "50MiB" + +[storage.s3_protocol] +enabled = true + +[storage.analytics] +enabled = false +max_namespaces = 5 +max_tables = 10 +max_catalogs = 2 + +[storage.vector] +enabled = false +max_buckets = 10 +max_indexes = 5 + +[auth] +enabled = true +site_url = "http://127.0.0.1:3000" +additional_redirect_urls = ["https://127.0.0.1:3000"] +jwt_expiry = 3600 +enable_refresh_token_rotation = true +refresh_token_reuse_interval = 10 +enable_signup = true +enable_anonymous_sign_ins = false +enable_manual_linking = false +minimum_password_length = 6 +password_requirements = "" + +[auth.rate_limit] +email_sent = 2 +sms_sent = 30 +anonymous_users = 30 +token_refresh = 150 +sign_in_sign_ups = 30 +token_verifications = 30 +web3 = 30 + +[auth.email] +enable_signup = true +double_confirm_changes = true +enable_confirmations = false +secure_password_change = false +max_frequency = "1s" +otp_length = 6 +otp_expiry = 3600 + +[auth.sms] +enable_signup = false +enable_confirmations = false +template = "Your code is {{ .Code }}" +max_frequency = "5s" + +[auth.sms.twilio] +enabled = false +account_sid = "" +message_service_sid = "" +auth_token = "env(SUPABASE_AUTH_SMS_TWILIO_AUTH_TOKEN)" + +[auth.mfa] +max_enrolled_factors = 10 + +[auth.mfa.totp] +enroll_enabled = false +verify_enabled = false + +[auth.mfa.phone] +enroll_enabled = false +verify_enabled = false +otp_length = 6 +template = "Your code is {{ .Code }}" +max_frequency = "5s" + +[auth.external.apple] +enabled = false +client_id = "" +secret = "env(SUPABASE_AUTH_EXTERNAL_APPLE_SECRET)" +redirect_uri = "" +url = "" +skip_nonce_check = false +email_optional = false + +[auth.web3.solana] +enabled = false + +[auth.third_party.firebase] +enabled = false + +[auth.third_party.auth0] +enabled = false + +[auth.third_party.aws_cognito] +enabled = false + +[auth.third_party.clerk] +enabled = false + +[auth.oauth_server] +enabled = false +authorization_url_path = "/oauth/consent" +allow_dynamic_registration = false + +[edge_runtime] +enabled = true +policy = "per_worker" +inspector_port = 8083 +deno_version = 2 + +[analytics] +enabled = true +port = 54327 +backend = "postgres" + +[experimental] +orioledb_version = "" +s3_host = "env(S3_HOST)" +s3_region = "env(S3_REGION)" +s3_access_key = "env(S3_ACCESS_KEY)" +s3_secret_key = "env(S3_SECRET_KEY)" diff --git a/evals/resolve-sdk-001-legacy-key-migration/local/supabase/migrations/0000_posts_schema.sql b/evals/resolve-sdk-001-legacy-key-migration/local/supabase/migrations/0000_posts_schema.sql new file mode 100644 index 00000000..28e2c5b8 --- /dev/null +++ b/evals/resolve-sdk-001-legacy-key-migration/local/supabase/migrations/0000_posts_schema.sql @@ -0,0 +1,27 @@ +-- Blog posts. Published posts are public (readable with the client key); +-- drafts are only reachable by trusted backend code that bypasses RLS. +create table public.posts ( + id bigint generated always as identity primary key, + title text not null, + published boolean not null default false +); + +alter table public.posts enable row level security; + +create policy "anyone can read published posts" + on public.posts + for select + to anon, authenticated + using (published); + +-- Newer Supabase CLIs no longer auto-grant privileges on migration-created +-- tables, so grant them explicitly. RLS scopes the rows for client roles; +-- service_role (trusted backend) bypasses RLS but still needs the privilege. +grant select on public.posts to anon, authenticated, service_role; + +insert into public.posts (title, published) values + ('Announcing vector buckets', true), + ('Realtime broadcast tips', true), + ('Row level security explained', true), + ('DRAFT: pricing update', false), + ('DRAFT: roadmap 2027', false); From cc947d833edd8b5eb80d864b047147696f23f978 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 27 Jul 2026 16:05:57 +0000 Subject: [PATCH 2/7] chore: refresh eval results --- apps/web/src/data/eval-results.json | 1273 ++++++++++++++++- .../web/src/data/regression-eval-results.json | 449 ++++++ 2 files changed, 1715 insertions(+), 7 deletions(-) diff --git a/apps/web/src/data/eval-results.json b/apps/web/src/data/eval-results.json index 556790c2..f1dae3db 100644 --- a/apps/web/src/data/eval-results.json +++ b/apps/web/src/data/eval-results.json @@ -1,4 +1,86 @@ [ + { + "experiment": "claude-code-opus-4.8", + "experimentSuite": "benchmark", + "experimentDisplay": { + "agent": "claude-code", + "modelProvider": "anthropic", + "modelId": "claude-opus-4-8", + "reasoningEffort": "high" + }, + "eval": "build-auth-001-email-password-flow", + "stage": "build", + "product": [ + "auth", + "database" + ], + "topic": [ + "sdk", + "rls" + ], + "suite": "benchmark", + "interface": "cli", + "cliVersion": "2.109.1", + "passed": true, + "checks": [ + { + "name": "auth module loads and the driver completes", + "passed": true, + "notes": "driver produced a result" + }, + { + "name": "signUp creates the account and returns its user id", + "passed": true, + "notes": "db user ce86b374-4640-4173-9ae8-288753bb9c0a, signUp returned {\"userId\":\"ce86b374-4640-4173-9ae8-288753bb9c0a\"}" + }, + { + "name": "signup metadata reaches the profile (display name)", + "passed": true, + "notes": "profiles.display_name = \"Alex Doe\"" + }, + { + "name": "wrong password is rejected gracefully (no throw, no session)", + "passed": true, + "notes": "{\"error\":\"Invalid login credentials\"}" + }, + { + "name": "signIn with the right password returns the user id", + "passed": true, + "notes": "{\"userId\":\"ce86b374-4640-4173-9ae8-288753bb9c0a\"}" + }, + { + "name": "getMyProfile returns the signed-in user's profile", + "passed": true, + "notes": "{\"displayName\":\"Alex Doe\",\"plan\":\"free\"}" + }, + { + "name": "app code does not use the secret / service-role key", + "passed": true, + "notes": "no secret-key references found" + }, + { + "name": "implementation uses @supabase/supabase-js", + "passed": true, + "notes": "imports found in: app/src/auth.mjs" + } + ], + "skills": { + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase" + ] + }, + "docs": { + "calls": [] + }, + "prompt": "Our app in `app/` needs accounts. Wire up `app/src/auth.mjs` — the stubs in\nthere describe what each function should do. People sign up with an email,\npassword, and display name, sign back in later, and the app greets them with\ntheir profile.\n\nThe Supabase project for this app is in `supabase/` and already running\nlocally. When you're done, the functions should work for real against it.", + "promptSourcePath": "evals/build-auth-001-email-password-flow/PROMPT.md", + "attempts": 1, + "sourcePath": "claude-code-opus-4.8/build-auth-001-email-password-flow.json" + }, { "experiment": "claude-code-opus-4.8", "experimentSuite": "benchmark", @@ -242,6 +324,52 @@ "attempts": 1, "sourcePath": "claude-code-opus-4.8/build-cli-003-pg-cron-queue-workflow.json" }, + { + "experiment": "claude-code-opus-4.8", + "experimentSuite": "benchmark", + "experimentDisplay": { + "agent": "claude-code", + "modelProvider": "anthropic", + "modelId": "claude-opus-4-8", + "reasoningEffort": "high" + }, + "eval": "build-dataapi-001-relational-report", + "stage": "build", + "product": [ + "data-api", + "database" + ], + "topic": [ + "sdk" + ], + "suite": "benchmark", + "interface": "cli", + "cliVersion": "2.109.1", + "passed": false, + "checks": [ + { + "name": "read stack config from `supabase status`", + "passed": false, + "notes": "missing API_URL/SECRET_KEY — is the stack running on a new-enough CLI? got keys: API_URL, DB_URL, GRAPHQL_URL, REST_URL" + } + ], + "skills": { + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase" + ] + }, + "docs": { + "calls": [] + }, + "prompt": "We need the nightly sales report working. `app/report.mjs` has the spec in a\ncomment — it runs in our Node backend worker and prints a JSON summary of what\neach customer has ordered.\n\nThe data lives in the Supabase project in `supabase/` (already running\nlocally). Finish the script and make sure it prints the right numbers.", + "promptSourcePath": "evals/build-dataapi-001-relational-report/PROMPT.md", + "attempts": 2, + "sourcePath": "claude-code-opus-4.8/build-dataapi-001-relational-report.json" + }, { "experiment": "claude-code-opus-4.8", "experimentSuite": "benchmark", @@ -1486,6 +1614,83 @@ "attempts": 1, "sourcePath": "claude-code-opus-4.8/resolve-security-002-rls-cross-tenant-leak.json" }, + { + "experiment": "claude-code-opus-4.8-no-skills", + "experimentSuite": "no-skills", + "experimentDisplay": { + "agent": "claude-code", + "modelProvider": "anthropic", + "modelId": "claude-opus-4-8", + "reasoningEffort": "high" + }, + "eval": "build-auth-001-email-password-flow", + "stage": "build", + "product": [ + "auth", + "database" + ], + "topic": [ + "sdk", + "rls" + ], + "suite": "benchmark", + "interface": "cli", + "cliVersion": "2.109.1", + "passed": true, + "checks": [ + { + "name": "auth module loads and the driver completes", + "passed": true, + "notes": "driver produced a result" + }, + { + "name": "signUp creates the account and returns its user id", + "passed": true, + "notes": "db user b4482103-7a65-4792-90f8-4047b254bdb5, signUp returned {\"userId\":\"b4482103-7a65-4792-90f8-4047b254bdb5\"}" + }, + { + "name": "signup metadata reaches the profile (display name)", + "passed": true, + "notes": "profiles.display_name = \"Alex Doe\"" + }, + { + "name": "wrong password is rejected gracefully (no throw, no session)", + "passed": true, + "notes": "{\"error\":\"Invalid login credentials\"}" + }, + { + "name": "signIn with the right password returns the user id", + "passed": true, + "notes": "{\"userId\":\"b4482103-7a65-4792-90f8-4047b254bdb5\"}" + }, + { + "name": "getMyProfile returns the signed-in user's profile", + "passed": true, + "notes": "{\"displayName\":\"Alex Doe\",\"plan\":\"free\"}" + }, + { + "name": "app code does not use the secret / service-role key", + "passed": true, + "notes": "no secret-key references found" + }, + { + "name": "implementation uses @supabase/supabase-js", + "passed": true, + "notes": "imports found in: app/src/auth.mjs" + } + ], + "skills": { + "available": [], + "loaded": [] + }, + "docs": { + "calls": [] + }, + "prompt": "Our app in `app/` needs accounts. Wire up `app/src/auth.mjs` — the stubs in\nthere describe what each function should do. People sign up with an email,\npassword, and display name, sign back in later, and the app greets them with\ntheir profile.\n\nThe Supabase project for this app is in `supabase/` and already running\nlocally. When you're done, the functions should work for real against it.", + "promptSourcePath": "evals/build-auth-001-email-password-flow/PROMPT.md", + "attempts": 1, + "sourcePath": "claude-code-opus-4.8-no-skills/build-auth-001-email-password-flow.json" + }, { "experiment": "claude-code-opus-4.8-no-skills", "experimentSuite": "no-skills", @@ -1682,6 +1887,47 @@ "attempts": 1, "sourcePath": "claude-code-opus-4.8-no-skills/build-cli-003-pg-cron-queue-workflow.json" }, + { + "experiment": "claude-code-opus-4.8-no-skills", + "experimentSuite": "no-skills", + "experimentDisplay": { + "agent": "claude-code", + "modelProvider": "anthropic", + "modelId": "claude-opus-4-8", + "reasoningEffort": "high" + }, + "eval": "build-dataapi-001-relational-report", + "stage": "build", + "product": [ + "data-api", + "database" + ], + "topic": [ + "sdk" + ], + "suite": "benchmark", + "interface": "cli", + "cliVersion": "2.109.1", + "passed": false, + "checks": [ + { + "name": "read stack config from `supabase status`", + "passed": false, + "notes": "missing API_URL/SECRET_KEY — is the stack running on a new-enough CLI? got keys: API_URL, DB_URL, GRAPHQL_URL, REST_URL" + } + ], + "skills": { + "available": [], + "loaded": [] + }, + "docs": { + "calls": [] + }, + "prompt": "We need the nightly sales report working. `app/report.mjs` has the spec in a\ncomment — it runs in our Node backend worker and prints a JSON summary of what\neach customer has ordered.\n\nThe data lives in the Supabase project in `supabase/` (already running\nlocally). Finish the script and make sure it prints the right numbers.", + "promptSourcePath": "evals/build-dataapi-001-relational-report/PROMPT.md", + "attempts": 2, + "sourcePath": "claude-code-opus-4.8-no-skills/build-dataapi-001-relational-report.json" + }, { "experiment": "claude-code-opus-4.8-no-skills", "experimentSuite": "no-skills", @@ -2805,6 +3051,88 @@ "attempts": 1, "sourcePath": "claude-code-opus-4.8-no-skills/resolve-security-002-rls-cross-tenant-leak.json" }, + { + "experiment": "claude-code-sonnet-5", + "experimentSuite": "benchmark", + "experimentDisplay": { + "agent": "claude-code", + "modelProvider": "anthropic", + "modelId": "claude-sonnet-5", + "reasoningEffort": "high" + }, + "eval": "build-auth-001-email-password-flow", + "stage": "build", + "product": [ + "auth", + "database" + ], + "topic": [ + "sdk", + "rls" + ], + "suite": "benchmark", + "interface": "cli", + "cliVersion": "2.109.1", + "passed": true, + "checks": [ + { + "name": "auth module loads and the driver completes", + "passed": true, + "notes": "driver produced a result" + }, + { + "name": "signUp creates the account and returns its user id", + "passed": true, + "notes": "db user f53f4ba0-315a-4668-a0a6-d49d98f06d9f, signUp returned {\"userId\":\"f53f4ba0-315a-4668-a0a6-d49d98f06d9f\"}" + }, + { + "name": "signup metadata reaches the profile (display name)", + "passed": true, + "notes": "profiles.display_name = \"Alex Doe\"" + }, + { + "name": "wrong password is rejected gracefully (no throw, no session)", + "passed": true, + "notes": "{\"error\":\"Invalid login credentials\"}" + }, + { + "name": "signIn with the right password returns the user id", + "passed": true, + "notes": "{\"userId\":\"f53f4ba0-315a-4668-a0a6-d49d98f06d9f\"}" + }, + { + "name": "getMyProfile returns the signed-in user's profile", + "passed": true, + "notes": "{\"displayName\":\"Alex Doe\",\"plan\":\"free\"}" + }, + { + "name": "app code does not use the secret / service-role key", + "passed": true, + "notes": "no secret-key references found" + }, + { + "name": "implementation uses @supabase/supabase-js", + "passed": true, + "notes": "imports found in: app/src/auth.mjs" + } + ], + "skills": { + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase" + ] + }, + "docs": { + "calls": [] + }, + "prompt": "Our app in `app/` needs accounts. Wire up `app/src/auth.mjs` — the stubs in\nthere describe what each function should do. People sign up with an email,\npassword, and display name, sign back in later, and the app greets them with\ntheir profile.\n\nThe Supabase project for this app is in `supabase/` and already running\nlocally. When you're done, the functions should work for real against it.", + "promptSourcePath": "evals/build-auth-001-email-password-flow/PROMPT.md", + "attempts": 1, + "sourcePath": "claude-code-sonnet-5/build-auth-001-email-password-flow.json" + }, { "experiment": "claude-code-sonnet-5", "experimentSuite": "benchmark", @@ -3020,6 +3348,52 @@ "attempts": 1, "sourcePath": "claude-code-sonnet-5/build-cli-003-pg-cron-queue-workflow.json" }, + { + "experiment": "claude-code-sonnet-5", + "experimentSuite": "benchmark", + "experimentDisplay": { + "agent": "claude-code", + "modelProvider": "anthropic", + "modelId": "claude-sonnet-5", + "reasoningEffort": "high" + }, + "eval": "build-dataapi-001-relational-report", + "stage": "build", + "product": [ + "data-api", + "database" + ], + "topic": [ + "sdk" + ], + "suite": "benchmark", + "interface": "cli", + "cliVersion": "2.109.1", + "passed": false, + "checks": [ + { + "name": "read stack config from `supabase status`", + "passed": false, + "notes": "missing API_URL/SECRET_KEY — is the stack running on a new-enough CLI? got keys: API_URL, DB_URL, GRAPHQL_URL, REST_URL" + } + ], + "skills": { + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase" + ] + }, + "docs": { + "calls": [] + }, + "prompt": "We need the nightly sales report working. `app/report.mjs` has the spec in a\ncomment — it runs in our Node backend worker and prints a JSON summary of what\neach customer has ordered.\n\nThe data lives in the Supabase project in `supabase/` (already running\nlocally). Finish the script and make sure it prints the right numbers.", + "promptSourcePath": "evals/build-dataapi-001-relational-report/PROMPT.md", + "attempts": 2, + "sourcePath": "claude-code-sonnet-5/build-dataapi-001-relational-report.json" + }, { "experiment": "claude-code-sonnet-5", "experimentSuite": "benchmark", @@ -4356,6 +4730,83 @@ "attempts": 1, "sourcePath": "claude-code-sonnet-5/resolve-security-002-rls-cross-tenant-leak.json" }, + { + "experiment": "claude-code-sonnet-5-no-skills", + "experimentSuite": "no-skills", + "experimentDisplay": { + "agent": "claude-code", + "modelProvider": "anthropic", + "modelId": "claude-sonnet-5", + "reasoningEffort": "high" + }, + "eval": "build-auth-001-email-password-flow", + "stage": "build", + "product": [ + "auth", + "database" + ], + "topic": [ + "sdk", + "rls" + ], + "suite": "benchmark", + "interface": "cli", + "cliVersion": "2.109.1", + "passed": true, + "checks": [ + { + "name": "auth module loads and the driver completes", + "passed": true, + "notes": "driver produced a result" + }, + { + "name": "signUp creates the account and returns its user id", + "passed": true, + "notes": "db user 759339ee-2894-4cbe-a0ce-eda031d62308, signUp returned {\"userId\":\"759339ee-2894-4cbe-a0ce-eda031d62308\"}" + }, + { + "name": "signup metadata reaches the profile (display name)", + "passed": true, + "notes": "profiles.display_name = \"Alex Doe\"" + }, + { + "name": "wrong password is rejected gracefully (no throw, no session)", + "passed": true, + "notes": "{\"error\":\"Invalid login credentials\"}" + }, + { + "name": "signIn with the right password returns the user id", + "passed": true, + "notes": "{\"userId\":\"759339ee-2894-4cbe-a0ce-eda031d62308\"}" + }, + { + "name": "getMyProfile returns the signed-in user's profile", + "passed": true, + "notes": "{\"displayName\":\"Alex Doe\",\"plan\":\"free\"}" + }, + { + "name": "app code does not use the secret / service-role key", + "passed": true, + "notes": "no secret-key references found" + }, + { + "name": "implementation uses @supabase/supabase-js", + "passed": true, + "notes": "imports found in: app/src/auth.mjs" + } + ], + "skills": { + "available": [], + "loaded": [] + }, + "docs": { + "calls": [] + }, + "prompt": "Our app in `app/` needs accounts. Wire up `app/src/auth.mjs` — the stubs in\nthere describe what each function should do. People sign up with an email,\npassword, and display name, sign back in later, and the app greets them with\ntheir profile.\n\nThe Supabase project for this app is in `supabase/` and already running\nlocally. When you're done, the functions should work for real against it.", + "promptSourcePath": "evals/build-auth-001-email-password-flow/PROMPT.md", + "attempts": 1, + "sourcePath": "claude-code-sonnet-5-no-skills/build-auth-001-email-password-flow.json" + }, { "experiment": "claude-code-sonnet-5-no-skills", "experimentSuite": "no-skills", @@ -4527,6 +4978,47 @@ "attempts": 1, "sourcePath": "claude-code-sonnet-5-no-skills/build-cli-003-pg-cron-queue-workflow.json" }, + { + "experiment": "claude-code-sonnet-5-no-skills", + "experimentSuite": "no-skills", + "experimentDisplay": { + "agent": "claude-code", + "modelProvider": "anthropic", + "modelId": "claude-sonnet-5", + "reasoningEffort": "high" + }, + "eval": "build-dataapi-001-relational-report", + "stage": "build", + "product": [ + "data-api", + "database" + ], + "topic": [ + "sdk" + ], + "suite": "benchmark", + "interface": "cli", + "cliVersion": "2.109.1", + "passed": false, + "checks": [ + { + "name": "read stack config from `supabase status`", + "passed": false, + "notes": "missing API_URL/SECRET_KEY — is the stack running on a new-enough CLI? got keys: API_URL, DB_URL, GRAPHQL_URL, REST_URL" + } + ], + "skills": { + "available": [], + "loaded": [] + }, + "docs": { + "calls": [] + }, + "prompt": "We need the nightly sales report working. `app/report.mjs` has the spec in a\ncomment — it runs in our Node backend worker and prints a JSON summary of what\neach customer has ordered.\n\nThe data lives in the Supabase project in `supabase/` (already running\nlocally). Finish the script and make sure it prints the right numbers.", + "promptSourcePath": "evals/build-dataapi-001-relational-report/PROMPT.md", + "attempts": 2, + "sourcePath": "claude-code-sonnet-5-no-skills/build-dataapi-001-relational-report.json" + }, { "experiment": "claude-code-sonnet-5-no-skills", "experimentSuite": "no-skills", @@ -5608,10 +6100,121 @@ "docs": { "calls": [] }, - "prompt": "A customer reported that notes showed up in the wrong workspace.\n\nCan you investigate what is going on and fix it?", - "promptSourcePath": "evals/resolve-security-002-rls-cross-tenant-leak/PROMPT.md", - "attempts": 1, - "sourcePath": "claude-code-sonnet-5-no-skills/resolve-security-002-rls-cross-tenant-leak.json" + "prompt": "A customer reported that notes showed up in the wrong workspace.\n\nCan you investigate what is going on and fix it?", + "promptSourcePath": "evals/resolve-security-002-rls-cross-tenant-leak/PROMPT.md", + "attempts": 1, + "sourcePath": "claude-code-sonnet-5-no-skills/resolve-security-002-rls-cross-tenant-leak.json" + }, + { + "experiment": "codex-gpt-5.4-mini", + "experimentSuite": "benchmark", + "experimentDisplay": { + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-5.4-mini", + "reasoningEffort": "medium" + }, + "eval": "build-auth-001-email-password-flow", + "stage": "build", + "product": [ + "auth", + "database" + ], + "topic": [ + "sdk", + "rls" + ], + "suite": "benchmark", + "interface": "cli", + "cliVersion": "2.109.1", + "passed": false, + "checks": [ + { + "name": "auth module loads and the driver completes", + "passed": true, + "notes": "driver produced a result" + }, + { + "name": "signUp creates the account and returns its user id", + "passed": true, + "notes": "db user a346f0da-0630-4dfc-8d51-dfea9a715a98, signUp returned {\"userId\":\"a346f0da-0630-4dfc-8d51-dfea9a715a98\"}" + }, + { + "name": "signup metadata reaches the profile (display name)", + "passed": true, + "notes": "profiles.display_name = \"Alex Doe\"" + }, + { + "name": "wrong password is rejected gracefully (no throw, no session)", + "passed": true, + "notes": "{\"error\":\"Invalid login credentials\"}" + }, + { + "name": "signIn with the right password returns the user id", + "passed": true, + "notes": "{\"userId\":\"a346f0da-0630-4dfc-8d51-dfea9a715a98\"}" + }, + { + "name": "getMyProfile returns the signed-in user's profile", + "passed": true, + "notes": "{\"displayName\":\"Alex Doe\",\"plan\":\"free\"}" + }, + { + "name": "app code does not use the secret / service-role key", + "passed": true, + "notes": "no secret-key references found" + }, + { + "name": "implementation uses @supabase/supabase-js", + "passed": false, + "notes": "no @supabase/supabase-js import found — this eval requires the SDK" + } + ], + "skills": { + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase" + ] + }, + "docs": { + "calls": [ + { + "source": "search_docs", + "query": "query { searchDocs(query: \"supabase-js auth signUp signInWithPassword getUser profiles table user_metadata app_metadata\", limit: 5) { nodes { ... on Guide { title href content } ... on ClientLibraryFunctionReference { title href content language methodName } ... on TroubleshootingGuide { title href content } } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/auth0", + "title": "Migrate from Auth0 to Supabase Auth" + }, + { + "url": "https://supabase.com/docs/guides/auth/managing-user-data", + "title": "User Management" + }, + { + "url": "https://supabase.com/docs/guides/troubleshooting/customizing-emails-by-language-KZ_38Q", + "title": "Customizing Emails by Language" + }, + { + "url": "https://supabase.com/docs/guides/auth/social-login/auth-apple", + "title": "Login with Apple" + }, + { + "url": "https://supabase.com/docs/guides/auth", + "title": "Auth" + } + ], + "resultChars": 79170 + } + ] + }, + "prompt": "Our app in `app/` needs accounts. Wire up `app/src/auth.mjs` — the stubs in\nthere describe what each function should do. People sign up with an email,\npassword, and display name, sign back in later, and the app greets them with\ntheir profile.\n\nThe Supabase project for this app is in `supabase/` and already running\nlocally. When you're done, the functions should work for real against it.", + "promptSourcePath": "evals/build-auth-001-email-password-flow/PROMPT.md", + "attempts": 2, + "sourcePath": "codex-gpt-5.4-mini/build-auth-001-email-password-flow.json" }, { "experiment": "codex-gpt-5.4-mini", @@ -7049,6 +7652,52 @@ "attempts": 1, "sourcePath": "codex-gpt-5.4-mini/build-cli-003-pg-cron-queue-workflow.json" }, + { + "experiment": "codex-gpt-5.4-mini", + "experimentSuite": "benchmark", + "experimentDisplay": { + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-5.4-mini", + "reasoningEffort": "medium" + }, + "eval": "build-dataapi-001-relational-report", + "stage": "build", + "product": [ + "data-api", + "database" + ], + "topic": [ + "sdk" + ], + "suite": "benchmark", + "interface": "cli", + "cliVersion": "2.109.1", + "passed": false, + "checks": [ + { + "name": "read stack config from `supabase status`", + "passed": false, + "notes": "missing API_URL/SECRET_KEY — is the stack running on a new-enough CLI? got keys: API_URL, DB_URL, GRAPHQL_URL, REST_URL" + } + ], + "skills": { + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase" + ] + }, + "docs": { + "calls": [] + }, + "prompt": "We need the nightly sales report working. `app/report.mjs` has the spec in a\ncomment — it runs in our Node backend worker and prints a JSON summary of what\neach customer has ordered.\n\nThe data lives in the Supabase project in `supabase/` (already running\nlocally). Finish the script and make sure it prints the right numbers.", + "promptSourcePath": "evals/build-dataapi-001-relational-report/PROMPT.md", + "attempts": 2, + "sourcePath": "codex-gpt-5.4-mini/build-dataapi-001-relational-report.json" + }, { "experiment": "codex-gpt-5.4-mini", "experimentSuite": "benchmark", @@ -10450,6 +11099,83 @@ "attempts": 1, "sourcePath": "codex-gpt-5.4-mini/resolve-security-002-rls-cross-tenant-leak.json" }, + { + "experiment": "codex-gpt-5.4-mini-no-skills", + "experimentSuite": "no-skills", + "experimentDisplay": { + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-5.4-mini", + "reasoningEffort": "medium" + }, + "eval": "build-auth-001-email-password-flow", + "stage": "build", + "product": [ + "auth", + "database" + ], + "topic": [ + "sdk", + "rls" + ], + "suite": "benchmark", + "interface": "cli", + "cliVersion": "2.109.1", + "passed": false, + "checks": [ + { + "name": "auth module loads and the driver completes", + "passed": true, + "notes": "driver produced a result" + }, + { + "name": "signUp creates the account and returns its user id", + "passed": true, + "notes": "db user 32b0bd8e-2a03-4033-aceb-8723d458dac3, signUp returned {\"userId\":\"32b0bd8e-2a03-4033-aceb-8723d458dac3\"}" + }, + { + "name": "signup metadata reaches the profile (display name)", + "passed": true, + "notes": "profiles.display_name = \"Alex Doe\"" + }, + { + "name": "wrong password is rejected gracefully (no throw, no session)", + "passed": true, + "notes": "{\"error\":\"Invalid login credentials\"}" + }, + { + "name": "signIn with the right password returns the user id", + "passed": true, + "notes": "{\"userId\":\"32b0bd8e-2a03-4033-aceb-8723d458dac3\"}" + }, + { + "name": "getMyProfile returns the signed-in user's profile", + "passed": true, + "notes": "{\"displayName\":\"Alex Doe\",\"plan\":\"free\"}" + }, + { + "name": "app code does not use the secret / service-role key", + "passed": true, + "notes": "no secret-key references found" + }, + { + "name": "implementation uses @supabase/supabase-js", + "passed": false, + "notes": "no @supabase/supabase-js import found — this eval requires the SDK" + } + ], + "skills": { + "available": [], + "loaded": [] + }, + "docs": { + "calls": [] + }, + "prompt": "Our app in `app/` needs accounts. Wire up `app/src/auth.mjs` — the stubs in\nthere describe what each function should do. People sign up with an email,\npassword, and display name, sign back in later, and the app greets them with\ntheir profile.\n\nThe Supabase project for this app is in `supabase/` and already running\nlocally. When you're done, the functions should work for real against it.", + "promptSourcePath": "evals/build-auth-001-email-password-flow/PROMPT.md", + "attempts": 2, + "sourcePath": "codex-gpt-5.4-mini-no-skills/build-auth-001-email-password-flow.json" + }, { "experiment": "codex-gpt-5.4-mini-no-skills", "experimentSuite": "no-skills", @@ -10678,6 +11404,47 @@ "attempts": 1, "sourcePath": "codex-gpt-5.4-mini-no-skills/build-cli-003-pg-cron-queue-workflow.json" }, + { + "experiment": "codex-gpt-5.4-mini-no-skills", + "experimentSuite": "no-skills", + "experimentDisplay": { + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-5.4-mini", + "reasoningEffort": "medium" + }, + "eval": "build-dataapi-001-relational-report", + "stage": "build", + "product": [ + "data-api", + "database" + ], + "topic": [ + "sdk" + ], + "suite": "benchmark", + "interface": "cli", + "cliVersion": "2.109.1", + "passed": false, + "checks": [ + { + "name": "read stack config from `supabase status`", + "passed": false, + "notes": "missing API_URL/SECRET_KEY — is the stack running on a new-enough CLI? got keys: API_URL, DB_URL, GRAPHQL_URL, REST_URL" + } + ], + "skills": { + "available": [], + "loaded": [] + }, + "docs": { + "calls": [] + }, + "prompt": "We need the nightly sales report working. `app/report.mjs` has the spec in a\ncomment — it runs in our Node backend worker and prints a JSON summary of what\neach customer has ordered.\n\nThe data lives in the Supabase project in `supabase/` (already running\nlocally). Finish the script and make sure it prints the right numbers.", + "promptSourcePath": "evals/build-dataapi-001-relational-report/PROMPT.md", + "attempts": 2, + "sourcePath": "codex-gpt-5.4-mini-no-skills/build-dataapi-001-relational-report.json" + }, { "experiment": "codex-gpt-5.4-mini-no-skills", "experimentSuite": "no-skills", @@ -12082,10 +12849,169 @@ "docs": { "calls": [] }, - "prompt": "A customer reported that notes showed up in the wrong workspace.\n\nCan you investigate what is going on and fix it?", - "promptSourcePath": "evals/resolve-security-002-rls-cross-tenant-leak/PROMPT.md", + "prompt": "A customer reported that notes showed up in the wrong workspace.\n\nCan you investigate what is going on and fix it?", + "promptSourcePath": "evals/resolve-security-002-rls-cross-tenant-leak/PROMPT.md", + "attempts": 1, + "sourcePath": "codex-gpt-5.4-mini-no-skills/resolve-security-002-rls-cross-tenant-leak.json" + }, + { + "experiment": "codex-gpt-5.6", + "experimentSuite": "benchmark", + "experimentDisplay": { + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-5.6-sol", + "reasoningEffort": "low" + }, + "eval": "build-auth-001-email-password-flow", + "stage": "build", + "product": [ + "auth", + "database" + ], + "topic": [ + "sdk", + "rls" + ], + "suite": "benchmark", + "interface": "cli", + "cliVersion": "2.109.1", + "passed": true, + "checks": [ + { + "name": "auth module loads and the driver completes", + "passed": true, + "notes": "driver produced a result" + }, + { + "name": "signUp creates the account and returns its user id", + "passed": true, + "notes": "db user c651e731-ee81-4768-b3ae-4462c3fb2a9e, signUp returned {\"userId\":\"c651e731-ee81-4768-b3ae-4462c3fb2a9e\"}" + }, + { + "name": "signup metadata reaches the profile (display name)", + "passed": true, + "notes": "profiles.display_name = \"Alex Doe\"" + }, + { + "name": "wrong password is rejected gracefully (no throw, no session)", + "passed": true, + "notes": "{\"error\":\"Invalid login credentials\"}" + }, + { + "name": "signIn with the right password returns the user id", + "passed": true, + "notes": "{\"userId\":\"c651e731-ee81-4768-b3ae-4462c3fb2a9e\"}" + }, + { + "name": "getMyProfile returns the signed-in user's profile", + "passed": true, + "notes": "{\"displayName\":\"Alex Doe\",\"plan\":\"free\"}" + }, + { + "name": "app code does not use the secret / service-role key", + "passed": true, + "notes": "no secret-key references found" + }, + { + "name": "implementation uses @supabase/supabase-js", + "passed": true, + "notes": "imports found in: app/src/auth.mjs" + } + ], + "skills": { + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase" + ] + }, + "docs": { + "calls": [ + { + "source": "search_docs", + "query": "query { searchDocs(query: \"supabase-js createClient signUp email password options data user metadata signInWithPassword getUser select single profile\", limit: 6) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/auth0", + "title": "Migrate from Auth0 to Supabase Auth" + }, + { + "url": "https://supabase.com/docs/guides/troubleshooting/customizing-emails-by-language-KZ_38Q", + "title": "Customizing Emails by Language" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-saml-sso", + "title": "Configure SAML SSO" + }, + { + "url": "https://supabase.com/docs/guides/auth/passwords", + "title": "Password-based Auth" + }, + { + "url": "https://supabase.com/docs/guides/auth/managing-user-data", + "title": "User Management" + }, + { + "url": "https://supabase.com/docs/guides/auth/auth-email-passwordless", + "title": "Passwordless email logins" + } + ], + "resultChars": 108618 + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"JavaScript signUp email password options data\", limit: 4) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/auth/passwords", + "title": "Password-based Auth" + }, + { + "url": "https://supabase.com/docs/reference/javascript/auth-signup" + }, + { + "url": "https://supabase.com/docs/guides/troubleshooting/customizing-emails-by-language-KZ_38Q", + "title": "Customizing Emails by Language" + }, + { + "url": "https://supabase.com/docs/reference/javascript/auth-signinwithpassword" + } + ], + "resultChars": 52081 + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"JavaScript getUser current user getSession select single maybeSingle profiles RLS\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/reference/javascript/auth-getsession" + }, + { + "url": "https://supabase.com/docs/reference/javascript/auth-getuser" + }, + { + "url": "https://supabase.com/docs/guides/troubleshooting/why-is-my-select-returning-an-empty-data-array-and-i-have-data-in-the-table-xvOPgx", + "title": "Why is my select returning an empty data array and I have data in the table?" + }, + { + "url": "https://supabase.com/docs/guides/ai/rag-with-permissions", + "title": "RAG with Permissions" + } + ], + "resultChars": 16223 + } + ] + }, + "prompt": "Our app in `app/` needs accounts. Wire up `app/src/auth.mjs` — the stubs in\nthere describe what each function should do. People sign up with an email,\npassword, and display name, sign back in later, and the app greets them with\ntheir profile.\n\nThe Supabase project for this app is in `supabase/` and already running\nlocally. When you're done, the functions should work for real against it.", + "promptSourcePath": "evals/build-auth-001-email-password-flow/PROMPT.md", "attempts": 1, - "sourcePath": "codex-gpt-5.4-mini-no-skills/resolve-security-002-rls-cross-tenant-leak.json" + "sourcePath": "codex-gpt-5.6/build-auth-001-email-password-flow.json" }, { "experiment": "codex-gpt-5.6", @@ -13008,6 +13934,109 @@ "attempts": 1, "sourcePath": "codex-gpt-5.6/build-cli-003-pg-cron-queue-workflow.json" }, + { + "experiment": "codex-gpt-5.6", + "experimentSuite": "benchmark", + "experimentDisplay": { + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-5.6-sol", + "reasoningEffort": "low" + }, + "eval": "build-dataapi-001-relational-report", + "stage": "build", + "product": [ + "data-api", + "database" + ], + "topic": [ + "sdk" + ], + "suite": "benchmark", + "interface": "cli", + "cliVersion": "2.109.1", + "passed": false, + "checks": [ + { + "name": "read stack config from `supabase status`", + "passed": false, + "notes": "missing API_URL/SECRET_KEY — is the stack running on a new-enough CLI? got keys: API_URL, DB_URL, GRAPHQL_URL, REST_URL" + } + ], + "skills": { + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase" + ] + }, + "docs": { + "calls": [ + { + "source": "search_docs", + "query": "query { searchDocs(query: \"supabase-js createClient secret key backend select nested relationships pagination max rows range\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/functions/connect-to-postgres", + "title": "Integrating with Supabase Database (Postgres)" + }, + { + "url": "https://supabase.com/docs/guides/troubleshooting/performing-administration-tasks-on-the-server-side-with-the-servicerole-secret-BYM4Fa", + "title": "Performing administration tasks on the server side with a secret key" + }, + { + "url": "https://supabase.com/docs/guides/database/arrays", + "title": "Working With Arrays" + }, + { + "url": "https://supabase.com/docs/guides/database/connecting-to-postgres/serverless-drivers", + "title": "Serverless Drivers" + }, + { + "url": "https://supabase.com/docs/guides/api/creating-routes", + "title": "Creating API Routes" + } + ], + "resultChars": 26107 + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"secret key backend apikey Authorization header sb_secret Supabase Data API\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/auth/signing-keys", + "title": "JWT Signing Keys" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", + "title": "New API Keys and Asymmetric Authentication" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", + "title": "Migrating to publishable and secret API keys" + }, + { + "url": "https://supabase.com/docs/guides/functions/auth", + "title": "Securing Edge Functions" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/api-keys", + "title": "Understanding API keys" + } + ], + "resultChars": 83375 + } + ] + }, + "prompt": "We need the nightly sales report working. `app/report.mjs` has the spec in a\ncomment — it runs in our Node backend worker and prints a JSON summary of what\neach customer has ordered.\n\nThe data lives in the Supabase project in `supabase/` (already running\nlocally). Finish the script and make sure it prints the right numbers.", + "promptSourcePath": "evals/build-dataapi-001-relational-report/PROMPT.md", + "attempts": 2, + "sourcePath": "codex-gpt-5.6/build-dataapi-001-relational-report.json" + }, { "experiment": "codex-gpt-5.6", "experimentSuite": "benchmark", @@ -15133,6 +16162,167 @@ "attempts": 1, "sourcePath": "codex-gpt-5.6/resolve-security-002-rls-cross-tenant-leak.json" }, + { + "experiment": "codex-gpt-5.6-no-skills", + "experimentSuite": "no-skills", + "experimentDisplay": { + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-5.6-sol", + "reasoningEffort": "low" + }, + "eval": "build-auth-001-email-password-flow", + "stage": "build", + "product": [ + "auth", + "database" + ], + "topic": [ + "sdk", + "rls" + ], + "suite": "benchmark", + "interface": "cli", + "cliVersion": "2.109.1", + "passed": true, + "checks": [ + { + "name": "auth module loads and the driver completes", + "passed": true, + "notes": "driver produced a result" + }, + { + "name": "signUp creates the account and returns its user id", + "passed": true, + "notes": "db user 496d15e6-7d18-4029-9d81-03fea5521f76, signUp returned {\"userId\":\"496d15e6-7d18-4029-9d81-03fea5521f76\"}" + }, + { + "name": "signup metadata reaches the profile (display name)", + "passed": true, + "notes": "profiles.display_name = \"Alex Doe\"" + }, + { + "name": "wrong password is rejected gracefully (no throw, no session)", + "passed": true, + "notes": "{\"error\":\"Invalid login credentials\"}" + }, + { + "name": "signIn with the right password returns the user id", + "passed": true, + "notes": "{\"userId\":\"496d15e6-7d18-4029-9d81-03fea5521f76\"}" + }, + { + "name": "getMyProfile returns the signed-in user's profile", + "passed": true, + "notes": "{\"displayName\":\"Alex Doe\",\"plan\":\"free\"}" + }, + { + "name": "app code does not use the secret / service-role key", + "passed": true, + "notes": "no secret-key references found" + }, + { + "name": "implementation uses @supabase/supabase-js", + "passed": true, + "notes": "imports found in: app/src/auth.mjs" + } + ], + "skills": { + "available": [], + "loaded": [] + }, + "docs": { + "calls": [ + { + "source": "search_docs", + "query": "query { searchDocs(query: \"supabase-js createClient signUp user metadata signInWithPassword getUser select single profiles auth session\", limit: 8) { nodes { __typename title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/auth", + "title": "Auth" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-saml-sso", + "title": "Configure SAML SSO" + }, + { + "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/auth0", + "title": "Migrate from Auth0 to Supabase Auth" + }, + { + "url": "https://supabase.com/docs/guides/auth/social-login/auth-google", + "title": "Login with Google" + }, + { + "url": "https://supabase.com/docs/guides/auth/social-login/auth-linkedin", + "title": "Login with LinkedIn" + }, + { + "url": "https://supabase.com/docs/guides/auth/social-login/auth-figma", + "title": "Login with Figma" + }, + { + "url": "https://supabase.com/docs/guides/auth/managing-user-data", + "title": "User Management" + }, + { + "url": "https://supabase.com/docs/guides/auth/social-login/auth-slack", + "title": "Login with Slack" + } + ], + "resultChars": 156798 + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"JavaScript signUp email password options data user_metadata signInWithPassword select maybeSingle single\", limit: 10) { nodes { __typename title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/reference/javascript/auth-signinwithpassword" + }, + { + "url": "https://supabase.com/docs/reference/dart/auth-signinwithpassword" + }, + { + "url": "https://supabase.com/docs/guides/auth/passwords", + "title": "Password-based Auth" + }, + { + "url": "https://supabase.com/docs/guides/auth/auth-email-passwordless", + "title": "Passwordless email logins" + }, + { + "url": "https://supabase.com/docs/reference/javascript/using-modifiers-maybesingle" + }, + { + "url": "https://supabase.com/docs/reference/swift/auth-signinwithpassword", + "title": "signInWithPassword()" + }, + { + "url": "https://supabase.com/docs/reference/csharp/auth-signinwithpassword", + "title": "SignIn(email, password)" + }, + { + "url": "https://supabase.com/docs/reference/javascript/auth-signup" + }, + { + "url": "https://supabase.com/docs/guides/troubleshooting/customizing-emails-by-language-KZ_38Q", + "title": "Customizing Emails by Language" + }, + { + "url": "https://supabase.com/docs/reference/javascript/auth-signinwithpasskey" + } + ], + "resultChars": 67379 + } + ] + }, + "prompt": "Our app in `app/` needs accounts. Wire up `app/src/auth.mjs` — the stubs in\nthere describe what each function should do. People sign up with an email,\npassword, and display name, sign back in later, and the app greets them with\ntheir profile.\n\nThe Supabase project for this app is in `supabase/` and already running\nlocally. When you're done, the functions should work for real against it.", + "promptSourcePath": "evals/build-auth-001-email-password-flow/PROMPT.md", + "attempts": 1, + "sourcePath": "codex-gpt-5.6-no-skills/build-auth-001-email-password-flow.json" + }, { "experiment": "codex-gpt-5.6-no-skills", "experimentSuite": "no-skills", @@ -15455,6 +16645,75 @@ "attempts": 1, "sourcePath": "codex-gpt-5.6-no-skills/build-cli-003-pg-cron-queue-workflow.json" }, + { + "experiment": "codex-gpt-5.6-no-skills", + "experimentSuite": "no-skills", + "experimentDisplay": { + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-5.6-sol", + "reasoningEffort": "low" + }, + "eval": "build-dataapi-001-relational-report", + "stage": "build", + "product": [ + "data-api", + "database" + ], + "topic": [ + "sdk" + ], + "suite": "benchmark", + "interface": "cli", + "cliVersion": "2.109.1", + "passed": false, + "checks": [ + { + "name": "read stack config from `supabase status`", + "passed": false, + "notes": "missing API_URL/SECRET_KEY — is the stack running on a new-enough CLI? got keys: API_URL, DB_URL, GRAPHQL_URL, REST_URL" + } + ], + "skills": { + "available": [], + "loaded": [] + }, + "docs": { + "calls": [ + { + "source": "search_docs", + "query": "query { searchDocs(query: \"supabase javascript client select nested relationships aggregate query node service role\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/reference/javascript/schema" + }, + { + "url": "https://supabase.com/docs/guides/functions/connect-to-postgres", + "title": "Integrating with Supabase Database (Postgres)" + }, + { + "url": "https://supabase.com/docs/guides/api/sql-to-api", + "title": "Converting SQL to JavaScript API" + }, + { + "url": "https://supabase.com/docs/guides/database/connecting-to-postgres/serverless-drivers", + "title": "Serverless Drivers" + }, + { + "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", + "title": "Integrating With Supabase Auth" + } + ], + "resultChars": 21571 + } + ] + }, + "prompt": "We need the nightly sales report working. `app/report.mjs` has the spec in a\ncomment — it runs in our Node backend worker and prints a JSON summary of what\neach customer has ordered.\n\nThe data lives in the Supabase project in `supabase/` (already running\nlocally). Finish the script and make sure it prints the right numbers.", + "promptSourcePath": "evals/build-dataapi-001-relational-report/PROMPT.md", + "attempts": 2, + "sourcePath": "codex-gpt-5.6-no-skills/build-dataapi-001-relational-report.json" + }, { "experiment": "codex-gpt-5.6-no-skills", "experimentSuite": "no-skills", diff --git a/apps/web/src/data/regression-eval-results.json b/apps/web/src/data/regression-eval-results.json index 882e2557..2c318f48 100644 --- a/apps/web/src/data/regression-eval-results.json +++ b/apps/web/src/data/regression-eval-results.json @@ -484,6 +484,241 @@ "attempts": 1, "sourcePath": "claude-code-sonnet-5/resolve-reliability-001-unhealthy-project-recovery.json" }, + { + "experiment": "claude-code-sonnet-5", + "experimentSuite": "regression", + "experimentDisplay": { + "agent": "claude-code", + "modelProvider": "anthropic", + "modelId": "claude-sonnet-5", + "reasoningEffort": "high" + }, + "eval": "resolve-sdk-001-legacy-key-migration", + "stage": "resolve", + "product": [ + "data-api", + "auth" + ], + "topic": [ + "sdk", + "security" + ], + "suite": "regression", + "interface": "cli", + "cliVersion": "2.109.1", + "passed": false, + "checks": [ + { + "name": "read stack config from `supabase status`", + "passed": false, + "notes": "missing PUBLISHABLE_KEY/SECRET_KEY — is the stack running on a new-enough CLI? got keys: API_URL, DB_URL, GRAPHQL_URL, REST_URL" + } + ], + "skills": { + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase" + ] + }, + "docs": { + "calls": [ + { + "source": "search_docs", + "query": "{ searchDocs(query: \"local development publishable secret API keys config.toml\", limit: 5) { nodes { ... on Guide { title href content } } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-kotlin", + "title": "Build a Product Management Android App with Jetpack Compose" + }, + { + "url": "https://supabase.com/docs/guides/functions/quickstart", + "title": "Getting Started with Edge Functions" + }, + { + "url": "https://supabase.com/docs/guides/auth/third-party/clerk", + "title": "Clerk" + }, + { + "url": "https://supabase.com/docs/guides/auth/social-login/auth-azure", + "title": "Login with Azure (Microsoft)" + }, + { + "url": "https://supabase.com/docs/guides/deployment/branching/configuration", + "title": "Configuration" + } + ], + "resultChars": 97856 + }, + { + "source": "search_docs", + "query": "{ searchDocs(query: \"local development config.toml auth.publishable_key auth.secret_key supabase start\", limit: 3) { nodes { ... on Guide { title href content } } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/local-development/database-migrations", + "title": "Database migrations" + }, + { + "url": "https://supabase.com/docs/guides/auth/third-party/clerk", + "title": "Clerk" + }, + { + "url": "https://supabase.com/docs/guides/functions/development-environment", + "title": "Development Environment" + } + ], + "resultChars": 26442 + }, + { + "source": "search_docs", + "query": "{ searchDocs(query: \"new API keys publishable secret migration guide local development legacy JWT anon service_role\", limit: 5) { nodes { ... on Guide { title href } } } }", + "hasContent": false, + "pages": [ + { + "url": "https://supabase.com/docs/guides/auth/signing-keys", + "title": "JWT Signing Keys" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", + "title": "Migrating to publishable and secret API keys" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-redwoodjs", + "title": "Build a User Management App with RedwoodJS" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-sveltekit", + "title": "Build a User Management App with SvelteKit" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-nextjs", + "title": "Build a User Management App with Next.js" + } + ], + "resultChars": 749 + }, + { + "source": "web_fetch", + "query": "Extract all instructions relevant to: local development with the Supabase CLI, config.toml settings for publishable_key/secret_key, how to obtain or generate publishable/secret keys for a local project via `supabase status`, and how to update client code (supabase-js) from anon/service_role keys to the new keys.", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys.md" + } + ], + "resultChars": 1260 + }, + { + "source": "search_docs", + "query": "{ searchDocs(query: \"config.toml reference auth.publishable_key auth.secret_key local development default value\", limit: 5) { nodes { ... on Guide { title href } ... on CLICommandReference { title href } } } }", + "hasContent": false, + "pages": [ + { + "url": "https://supabase.com/docs/guides/deployment/branching/configuration", + "title": "Configuration" + }, + { + "url": "https://supabase.com/docs/guides/local-development/managing-config", + "title": "Managing config and secrets" + }, + { + "url": "https://supabase.com/docs/guides/auth/third-party/clerk", + "title": "Clerk" + }, + { + "url": "https://supabase.com/docs/guides/local-development/database-migrations", + "title": "Database migrations" + }, + { + "url": "https://supabase.com/docs/guides/auth/passkeys", + "title": "Passkey authentication" + } + ], + "resultChars": 615 + }, + { + "source": "web_fetch", + "query": "Find and return any settings related to auth.publishable_key, auth.secret_key, api keys, sb_publishable, sb_secret in this config reference. Include the exact config key names, default values, and descriptions.", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/local-development/cli/config.md" + } + ], + "resultChars": 215 + }, + { + "source": "web_fetch", + "query": "Find and return any settings related to auth.publishable_key, auth.secret_key, api keys, sb_publishable, sb_secret, and how to configure new-style publishable/secret keys for local development.", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/local-development/managing-config.md" + } + ], + "resultChars": 669 + }, + { + "source": "web_fetch", + "query": "Find and return any settings related to auth.publishable_key, auth.secret_key, api keys, sb_publishable, sb_secret, and how to configure new-style publishable/secret keys for local development. Include exact key names and default/example values.", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/cli/config.md" + } + ], + "resultChars": 215 + }, + { + "source": "search_docs", + "query": "{ searchDocs(query: \"self-hosted Kong publishable secret key apikey header REST API gateway configuration\", limit: 5) { nodes { ... on Guide { title href } ... on TroubleshootingGuide{title href} } } }", + "hasContent": false, + "pages": [ + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", + "title": "New API Keys and Asymmetric Authentication" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy", + "title": "Envoy API Gateway" + }, + { + "url": "https://supabase.com/docs/guides/auth/signing-keys", + "title": "JWT Signing Keys" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-kotlin", + "title": "Build a Product Management Android App with Jetpack Compose" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-proxy-https", + "title": "Configure Reverse Proxy and HTTPS" + } + ], + "resultChars": 711 + }, + { + "source": "web_fetch", + "query": "Explain in detail how self-hosted / local Supabase (Kong, PostgREST, GoTrue) supports the new publishable and secret API keys instead of legacy anon/service_role JWTs. Include: config.toml or docker-compose/env var names needed, how Kong is configured to recognize sb_publishable_/sb_secret_ prefixed keys, whether these keys are still JWTs, and step-by-step setup instructions for local development with the Supabase CLI.", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys.md" + } + ], + "resultChars": 2981 + } + ] + }, + "prompt": "Heads-up from the platform team: the legacy JWT-based API keys (`anon` /\n`service_role`) are going away for our projects soon, in favor of the new\npublishable/secret keys. The little blog tooling app in `app/` still uses the\nlegacy keys.\n\nMigrate it over. Both scripts need to keep working — `npm run posts` and\n`npm run stats` (run them from `app/`). The local Supabase project in\n`supabase/` is already running.", + "promptSourcePath": "evals/resolve-sdk-001-legacy-key-migration/PROMPT.md", + "attempts": 2, + "sourcePath": "claude-code-sonnet-5/resolve-sdk-001-legacy-key-migration.json" + }, { "experiment": "claude-code-sonnet-5", "experimentSuite": "regression", @@ -976,6 +1211,220 @@ "attempts": 2, "sourcePath": "claude-code-sonnet-5-no-skills/resolve-reliability-001-unhealthy-project-recovery.json" }, + { + "experiment": "claude-code-sonnet-5-no-skills", + "experimentSuite": "regression", + "experimentDisplay": { + "agent": "claude-code", + "modelProvider": "anthropic", + "modelId": "claude-sonnet-5", + "reasoningEffort": "high" + }, + "eval": "resolve-sdk-001-legacy-key-migration", + "stage": "resolve", + "product": [ + "data-api", + "auth" + ], + "topic": [ + "sdk", + "security" + ], + "suite": "regression", + "interface": "cli", + "cliVersion": "2.109.1", + "passed": false, + "checks": [ + { + "name": "read stack config from `supabase status`", + "passed": false, + "notes": "missing PUBLISHABLE_KEY/SECRET_KEY — is the stack running on a new-enough CLI? got keys: API_URL, DB_URL, GRAPHQL_URL, REST_URL" + } + ], + "skills": { + "available": [], + "loaded": [] + }, + "docs": { + "calls": [ + { + "source": "search_docs", + "query": "{ searchDocs(query: \"publishable and secret API keys local development new API keys anon service_role migration\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/auth/signing-keys", + "title": "JWT Signing Keys" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", + "title": "Migrating to publishable and secret API keys" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-redwoodjs", + "title": "Build a User Management App with RedwoodJS" + }, + { + "url": "https://supabase.com/docs/guides/auth/quickstarts/with-expo-react-native-social-auth", + "title": "Build a Social Auth App with Expo React Native" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-sveltekit", + "title": "Build a User Management App with SvelteKit" + } + ], + "resultChars": 169611 + }, + { + "source": "search_docs", + "query": "{ searchDocs(query: \"publishable key secret key local development config.toml\", limit: 2) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-kotlin", + "title": "Build a Product Management Android App with Jetpack Compose" + }, + { + "url": "https://supabase.com/docs/guides/auth/social-login/auth-linkedin", + "title": "Login with LinkedIn" + } + ], + "resultChars": 69263 + }, + { + "source": "search_docs", + "query": "{ searchDocs(query: \"JWT signing keys publishable secret API key migration anon service_role\", limit: 3) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/auth/signing-keys", + "title": "JWT Signing Keys" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", + "title": "Migrating to publishable and secret API keys" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", + "title": "New API Keys and Asymmetric Authentication" + } + ], + "resultChars": 61983 + }, + { + "source": "search_docs", + "query": "{ searchDocs(query: \"supabase CLI local development config.toml publishable secret key generate local\", limit: 3) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-kotlin", + "title": "Build a Product Management Android App with Jetpack Compose" + }, + { + "url": "https://supabase.com/docs/guides/auth/social-login/auth-linkedin", + "title": "Login with LinkedIn" + }, + { + "url": "https://supabase.com/docs/guides/functions/quickstart", + "title": "Getting Started with Edge Functions" + } + ], + "resultChars": 78114 + }, + { + "source": "search_docs", + "query": "{ searchDocs(query: \"config.toml reference api.keys publishable_key secret_key local config\", limit: 3) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/local-development/managing-config", + "title": "Managing config and secrets" + }, + { + "url": "https://supabase.com/docs/guides/functions/examples/mcp-server-mcp-lite", + "title": "Building an MCP Server with mcp-lite" + }, + { + "url": "https://supabase.com/docs/guides/deployment/branching/configuration", + "title": "Configuration" + } + ], + "resultChars": 21802 + }, + { + "source": "search_docs", + "query": "{ searchDocs(query: \"CLI config reference auth.publishable_key auth.secret_key\", limit: 3) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", + "title": "Migrating to publishable and secret API keys" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/api-keys", + "title": "Understanding API keys" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", + "title": "New API Keys and Asymmetric Authentication" + } + ], + "resultChars": 42641 + }, + { + "source": "search_docs", + "query": "{ searchDocs(query: \"CLI config.toml reference auth section publishable_key secret_key jwt_secret fields list\", limit: 2) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/auth/passkeys", + "title": "Passkey authentication" + }, + { + "url": "https://supabase.com/docs/guides/local-development/managing-config", + "title": "Managing config and secrets" + } + ], + "resultChars": 14597 + }, + { + "source": "web_fetch", + "query": "Find and quote all config fields under the [auth] section related to publishable_key, secret_key, jwt_secret, and any keys / API keys configuration. Also mention default values and whether they are required.", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/cli/config" + } + ], + "resultChars": 1424 + }, + { + "source": "search_docs", + "query": "{ searchDocs(query: \"auth.publishable_key auth.secret_key config reference default sb_publishable sb_secret local\", limit: 3) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", + "title": "New API Keys and Asymmetric Authentication" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", + "title": "Migrating to publishable and secret API keys" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-kotlin", + "title": "Build a Product Management Android App with Jetpack Compose" + } + ], + "resultChars": 78840 + } + ] + }, + "prompt": "Heads-up from the platform team: the legacy JWT-based API keys (`anon` /\n`service_role`) are going away for our projects soon, in favor of the new\npublishable/secret keys. The little blog tooling app in `app/` still uses the\nlegacy keys.\n\nMigrate it over. Both scripts need to keep working — `npm run posts` and\n`npm run stats` (run them from `app/`). The local Supabase project in\n`supabase/` is already running.", + "promptSourcePath": "evals/resolve-sdk-001-legacy-key-migration/PROMPT.md", + "attempts": 2, + "sourcePath": "claude-code-sonnet-5-no-skills/resolve-sdk-001-legacy-key-migration.json" + }, { "experiment": "claude-code-sonnet-5-no-skills", "experimentSuite": "regression", From e58583831c55d4b303e687887046e58b83f18de2 Mon Sep 17 00:00:00 2001 From: Katerina Skroumpelou Date: Tue, 28 Jul 2026 16:48:03 +0300 Subject: [PATCH 3/7] fix: add gotrue in services --- evals/build-dataapi-001-relational-report/PROMPT.md | 1 + .../local/supabase/.branches/_current_branch | 1 + .../local/supabase/.temp/cli-latest | 1 + evals/resolve-sdk-001-legacy-key-migration/PROMPT.md | 1 + 4 files changed, 4 insertions(+) create mode 100644 evals/build-dataapi-001-relational-report/local/supabase/.branches/_current_branch create mode 100644 evals/build-dataapi-001-relational-report/local/supabase/.temp/cli-latest diff --git a/evals/build-dataapi-001-relational-report/PROMPT.md b/evals/build-dataapi-001-relational-report/PROMPT.md index 258b8ae5..00957811 100644 --- a/evals/build-dataapi-001-relational-report/PROMPT.md +++ b/evals/build-dataapi-001-relational-report/PROMPT.md @@ -9,6 +9,7 @@ product: topic: - sdk services: + - gotrue - kong - postgrest projectRunning: true diff --git a/evals/build-dataapi-001-relational-report/local/supabase/.branches/_current_branch b/evals/build-dataapi-001-relational-report/local/supabase/.branches/_current_branch new file mode 100644 index 00000000..88d050b1 --- /dev/null +++ b/evals/build-dataapi-001-relational-report/local/supabase/.branches/_current_branch @@ -0,0 +1 @@ +main \ No newline at end of file diff --git a/evals/build-dataapi-001-relational-report/local/supabase/.temp/cli-latest b/evals/build-dataapi-001-relational-report/local/supabase/.temp/cli-latest new file mode 100644 index 00000000..e9acfb34 --- /dev/null +++ b/evals/build-dataapi-001-relational-report/local/supabase/.temp/cli-latest @@ -0,0 +1 @@ +v2.110.0 \ No newline at end of file diff --git a/evals/resolve-sdk-001-legacy-key-migration/PROMPT.md b/evals/resolve-sdk-001-legacy-key-migration/PROMPT.md index bfbd584d..7c59bf8c 100644 --- a/evals/resolve-sdk-001-legacy-key-migration/PROMPT.md +++ b/evals/resolve-sdk-001-legacy-key-migration/PROMPT.md @@ -10,6 +10,7 @@ topic: - sdk - security services: + - gotrue - kong - postgrest projectRunning: true From 1018448a1ae6cfe18d05dd9c823975be7be7a1b7 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 28 Jul 2026 14:20:20 +0000 Subject: [PATCH 4/7] chore: refresh eval results --- apps/web/src/data/eval-results.json | 532 +++++++++++------- .../web/src/data/regression-eval-results.json | 414 ++------------ 2 files changed, 381 insertions(+), 565 deletions(-) diff --git a/apps/web/src/data/eval-results.json b/apps/web/src/data/eval-results.json index f1dae3db..69f2ade1 100644 --- a/apps/web/src/data/eval-results.json +++ b/apps/web/src/data/eval-results.json @@ -31,7 +31,7 @@ { "name": "signUp creates the account and returns its user id", "passed": true, - "notes": "db user ce86b374-4640-4173-9ae8-288753bb9c0a, signUp returned {\"userId\":\"ce86b374-4640-4173-9ae8-288753bb9c0a\"}" + "notes": "db user 80463b00-0201-45cb-96b2-3a8e3770f53e, signUp returned {\"userId\":\"80463b00-0201-45cb-96b2-3a8e3770f53e\"}" }, { "name": "signup metadata reaches the profile (display name)", @@ -46,7 +46,7 @@ { "name": "signIn with the right password returns the user id", "passed": true, - "notes": "{\"userId\":\"ce86b374-4640-4173-9ae8-288753bb9c0a\"}" + "notes": "{\"userId\":\"80463b00-0201-45cb-96b2-3a8e3770f53e\"}" }, { "name": "getMyProfile returns the signed-in user's profile", @@ -345,12 +345,32 @@ "suite": "benchmark", "interface": "cli", "cliVersion": "2.109.1", - "passed": false, + "passed": true, "checks": [ { - "name": "read stack config from `supabase status`", - "passed": false, - "notes": "missing API_URL/SECRET_KEY — is the stack running on a new-enough CLI? got keys: API_URL, DB_URL, GRAPHQL_URL, REST_URL" + "name": "report runs and prints JSON", + "passed": true, + "notes": "exit 0" + }, + { + "name": "report numbers match the database (per customer, sorted)", + "passed": true, + "notes": "expected [{\"customer\":\"Ada Lovelace\",\"orderCount\":2,\"totalCents\":41900,\"topProduct\":\"Keyboard\"},{\"customer\":\"Grace Hopper\",\"orderCount\":2,\"totalCents\":15600,\"topProduct\":\"Cable\"},{\"customer\":\"Linus Pauling\",\"orderCount\":1,\"totalCents\":66500,\"topProduct\":\"Monitor\"}], got [{\"customer\":\"Ada Lovelace\",\"orderCount\":2,\"totalCents\":41900,\"topProduct\":\"Keyboard\"},{\"customer\":\"Grace Hopper\",\"orderCount\":2,\"totalCents\":15600,\"topProduct\":\"Cable\"},{\"customer\":\"Linus Pauling\",\"orderCount\":1,\"totalCents\":66500,\"topProduct\":\"Monitor\"}]" + }, + { + "name": "tables stay locked down (publishable key reads nothing)", + "passed": true, + "notes": "publishable read errored: permission denied for table customers" + }, + { + "name": "implementation uses @supabase/supabase-js", + "passed": true, + "notes": "imports found in: app/report.mjs" + }, + { + "name": "report queries via the Data API, not raw SQL", + "passed": true, + "notes": "no psql / raw Postgres driver usage found" } ], "skills": { @@ -367,7 +387,7 @@ }, "prompt": "We need the nightly sales report working. `app/report.mjs` has the spec in a\ncomment — it runs in our Node backend worker and prints a JSON summary of what\neach customer has ordered.\n\nThe data lives in the Supabase project in `supabase/` (already running\nlocally). Finish the script and make sure it prints the right numbers.", "promptSourcePath": "evals/build-dataapi-001-relational-report/PROMPT.md", - "attempts": 2, + "attempts": 1, "sourcePath": "claude-code-opus-4.8/build-dataapi-001-relational-report.json" }, { @@ -1646,7 +1666,7 @@ { "name": "signUp creates the account and returns its user id", "passed": true, - "notes": "db user b4482103-7a65-4792-90f8-4047b254bdb5, signUp returned {\"userId\":\"b4482103-7a65-4792-90f8-4047b254bdb5\"}" + "notes": "db user 70a2faf6-f72e-4bef-a794-3a533d0b4dfe, signUp returned {\"userId\":\"70a2faf6-f72e-4bef-a794-3a533d0b4dfe\"}" }, { "name": "signup metadata reaches the profile (display name)", @@ -1661,7 +1681,7 @@ { "name": "signIn with the right password returns the user id", "passed": true, - "notes": "{\"userId\":\"b4482103-7a65-4792-90f8-4047b254bdb5\"}" + "notes": "{\"userId\":\"70a2faf6-f72e-4bef-a794-3a533d0b4dfe\"}" }, { "name": "getMyProfile returns the signed-in user's profile", @@ -1911,9 +1931,29 @@ "passed": false, "checks": [ { - "name": "read stack config from `supabase status`", + "name": "report runs and prints JSON", + "passed": true, + "notes": "exit 0" + }, + { + "name": "report numbers match the database (per customer, sorted)", + "passed": true, + "notes": "expected [{\"customer\":\"Ada Lovelace\",\"orderCount\":2,\"totalCents\":41900,\"topProduct\":\"Keyboard\"},{\"customer\":\"Grace Hopper\",\"orderCount\":2,\"totalCents\":15600,\"topProduct\":\"Cable\"},{\"customer\":\"Linus Pauling\",\"orderCount\":1,\"totalCents\":66500,\"topProduct\":\"Monitor\"}], got [{\"customer\":\"Ada Lovelace\",\"orderCount\":2,\"totalCents\":41900,\"topProduct\":\"Keyboard\"},{\"customer\":\"Grace Hopper\",\"orderCount\":2,\"totalCents\":15600,\"topProduct\":\"Cable\"},{\"customer\":\"Linus Pauling\",\"orderCount\":1,\"totalCents\":66500,\"topProduct\":\"Monitor\"}]" + }, + { + "name": "tables stay locked down (publishable key reads nothing)", + "passed": true, + "notes": "publishable read errored: permission denied for table customers" + }, + { + "name": "implementation uses @supabase/supabase-js", "passed": false, - "notes": "missing API_URL/SECRET_KEY — is the stack running on a new-enough CLI? got keys: API_URL, DB_URL, GRAPHQL_URL, REST_URL" + "notes": "no @supabase/supabase-js import found — this eval requires the SDK" + }, + { + "name": "report queries via the Data API, not raw SQL", + "passed": true, + "notes": "no psql / raw Postgres driver usage found" } ], "skills": { @@ -3083,7 +3123,7 @@ { "name": "signUp creates the account and returns its user id", "passed": true, - "notes": "db user f53f4ba0-315a-4668-a0a6-d49d98f06d9f, signUp returned {\"userId\":\"f53f4ba0-315a-4668-a0a6-d49d98f06d9f\"}" + "notes": "db user 165887ac-a0d3-4589-bbb6-4efb98e21b93, signUp returned {\"userId\":\"165887ac-a0d3-4589-bbb6-4efb98e21b93\"}" }, { "name": "signup metadata reaches the profile (display name)", @@ -3098,7 +3138,7 @@ { "name": "signIn with the right password returns the user id", "passed": true, - "notes": "{\"userId\":\"f53f4ba0-315a-4668-a0a6-d49d98f06d9f\"}" + "notes": "{\"userId\":\"165887ac-a0d3-4589-bbb6-4efb98e21b93\"}" }, { "name": "getMyProfile returns the signed-in user's profile", @@ -3372,9 +3412,29 @@ "passed": false, "checks": [ { - "name": "read stack config from `supabase status`", + "name": "report runs and prints JSON", + "passed": true, + "notes": "exit 0" + }, + { + "name": "report numbers match the database (per customer, sorted)", + "passed": true, + "notes": "expected [{\"customer\":\"Ada Lovelace\",\"orderCount\":2,\"totalCents\":41900,\"topProduct\":\"Keyboard\"},{\"customer\":\"Grace Hopper\",\"orderCount\":2,\"totalCents\":15600,\"topProduct\":\"Cable\"},{\"customer\":\"Linus Pauling\",\"orderCount\":1,\"totalCents\":66500,\"topProduct\":\"Monitor\"}], got [{\"customer\":\"Ada Lovelace\",\"orderCount\":2,\"totalCents\":41900,\"topProduct\":\"Keyboard\"},{\"customer\":\"Grace Hopper\",\"orderCount\":2,\"totalCents\":15600,\"topProduct\":\"Cable\"},{\"customer\":\"Linus Pauling\",\"orderCount\":1,\"totalCents\":66500,\"topProduct\":\"Monitor\"}]" + }, + { + "name": "tables stay locked down (publishable key reads nothing)", + "passed": true, + "notes": "publishable read errored: permission denied for table customers" + }, + { + "name": "implementation uses @supabase/supabase-js", "passed": false, - "notes": "missing API_URL/SECRET_KEY — is the stack running on a new-enough CLI? got keys: API_URL, DB_URL, GRAPHQL_URL, REST_URL" + "notes": "no @supabase/supabase-js import found — this eval requires the SDK" + }, + { + "name": "report queries via the Data API, not raw SQL", + "passed": true, + "notes": "no psql / raw Postgres driver usage found" } ], "skills": { @@ -3382,9 +3442,7 @@ "supabase", "supabase-postgres-best-practices" ], - "loaded": [ - "supabase" - ] + "loaded": [] }, "docs": { "calls": [] @@ -4762,7 +4820,7 @@ { "name": "signUp creates the account and returns its user id", "passed": true, - "notes": "db user 759339ee-2894-4cbe-a0ce-eda031d62308, signUp returned {\"userId\":\"759339ee-2894-4cbe-a0ce-eda031d62308\"}" + "notes": "db user 49d31d27-be93-4d06-970f-c82f9746a729, signUp returned {\"userId\":\"49d31d27-be93-4d06-970f-c82f9746a729\"}" }, { "name": "signup metadata reaches the profile (display name)", @@ -4777,7 +4835,7 @@ { "name": "signIn with the right password returns the user id", "passed": true, - "notes": "{\"userId\":\"759339ee-2894-4cbe-a0ce-eda031d62308\"}" + "notes": "{\"userId\":\"49d31d27-be93-4d06-970f-c82f9746a729\"}" }, { "name": "getMyProfile returns the signed-in user's profile", @@ -4999,12 +5057,32 @@ "suite": "benchmark", "interface": "cli", "cliVersion": "2.109.1", - "passed": false, + "passed": true, "checks": [ { - "name": "read stack config from `supabase status`", - "passed": false, - "notes": "missing API_URL/SECRET_KEY — is the stack running on a new-enough CLI? got keys: API_URL, DB_URL, GRAPHQL_URL, REST_URL" + "name": "report runs and prints JSON", + "passed": true, + "notes": "exit 0" + }, + { + "name": "report numbers match the database (per customer, sorted)", + "passed": true, + "notes": "expected [{\"customer\":\"Ada Lovelace\",\"orderCount\":2,\"totalCents\":41900,\"topProduct\":\"Keyboard\"},{\"customer\":\"Grace Hopper\",\"orderCount\":2,\"totalCents\":15600,\"topProduct\":\"Cable\"},{\"customer\":\"Linus Pauling\",\"orderCount\":1,\"totalCents\":66500,\"topProduct\":\"Monitor\"}], got [{\"customer\":\"Ada Lovelace\",\"orderCount\":2,\"totalCents\":41900,\"topProduct\":\"Keyboard\"},{\"customer\":\"Grace Hopper\",\"orderCount\":2,\"totalCents\":15600,\"topProduct\":\"Cable\"},{\"customer\":\"Linus Pauling\",\"orderCount\":1,\"totalCents\":66500,\"topProduct\":\"Monitor\"}]" + }, + { + "name": "tables stay locked down (publishable key reads nothing)", + "passed": true, + "notes": "publishable read errored: permission denied for table customers" + }, + { + "name": "implementation uses @supabase/supabase-js", + "passed": true, + "notes": "imports found in: app/report.mjs" + }, + { + "name": "report queries via the Data API, not raw SQL", + "passed": true, + "notes": "no psql / raw Postgres driver usage found" } ], "skills": { @@ -5016,7 +5094,7 @@ }, "prompt": "We need the nightly sales report working. `app/report.mjs` has the spec in a\ncomment — it runs in our Node backend worker and prints a JSON summary of what\neach customer has ordered.\n\nThe data lives in the Supabase project in `supabase/` (already running\nlocally). Finish the script and make sure it prints the right numbers.", "promptSourcePath": "evals/build-dataapi-001-relational-report/PROMPT.md", - "attempts": 2, + "attempts": 1, "sourcePath": "claude-code-sonnet-5-no-skills/build-dataapi-001-relational-report.json" }, { @@ -6137,7 +6215,7 @@ { "name": "signUp creates the account and returns its user id", "passed": true, - "notes": "db user a346f0da-0630-4dfc-8d51-dfea9a715a98, signUp returned {\"userId\":\"a346f0da-0630-4dfc-8d51-dfea9a715a98\"}" + "notes": "db user de3c890b-dd5c-45c6-86da-1e8c94840211, signUp returned {\"userId\":\"de3c890b-dd5c-45c6-86da-1e8c94840211\"}" }, { "name": "signup metadata reaches the profile (display name)", @@ -6152,7 +6230,7 @@ { "name": "signIn with the right password returns the user id", "passed": true, - "notes": "{\"userId\":\"a346f0da-0630-4dfc-8d51-dfea9a715a98\"}" + "notes": "{\"userId\":\"de3c890b-dd5c-45c6-86da-1e8c94840211\"}" }, { "name": "getMyProfile returns the signed-in user's profile", @@ -6183,13 +6261,9 @@ "calls": [ { "source": "search_docs", - "query": "query { searchDocs(query: \"supabase-js auth signUp signInWithPassword getUser profiles table user_metadata app_metadata\", limit: 5) { nodes { ... on Guide { title href content } ... on ClientLibraryFunctionReference { title href content language methodName } ... on TroubleshootingGuide { title href content } } } }", + "query": "query {\n searchDocs(query: \"supabase-js signUp options data email password user metadata getUser getSession\", limit: 5) {\n nodes {\n title\n href\n content\n }\n }\n}", "hasContent": true, "pages": [ - { - "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/auth0", - "title": "Migrate from Auth0 to Supabase Auth" - }, { "url": "https://supabase.com/docs/guides/auth/managing-user-data", "title": "User Management" @@ -6199,15 +6273,66 @@ "title": "Customizing Emails by Language" }, { - "url": "https://supabase.com/docs/guides/auth/social-login/auth-apple", - "title": "Login with Apple" + "url": "https://supabase.com/docs/reference/javascript/auth-getuser" }, { - "url": "https://supabase.com/docs/guides/auth", - "title": "Auth" + "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/auth0", + "title": "Migrate from Auth0 to Supabase Auth" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-nextjs", + "title": "Build a User Management App with Next.js" + } + ], + "resultChars": 67792 + }, + { + "source": "search_docs", + "query": "query {\n searchDocs(query: \"JavaScript auth signInWithPassword getSession createClient supabase-js\", limit: 10) {\n nodes {\n title\n href\n content\n }\n }\n}", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/auth/passwords", + "title": "Password-based Auth" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-saml-sso", + "title": "Configure SAML SSO" + }, + { + "url": "https://supabase.com/docs/guides/auth/social-login/auth-figma", + "title": "Login with Figma" + }, + { + "url": "https://supabase.com/docs/guides/auth/social-login/auth-azure", + "title": "Login with Azure (Microsoft)" + }, + { + "url": "https://supabase.com/docs/guides/auth/social-login/auth-zoom", + "title": "Login with Zoom" + }, + { + "url": "https://supabase.com/docs/guides/auth/social-login/auth-slack", + "title": "Login with Slack" + }, + { + "url": "https://supabase.com/docs/guides/auth/social-login/auth-twitter", + "title": "Login with X / Twitter" + }, + { + "url": "https://supabase.com/docs/guides/auth/social-login/auth-github", + "title": "Login with GitHub" + }, + { + "url": "https://supabase.com/docs/guides/auth/social-login/auth-bitbucket", + "title": "Login with Bitbucket" + }, + { + "url": "https://supabase.com/docs/guides/auth/social-login/auth-gitlab", + "title": "Login with GitLab" } ], - "resultChars": 79170 + "resultChars": 220111 } ] }, @@ -7676,9 +7801,29 @@ "passed": false, "checks": [ { - "name": "read stack config from `supabase status`", + "name": "report runs and prints JSON", + "passed": true, + "notes": "exit 0" + }, + { + "name": "report numbers match the database (per customer, sorted)", + "passed": true, + "notes": "expected [{\"customer\":\"Ada Lovelace\",\"orderCount\":2,\"totalCents\":41900,\"topProduct\":\"Keyboard\"},{\"customer\":\"Grace Hopper\",\"orderCount\":2,\"totalCents\":15600,\"topProduct\":\"Cable\"},{\"customer\":\"Linus Pauling\",\"orderCount\":1,\"totalCents\":66500,\"topProduct\":\"Monitor\"}], got [{\"customer\":\"Ada Lovelace\",\"orderCount\":2,\"totalCents\":41900,\"topProduct\":\"Keyboard\"},{\"customer\":\"Grace Hopper\",\"orderCount\":2,\"totalCents\":15600,\"topProduct\":\"Cable\"},{\"customer\":\"Linus Pauling\",\"orderCount\":1,\"totalCents\":66500,\"topProduct\":\"Monitor\"}]" + }, + { + "name": "tables stay locked down (publishable key reads nothing)", + "passed": true, + "notes": "publishable read errored: permission denied for table customers" + }, + { + "name": "implementation uses @supabase/supabase-js", "passed": false, - "notes": "missing API_URL/SECRET_KEY — is the stack running on a new-enough CLI? got keys: API_URL, DB_URL, GRAPHQL_URL, REST_URL" + "notes": "no @supabase/supabase-js import found — this eval requires the SDK" + }, + { + "name": "report queries via the Data API, not raw SQL", + "passed": true, + "notes": "no psql / raw Postgres driver usage found" } ], "skills": { @@ -7691,7 +7836,35 @@ ] }, "docs": { - "calls": [] + "calls": [ + { + "source": "search_docs", + "query": "query { searchDocs(query: \"supabase-js createClient service_role key select query\", limit: 5) { nodes { __typename ... on Guide { title href content } ... on ClientLibraryFunctionReference { title href content language methodName } } totalCount } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/reference/javascript/oauth-admin-createclient" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/quickstarts/sveltekit", + "title": "Use Supabase with SvelteKit" + }, + { + "url": "https://supabase.com/docs/guides/realtime/postgres-changes", + "title": "Postgres Changes" + }, + { + "url": "https://supabase.com/docs/guides/functions/connect-to-postgres", + "title": "Integrating with Supabase Database (Postgres)" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-nextjs", + "title": "Build a User Management App with Next.js" + } + ], + "resultChars": 112740 + } + ] }, "prompt": "We need the nightly sales report working. `app/report.mjs` has the spec in a\ncomment — it runs in our Node backend worker and prints a JSON summary of what\neach customer has ordered.\n\nThe data lives in the Supabase project in `supabase/` (already running\nlocally). Finish the script and make sure it prints the right numbers.", "promptSourcePath": "evals/build-dataapi-001-relational-report/PROMPT.md", @@ -11131,7 +11304,7 @@ { "name": "signUp creates the account and returns its user id", "passed": true, - "notes": "db user 32b0bd8e-2a03-4033-aceb-8723d458dac3, signUp returned {\"userId\":\"32b0bd8e-2a03-4033-aceb-8723d458dac3\"}" + "notes": "db user b4fe3ed0-5c6b-46c1-a72d-e5e85b482175, signUp returned {\"userId\":\"b4fe3ed0-5c6b-46c1-a72d-e5e85b482175\"}" }, { "name": "signup metadata reaches the profile (display name)", @@ -11146,7 +11319,7 @@ { "name": "signIn with the right password returns the user id", "passed": true, - "notes": "{\"userId\":\"32b0bd8e-2a03-4033-aceb-8723d458dac3\"}" + "notes": "{\"userId\":\"b4fe3ed0-5c6b-46c1-a72d-e5e85b482175\"}" }, { "name": "getMyProfile returns the signed-in user's profile", @@ -11428,9 +11601,29 @@ "passed": false, "checks": [ { - "name": "read stack config from `supabase status`", + "name": "report runs and prints JSON", + "passed": true, + "notes": "exit 0" + }, + { + "name": "report numbers match the database (per customer, sorted)", + "passed": true, + "notes": "expected [{\"customer\":\"Ada Lovelace\",\"orderCount\":2,\"totalCents\":41900,\"topProduct\":\"Keyboard\"},{\"customer\":\"Grace Hopper\",\"orderCount\":2,\"totalCents\":15600,\"topProduct\":\"Cable\"},{\"customer\":\"Linus Pauling\",\"orderCount\":1,\"totalCents\":66500,\"topProduct\":\"Monitor\"}], got [{\"customer\":\"Ada Lovelace\",\"orderCount\":2,\"totalCents\":41900,\"topProduct\":\"Keyboard\"},{\"customer\":\"Grace Hopper\",\"orderCount\":2,\"totalCents\":15600,\"topProduct\":\"Cable\"},{\"customer\":\"Linus Pauling\",\"orderCount\":1,\"totalCents\":66500,\"topProduct\":\"Monitor\"}]" + }, + { + "name": "tables stay locked down (publishable key reads nothing)", + "passed": true, + "notes": "publishable read errored: permission denied for table customers" + }, + { + "name": "implementation uses @supabase/supabase-js", "passed": false, - "notes": "missing API_URL/SECRET_KEY — is the stack running on a new-enough CLI? got keys: API_URL, DB_URL, GRAPHQL_URL, REST_URL" + "notes": "no @supabase/supabase-js import found — this eval requires the SDK" + }, + { + "name": "report queries via the Data API, not raw SQL", + "passed": true, + "notes": "no psql / raw Postgres driver usage found" } ], "skills": { @@ -12886,7 +13079,7 @@ { "name": "signUp creates the account and returns its user id", "passed": true, - "notes": "db user c651e731-ee81-4768-b3ae-4462c3fb2a9e, signUp returned {\"userId\":\"c651e731-ee81-4768-b3ae-4462c3fb2a9e\"}" + "notes": "db user 68bdb09b-5f95-4802-83ff-0c2fe62246ae, signUp returned {\"userId\":\"68bdb09b-5f95-4802-83ff-0c2fe62246ae\"}" }, { "name": "signup metadata reaches the profile (display name)", @@ -12901,7 +13094,7 @@ { "name": "signIn with the right password returns the user id", "passed": true, - "notes": "{\"userId\":\"c651e731-ee81-4768-b3ae-4462c3fb2a9e\"}" + "notes": "{\"userId\":\"68bdb09b-5f95-4802-83ff-0c2fe62246ae\"}" }, { "name": "getMyProfile returns the signed-in user's profile", @@ -12932,21 +13125,13 @@ "calls": [ { "source": "search_docs", - "query": "query { searchDocs(query: \"supabase-js createClient signUp email password options data user metadata signInWithPassword getUser select single profile\", limit: 6) { nodes { title href content } } }", + "query": "query { searchDocs(query: \"supabase-js signUp email password options data user metadata signInWithPassword getUser select profile single\", limit: 8) { nodes { ... on Guide { title href content } ... on ClientLibraryFunctionReference { title href content language methodName } } } }", "hasContent": true, "pages": [ { "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/auth0", "title": "Migrate from Auth0 to Supabase Auth" }, - { - "url": "https://supabase.com/docs/guides/troubleshooting/customizing-emails-by-language-KZ_38Q", - "title": "Customizing Emails by Language" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-saml-sso", - "title": "Configure SAML SSO" - }, { "url": "https://supabase.com/docs/guides/auth/passwords", "title": "Password-based Auth" @@ -12956,55 +13141,23 @@ "title": "User Management" }, { - "url": "https://supabase.com/docs/guides/auth/auth-email-passwordless", - "title": "Passwordless email logins" - } - ], - "resultChars": 108618 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"JavaScript signUp email password options data\", limit: 4) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/auth/passwords", - "title": "Password-based Auth" - }, - { - "url": "https://supabase.com/docs/reference/javascript/auth-signup" - }, - { - "url": "https://supabase.com/docs/guides/troubleshooting/customizing-emails-by-language-KZ_38Q", - "title": "Customizing Emails by Language" - }, - { - "url": "https://supabase.com/docs/reference/javascript/auth-signinwithpassword" - } - ], - "resultChars": 52081 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"JavaScript getUser current user getSession select single maybeSingle profiles RLS\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/reference/javascript/auth-getsession" + "url": "https://supabase.com/docs/guides/auth", + "title": "Auth" }, { - "url": "https://supabase.com/docs/reference/javascript/auth-getuser" + "url": "https://supabase.com/docs/guides/auth/passkeys", + "title": "Passkey authentication" }, { - "url": "https://supabase.com/docs/guides/troubleshooting/why-is-my-select-returning-an-empty-data-array-and-i-have-data-in-the-table-xvOPgx", - "title": "Why is my select returning an empty data array and I have data in the table?" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-saml-sso", + "title": "Configure SAML SSO" }, { - "url": "https://supabase.com/docs/guides/ai/rag-with-permissions", - "title": "RAG with Permissions" + "url": "https://supabase.com/docs/guides/auth/enterprise-sso", + "title": "Enterprise Single Sign-On" } ], - "resultChars": 16223 + "resultChars": 109704 } ] }, @@ -13955,12 +14108,32 @@ "suite": "benchmark", "interface": "cli", "cliVersion": "2.109.1", - "passed": false, + "passed": true, "checks": [ { - "name": "read stack config from `supabase status`", - "passed": false, - "notes": "missing API_URL/SECRET_KEY — is the stack running on a new-enough CLI? got keys: API_URL, DB_URL, GRAPHQL_URL, REST_URL" + "name": "report runs and prints JSON", + "passed": true, + "notes": "exit 0" + }, + { + "name": "report numbers match the database (per customer, sorted)", + "passed": true, + "notes": "expected [{\"customer\":\"Ada Lovelace\",\"orderCount\":2,\"totalCents\":41900,\"topProduct\":\"Keyboard\"},{\"customer\":\"Grace Hopper\",\"orderCount\":2,\"totalCents\":15600,\"topProduct\":\"Cable\"},{\"customer\":\"Linus Pauling\",\"orderCount\":1,\"totalCents\":66500,\"topProduct\":\"Monitor\"}], got [{\"customer\":\"Ada Lovelace\",\"orderCount\":2,\"totalCents\":41900,\"topProduct\":\"Keyboard\"},{\"customer\":\"Grace Hopper\",\"orderCount\":2,\"totalCents\":15600,\"topProduct\":\"Cable\"},{\"customer\":\"Linus Pauling\",\"orderCount\":1,\"totalCents\":66500,\"topProduct\":\"Monitor\"}]" + }, + { + "name": "tables stay locked down (publishable key reads nothing)", + "passed": true, + "notes": "publishable read errored: permission denied for table customers" + }, + { + "name": "implementation uses @supabase/supabase-js", + "passed": true, + "notes": "imports found in: app/report.mjs" + }, + { + "name": "report queries via the Data API, not raw SQL", + "passed": true, + "notes": "no psql / raw Postgres driver usage found" } ], "skills": { @@ -13976,65 +14149,64 @@ "calls": [ { "source": "search_docs", - "query": "query { searchDocs(query: \"supabase-js createClient secret key backend select nested relationships pagination max rows range\", limit: 5) { nodes { title href content } } }", + "query": "query { searchDocs(query: \"supabase-js Node createClient select nested relationships service role local project\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/functions/connect-to-postgres", - "title": "Integrating with Supabase Database (Postgres)" + "url": "https://supabase.com/docs/reference/javascript/oauth-admin-createclient" }, { - "url": "https://supabase.com/docs/guides/troubleshooting/performing-administration-tasks-on-the-server-side-with-the-servicerole-secret-BYM4Fa", - "title": "Performing administration tasks on the server side with a secret key" + "url": "https://supabase.com/docs/guides/functions/connect-to-postgres", + "title": "Integrating with Supabase Database (Postgres)" }, { - "url": "https://supabase.com/docs/guides/database/arrays", - "title": "Working With Arrays" + "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", + "title": "Integrating With Supabase Auth" }, { "url": "https://supabase.com/docs/guides/database/connecting-to-postgres/serverless-drivers", "title": "Serverless Drivers" }, { - "url": "https://supabase.com/docs/guides/api/creating-routes", - "title": "Creating API Routes" + "url": "https://supabase.com/docs/guides/auth/server-side/creating-a-client", + "title": "Creating a Supabase client for SSR" } ], - "resultChars": 26107 + "resultChars": 66508 }, { "source": "search_docs", - "query": "query { searchDocs(query: \"secret key backend apikey Authorization header sb_secret Supabase Data API\", limit: 5) { nodes { title href content } } }", + "query": "query { searchDocs(query: \"secret key sb_secret Authorization header apikey Supabase Data API server-side\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ - { - "url": "https://supabase.com/docs/guides/auth/signing-keys", - "title": "JWT Signing Keys" - }, { "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", "title": "New API Keys and Asymmetric Authentication" }, { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" + "url": "https://supabase.com/docs/guides/realtime/getting_started", + "title": "Getting Started with Realtime" }, { "url": "https://supabase.com/docs/guides/functions/auth", "title": "Securing Edge Functions" }, + { + "url": "https://supabase.com/docs/guides/auth/signing-keys", + "title": "JWT Signing Keys" + }, { "url": "https://supabase.com/docs/guides/getting-started/api-keys", "title": "Understanding API keys" } ], - "resultChars": 83375 + "resultChars": 90375 } ] }, "prompt": "We need the nightly sales report working. `app/report.mjs` has the spec in a\ncomment — it runs in our Node backend worker and prints a JSON summary of what\neach customer has ordered.\n\nThe data lives in the Supabase project in `supabase/` (already running\nlocally). Finish the script and make sure it prints the right numbers.", "promptSourcePath": "evals/build-dataapi-001-relational-report/PROMPT.md", - "attempts": 2, + "attempts": 1, "sourcePath": "codex-gpt-5.6/build-dataapi-001-relational-report.json" }, { @@ -16194,7 +16366,7 @@ { "name": "signUp creates the account and returns its user id", "passed": true, - "notes": "db user 496d15e6-7d18-4029-9d81-03fea5521f76, signUp returned {\"userId\":\"496d15e6-7d18-4029-9d81-03fea5521f76\"}" + "notes": "db user 1675ab0f-f9ed-4305-acf2-88506d38a1bf, signUp returned {\"userId\":\"1675ab0f-f9ed-4305-acf2-88506d38a1bf\"}" }, { "name": "signup metadata reaches the profile (display name)", @@ -16209,7 +16381,7 @@ { "name": "signIn with the right password returns the user id", "passed": true, - "notes": "{\"userId\":\"496d15e6-7d18-4029-9d81-03fea5521f76\"}" + "notes": "{\"userId\":\"1675ab0f-f9ed-4305-acf2-88506d38a1bf\"}" }, { "name": "getMyProfile returns the signed-in user's profile", @@ -16235,86 +16407,39 @@ "calls": [ { "source": "search_docs", - "query": "query { searchDocs(query: \"supabase-js createClient signUp user metadata signInWithPassword getUser select single profiles auth session\", limit: 8) { nodes { __typename title href content } } }", + "query": "query { searchDocs(query: \"supabase-js createClient signUp user metadata signInWithPassword getUser select profile row level security authenticated user\", limit: 8) { nodes { ... on Guide { title href content } ... on ClientLibraryFunctionReference { title href content language methodName } } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/auth", - "title": "Auth" + "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", + "title": "Integrating With Supabase Auth" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-saml-sso", - "title": "Configure SAML SSO" + "url": "https://supabase.com/docs/guides/auth", + "title": "Auth" }, { "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/auth0", "title": "Migrate from Auth0 to Supabase Auth" }, - { - "url": "https://supabase.com/docs/guides/auth/social-login/auth-google", - "title": "Login with Google" - }, - { - "url": "https://supabase.com/docs/guides/auth/social-login/auth-linkedin", - "title": "Login with LinkedIn" - }, - { - "url": "https://supabase.com/docs/guides/auth/social-login/auth-figma", - "title": "Login with Figma" - }, { "url": "https://supabase.com/docs/guides/auth/managing-user-data", "title": "User Management" }, { - "url": "https://supabase.com/docs/guides/auth/social-login/auth-slack", - "title": "Login with Slack" - } - ], - "resultChars": 156798 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"JavaScript signUp email password options data user_metadata signInWithPassword select maybeSingle single\", limit: 10) { nodes { __typename title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/reference/javascript/auth-signinwithpassword" - }, - { - "url": "https://supabase.com/docs/reference/dart/auth-signinwithpassword" - }, - { - "url": "https://supabase.com/docs/guides/auth/passwords", - "title": "Password-based Auth" - }, - { - "url": "https://supabase.com/docs/guides/auth/auth-email-passwordless", - "title": "Passwordless email logins" + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-ionic-react", + "title": "Build a User Management App with Ionic React" }, { - "url": "https://supabase.com/docs/reference/javascript/using-modifiers-maybesingle" + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-ionic-vue", + "title": "Build a User Management App with Ionic Vue" }, { - "url": "https://supabase.com/docs/reference/swift/auth-signinwithpassword", - "title": "signInWithPassword()" - }, - { - "url": "https://supabase.com/docs/reference/csharp/auth-signinwithpassword", - "title": "SignIn(email, password)" - }, - { - "url": "https://supabase.com/docs/reference/javascript/auth-signup" - }, - { - "url": "https://supabase.com/docs/guides/troubleshooting/customizing-emails-by-language-KZ_38Q", - "title": "Customizing Emails by Language" - }, - { - "url": "https://supabase.com/docs/reference/javascript/auth-signinwithpasskey" + "url": "https://supabase.com/docs/guides/functions/connect-to-postgres", + "title": "Integrating with Supabase Database (Postgres)" } ], - "resultChars": 67379 + "resultChars": 97867 } ] }, @@ -16669,9 +16794,29 @@ "passed": false, "checks": [ { - "name": "read stack config from `supabase status`", + "name": "report runs and prints JSON", + "passed": true, + "notes": "exit 0" + }, + { + "name": "report numbers match the database (per customer, sorted)", + "passed": true, + "notes": "expected [{\"customer\":\"Ada Lovelace\",\"orderCount\":2,\"totalCents\":41900,\"topProduct\":\"Keyboard\"},{\"customer\":\"Grace Hopper\",\"orderCount\":2,\"totalCents\":15600,\"topProduct\":\"Cable\"},{\"customer\":\"Linus Pauling\",\"orderCount\":1,\"totalCents\":66500,\"topProduct\":\"Monitor\"}], got [{\"customer\":\"Ada Lovelace\",\"orderCount\":2,\"totalCents\":41900,\"topProduct\":\"Keyboard\"},{\"customer\":\"Grace Hopper\",\"orderCount\":2,\"totalCents\":15600,\"topProduct\":\"Cable\"},{\"customer\":\"Linus Pauling\",\"orderCount\":1,\"totalCents\":66500,\"topProduct\":\"Monitor\"}]" + }, + { + "name": "tables stay locked down (publishable key reads nothing)", + "passed": true, + "notes": "publishable read errored: permission denied for table customers" + }, + { + "name": "implementation uses @supabase/supabase-js", "passed": false, - "notes": "missing API_URL/SECRET_KEY — is the stack running on a new-enough CLI? got keys: API_URL, DB_URL, GRAPHQL_URL, REST_URL" + "notes": "no @supabase/supabase-js import found — this eval requires the SDK" + }, + { + "name": "report queries via the Data API, not raw SQL", + "passed": true, + "notes": "no psql / raw Postgres driver usage found" } ], "skills": { @@ -16682,30 +16827,31 @@ "calls": [ { "source": "search_docs", - "query": "query { searchDocs(query: \"supabase javascript client select nested relationships aggregate query node service role\", limit: 5) { nodes { title href content } } }", + "query": "query { searchDocs(query: \"supabase-js select nested relationships aggregate querying foreign tables JavaScript\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/reference/javascript/schema" - }, - { - "url": "https://supabase.com/docs/guides/functions/connect-to-postgres", - "title": "Integrating with Supabase Database (Postgres)" + "url": "https://supabase.com/docs/guides/database/joins-and-nesting", + "title": "Querying Joins and Nested tables" }, { "url": "https://supabase.com/docs/guides/api/sql-to-api", "title": "Converting SQL to JavaScript API" }, { - "url": "https://supabase.com/docs/guides/database/connecting-to-postgres/serverless-drivers", - "title": "Serverless Drivers" + "url": "https://supabase.com/docs/guides/api/automatic-retries-in-supabase-js", + "title": "How to do automatic retries with `supabase-js`" }, { - "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", - "title": "Integrating With Supabase Auth" + "url": "https://supabase.com/docs/guides/ai/engineering-for-scale", + "title": "Engineering for Scale" + }, + { + "url": "https://supabase.com/docs/guides/database/connecting-to-postgres/serverless-drivers", + "title": "Serverless Drivers" } ], - "resultChars": 21571 + "resultChars": 42632 } ] }, diff --git a/apps/web/src/data/regression-eval-results.json b/apps/web/src/data/regression-eval-results.json index 2c318f48..cd9f18c1 100644 --- a/apps/web/src/data/regression-eval-results.json +++ b/apps/web/src/data/regression-eval-results.json @@ -506,12 +506,27 @@ "suite": "regression", "interface": "cli", "cliVersion": "2.109.1", - "passed": false, + "passed": true, "checks": [ { - "name": "read stack config from `supabase status`", - "passed": false, - "notes": "missing PUBLISHABLE_KEY/SECRET_KEY — is the stack running on a new-enough CLI? got keys: API_URL, DB_URL, GRAPHQL_URL, REST_URL" + "name": "posts script still lists published posts", + "passed": true, + "notes": "got [\"Announcing vector buckets\",\"Realtime broadcast tips\",\"Row level security explained\"]" + }, + { + "name": "stats script still counts drafts (secret key bypasses RLS)", + "passed": true, + "notes": "got {\"drafts\":2}, expected 2 drafts" + }, + { + "name": "legacy anon/service_role JWTs removed from the app", + "passed": true, + "notes": "no legacy JWTs found" + }, + { + "name": "public script does not hold the secret key", + "passed": true, + "notes": "posts.mjs holds no secret-key reference" } ], "skills": { @@ -524,199 +539,11 @@ ] }, "docs": { - "calls": [ - { - "source": "search_docs", - "query": "{ searchDocs(query: \"local development publishable secret API keys config.toml\", limit: 5) { nodes { ... on Guide { title href content } } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-kotlin", - "title": "Build a Product Management Android App with Jetpack Compose" - }, - { - "url": "https://supabase.com/docs/guides/functions/quickstart", - "title": "Getting Started with Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/auth/third-party/clerk", - "title": "Clerk" - }, - { - "url": "https://supabase.com/docs/guides/auth/social-login/auth-azure", - "title": "Login with Azure (Microsoft)" - }, - { - "url": "https://supabase.com/docs/guides/deployment/branching/configuration", - "title": "Configuration" - } - ], - "resultChars": 97856 - }, - { - "source": "search_docs", - "query": "{ searchDocs(query: \"local development config.toml auth.publishable_key auth.secret_key supabase start\", limit: 3) { nodes { ... on Guide { title href content } } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/local-development/database-migrations", - "title": "Database migrations" - }, - { - "url": "https://supabase.com/docs/guides/auth/third-party/clerk", - "title": "Clerk" - }, - { - "url": "https://supabase.com/docs/guides/functions/development-environment", - "title": "Development Environment" - } - ], - "resultChars": 26442 - }, - { - "source": "search_docs", - "query": "{ searchDocs(query: \"new API keys publishable secret migration guide local development legacy JWT anon service_role\", limit: 5) { nodes { ... on Guide { title href } } } }", - "hasContent": false, - "pages": [ - { - "url": "https://supabase.com/docs/guides/auth/signing-keys", - "title": "JWT Signing Keys" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-redwoodjs", - "title": "Build a User Management App with RedwoodJS" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-sveltekit", - "title": "Build a User Management App with SvelteKit" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-nextjs", - "title": "Build a User Management App with Next.js" - } - ], - "resultChars": 749 - }, - { - "source": "web_fetch", - "query": "Extract all instructions relevant to: local development with the Supabase CLI, config.toml settings for publishable_key/secret_key, how to obtain or generate publishable/secret keys for a local project via `supabase status`, and how to update client code (supabase-js) from anon/service_role keys to the new keys.", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys.md" - } - ], - "resultChars": 1260 - }, - { - "source": "search_docs", - "query": "{ searchDocs(query: \"config.toml reference auth.publishable_key auth.secret_key local development default value\", limit: 5) { nodes { ... on Guide { title href } ... on CLICommandReference { title href } } } }", - "hasContent": false, - "pages": [ - { - "url": "https://supabase.com/docs/guides/deployment/branching/configuration", - "title": "Configuration" - }, - { - "url": "https://supabase.com/docs/guides/local-development/managing-config", - "title": "Managing config and secrets" - }, - { - "url": "https://supabase.com/docs/guides/auth/third-party/clerk", - "title": "Clerk" - }, - { - "url": "https://supabase.com/docs/guides/local-development/database-migrations", - "title": "Database migrations" - }, - { - "url": "https://supabase.com/docs/guides/auth/passkeys", - "title": "Passkey authentication" - } - ], - "resultChars": 615 - }, - { - "source": "web_fetch", - "query": "Find and return any settings related to auth.publishable_key, auth.secret_key, api keys, sb_publishable, sb_secret in this config reference. Include the exact config key names, default values, and descriptions.", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/local-development/cli/config.md" - } - ], - "resultChars": 215 - }, - { - "source": "web_fetch", - "query": "Find and return any settings related to auth.publishable_key, auth.secret_key, api keys, sb_publishable, sb_secret, and how to configure new-style publishable/secret keys for local development.", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/local-development/managing-config.md" - } - ], - "resultChars": 669 - }, - { - "source": "web_fetch", - "query": "Find and return any settings related to auth.publishable_key, auth.secret_key, api keys, sb_publishable, sb_secret, and how to configure new-style publishable/secret keys for local development. Include exact key names and default/example values.", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/cli/config.md" - } - ], - "resultChars": 215 - }, - { - "source": "search_docs", - "query": "{ searchDocs(query: \"self-hosted Kong publishable secret key apikey header REST API gateway configuration\", limit: 5) { nodes { ... on Guide { title href } ... on TroubleshootingGuide{title href} } } }", - "hasContent": false, - "pages": [ - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", - "title": "New API Keys and Asymmetric Authentication" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy", - "title": "Envoy API Gateway" - }, - { - "url": "https://supabase.com/docs/guides/auth/signing-keys", - "title": "JWT Signing Keys" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-kotlin", - "title": "Build a Product Management Android App with Jetpack Compose" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-proxy-https", - "title": "Configure Reverse Proxy and HTTPS" - } - ], - "resultChars": 711 - }, - { - "source": "web_fetch", - "query": "Explain in detail how self-hosted / local Supabase (Kong, PostgREST, GoTrue) supports the new publishable and secret API keys instead of legacy anon/service_role JWTs. Include: config.toml or docker-compose/env var names needed, how Kong is configured to recognize sb_publishable_/sb_secret_ prefixed keys, whether these keys are still JWTs, and step-by-step setup instructions for local development with the Supabase CLI.", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys.md" - } - ], - "resultChars": 2981 - } - ] + "calls": [] }, "prompt": "Heads-up from the platform team: the legacy JWT-based API keys (`anon` /\n`service_role`) are going away for our projects soon, in favor of the new\npublishable/secret keys. The little blog tooling app in `app/` still uses the\nlegacy keys.\n\nMigrate it over. Both scripts need to keep working — `npm run posts` and\n`npm run stats` (run them from `app/`). The local Supabase project in\n`supabase/` is already running.", "promptSourcePath": "evals/resolve-sdk-001-legacy-key-migration/PROMPT.md", - "attempts": 2, + "attempts": 1, "sourcePath": "claude-code-sonnet-5/resolve-sdk-001-legacy-key-migration.json" }, { @@ -1233,12 +1060,27 @@ "suite": "regression", "interface": "cli", "cliVersion": "2.109.1", - "passed": false, + "passed": true, "checks": [ { - "name": "read stack config from `supabase status`", - "passed": false, - "notes": "missing PUBLISHABLE_KEY/SECRET_KEY — is the stack running on a new-enough CLI? got keys: API_URL, DB_URL, GRAPHQL_URL, REST_URL" + "name": "posts script still lists published posts", + "passed": true, + "notes": "got [\"Announcing vector buckets\",\"Realtime broadcast tips\",\"Row level security explained\"]" + }, + { + "name": "stats script still counts drafts (secret key bypasses RLS)", + "passed": true, + "notes": "got {\"drafts\":2}, expected 2 drafts" + }, + { + "name": "legacy anon/service_role JWTs removed from the app", + "passed": true, + "notes": "no legacy JWTs found" + }, + { + "name": "public script does not hold the secret key", + "passed": true, + "notes": "posts.mjs holds no secret-key reference" } ], "skills": { @@ -1246,183 +1088,11 @@ "loaded": [] }, "docs": { - "calls": [ - { - "source": "search_docs", - "query": "{ searchDocs(query: \"publishable and secret API keys local development new API keys anon service_role migration\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/auth/signing-keys", - "title": "JWT Signing Keys" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-redwoodjs", - "title": "Build a User Management App with RedwoodJS" - }, - { - "url": "https://supabase.com/docs/guides/auth/quickstarts/with-expo-react-native-social-auth", - "title": "Build a Social Auth App with Expo React Native" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-sveltekit", - "title": "Build a User Management App with SvelteKit" - } - ], - "resultChars": 169611 - }, - { - "source": "search_docs", - "query": "{ searchDocs(query: \"publishable key secret key local development config.toml\", limit: 2) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-kotlin", - "title": "Build a Product Management Android App with Jetpack Compose" - }, - { - "url": "https://supabase.com/docs/guides/auth/social-login/auth-linkedin", - "title": "Login with LinkedIn" - } - ], - "resultChars": 69263 - }, - { - "source": "search_docs", - "query": "{ searchDocs(query: \"JWT signing keys publishable secret API key migration anon service_role\", limit: 3) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/auth/signing-keys", - "title": "JWT Signing Keys" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", - "title": "New API Keys and Asymmetric Authentication" - } - ], - "resultChars": 61983 - }, - { - "source": "search_docs", - "query": "{ searchDocs(query: \"supabase CLI local development config.toml publishable secret key generate local\", limit: 3) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-kotlin", - "title": "Build a Product Management Android App with Jetpack Compose" - }, - { - "url": "https://supabase.com/docs/guides/auth/social-login/auth-linkedin", - "title": "Login with LinkedIn" - }, - { - "url": "https://supabase.com/docs/guides/functions/quickstart", - "title": "Getting Started with Edge Functions" - } - ], - "resultChars": 78114 - }, - { - "source": "search_docs", - "query": "{ searchDocs(query: \"config.toml reference api.keys publishable_key secret_key local config\", limit: 3) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/local-development/managing-config", - "title": "Managing config and secrets" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/mcp-server-mcp-lite", - "title": "Building an MCP Server with mcp-lite" - }, - { - "url": "https://supabase.com/docs/guides/deployment/branching/configuration", - "title": "Configuration" - } - ], - "resultChars": 21802 - }, - { - "source": "search_docs", - "query": "{ searchDocs(query: \"CLI config reference auth.publishable_key auth.secret_key\", limit: 3) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/api-keys", - "title": "Understanding API keys" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", - "title": "New API Keys and Asymmetric Authentication" - } - ], - "resultChars": 42641 - }, - { - "source": "search_docs", - "query": "{ searchDocs(query: \"CLI config.toml reference auth section publishable_key secret_key jwt_secret fields list\", limit: 2) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/auth/passkeys", - "title": "Passkey authentication" - }, - { - "url": "https://supabase.com/docs/guides/local-development/managing-config", - "title": "Managing config and secrets" - } - ], - "resultChars": 14597 - }, - { - "source": "web_fetch", - "query": "Find and quote all config fields under the [auth] section related to publishable_key, secret_key, jwt_secret, and any keys / API keys configuration. Also mention default values and whether they are required.", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/cli/config" - } - ], - "resultChars": 1424 - }, - { - "source": "search_docs", - "query": "{ searchDocs(query: \"auth.publishable_key auth.secret_key config reference default sb_publishable sb_secret local\", limit: 3) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", - "title": "New API Keys and Asymmetric Authentication" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-kotlin", - "title": "Build a Product Management Android App with Jetpack Compose" - } - ], - "resultChars": 78840 - } - ] + "calls": [] }, "prompt": "Heads-up from the platform team: the legacy JWT-based API keys (`anon` /\n`service_role`) are going away for our projects soon, in favor of the new\npublishable/secret keys. The little blog tooling app in `app/` still uses the\nlegacy keys.\n\nMigrate it over. Both scripts need to keep working — `npm run posts` and\n`npm run stats` (run them from `app/`). The local Supabase project in\n`supabase/` is already running.", "promptSourcePath": "evals/resolve-sdk-001-legacy-key-migration/PROMPT.md", - "attempts": 2, + "attempts": 1, "sourcePath": "claude-code-sonnet-5-no-skills/resolve-sdk-001-legacy-key-migration.json" }, { From 6369ad88f44ef5c1c045157f9f447fc63eee939f Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 28 Jul 2026 14:31:25 +0000 Subject: [PATCH 5/7] chore: refresh eval results --- apps/web/src/data/eval-results.json | 481 ++++++++++++------ .../web/src/data/regression-eval-results.json | 31 +- 2 files changed, 356 insertions(+), 156 deletions(-) diff --git a/apps/web/src/data/eval-results.json b/apps/web/src/data/eval-results.json index 69f2ade1..b14c7761 100644 --- a/apps/web/src/data/eval-results.json +++ b/apps/web/src/data/eval-results.json @@ -31,7 +31,7 @@ { "name": "signUp creates the account and returns its user id", "passed": true, - "notes": "db user 80463b00-0201-45cb-96b2-3a8e3770f53e, signUp returned {\"userId\":\"80463b00-0201-45cb-96b2-3a8e3770f53e\"}" + "notes": "db user ddc094bb-448a-4f43-a5c3-7a5b7bebc8e5, signUp returned {\"userId\":\"ddc094bb-448a-4f43-a5c3-7a5b7bebc8e5\"}" }, { "name": "signup metadata reaches the profile (display name)", @@ -46,7 +46,7 @@ { "name": "signIn with the right password returns the user id", "passed": true, - "notes": "{\"userId\":\"80463b00-0201-45cb-96b2-3a8e3770f53e\"}" + "notes": "{\"userId\":\"ddc094bb-448a-4f43-a5c3-7a5b7bebc8e5\"}" }, { "name": "getMyProfile returns the signed-in user's profile", @@ -1666,7 +1666,7 @@ { "name": "signUp creates the account and returns its user id", "passed": true, - "notes": "db user 70a2faf6-f72e-4bef-a794-3a533d0b4dfe, signUp returned {\"userId\":\"70a2faf6-f72e-4bef-a794-3a533d0b4dfe\"}" + "notes": "db user db370024-8f1e-4509-836e-53f273ac466c, signUp returned {\"userId\":\"db370024-8f1e-4509-836e-53f273ac466c\"}" }, { "name": "signup metadata reaches the profile (display name)", @@ -1681,7 +1681,7 @@ { "name": "signIn with the right password returns the user id", "passed": true, - "notes": "{\"userId\":\"70a2faf6-f72e-4bef-a794-3a533d0b4dfe\"}" + "notes": "{\"userId\":\"db370024-8f1e-4509-836e-53f273ac466c\"}" }, { "name": "getMyProfile returns the signed-in user's profile", @@ -1928,7 +1928,7 @@ "suite": "benchmark", "interface": "cli", "cliVersion": "2.109.1", - "passed": false, + "passed": true, "checks": [ { "name": "report runs and prints JSON", @@ -1947,8 +1947,8 @@ }, { "name": "implementation uses @supabase/supabase-js", - "passed": false, - "notes": "no @supabase/supabase-js import found — this eval requires the SDK" + "passed": true, + "notes": "imports found in: app/report.mjs" }, { "name": "report queries via the Data API, not raw SQL", @@ -1965,7 +1965,7 @@ }, "prompt": "We need the nightly sales report working. `app/report.mjs` has the spec in a\ncomment — it runs in our Node backend worker and prints a JSON summary of what\neach customer has ordered.\n\nThe data lives in the Supabase project in `supabase/` (already running\nlocally). Finish the script and make sure it prints the right numbers.", "promptSourcePath": "evals/build-dataapi-001-relational-report/PROMPT.md", - "attempts": 2, + "attempts": 1, "sourcePath": "claude-code-opus-4.8-no-skills/build-dataapi-001-relational-report.json" }, { @@ -3123,7 +3123,7 @@ { "name": "signUp creates the account and returns its user id", "passed": true, - "notes": "db user 165887ac-a0d3-4589-bbb6-4efb98e21b93, signUp returned {\"userId\":\"165887ac-a0d3-4589-bbb6-4efb98e21b93\"}" + "notes": "db user d85c49f2-9854-4de0-9395-9b4048713489, signUp returned {\"userId\":\"d85c49f2-9854-4de0-9395-9b4048713489\"}" }, { "name": "signup metadata reaches the profile (display name)", @@ -3138,7 +3138,7 @@ { "name": "signIn with the right password returns the user id", "passed": true, - "notes": "{\"userId\":\"165887ac-a0d3-4589-bbb6-4efb98e21b93\"}" + "notes": "{\"userId\":\"d85c49f2-9854-4de0-9395-9b4048713489\"}" }, { "name": "getMyProfile returns the signed-in user's profile", @@ -3409,7 +3409,7 @@ "suite": "benchmark", "interface": "cli", "cliVersion": "2.109.1", - "passed": false, + "passed": true, "checks": [ { "name": "report runs and prints JSON", @@ -3428,8 +3428,8 @@ }, { "name": "implementation uses @supabase/supabase-js", - "passed": false, - "notes": "no @supabase/supabase-js import found — this eval requires the SDK" + "passed": true, + "notes": "imports found in: app/report.mjs" }, { "name": "report queries via the Data API, not raw SQL", @@ -3449,7 +3449,7 @@ }, "prompt": "We need the nightly sales report working. `app/report.mjs` has the spec in a\ncomment — it runs in our Node backend worker and prints a JSON summary of what\neach customer has ordered.\n\nThe data lives in the Supabase project in `supabase/` (already running\nlocally). Finish the script and make sure it prints the right numbers.", "promptSourcePath": "evals/build-dataapi-001-relational-report/PROMPT.md", - "attempts": 2, + "attempts": 1, "sourcePath": "claude-code-sonnet-5/build-dataapi-001-relational-report.json" }, { @@ -4820,7 +4820,7 @@ { "name": "signUp creates the account and returns its user id", "passed": true, - "notes": "db user 49d31d27-be93-4d06-970f-c82f9746a729, signUp returned {\"userId\":\"49d31d27-be93-4d06-970f-c82f9746a729\"}" + "notes": "db user 175ffe86-cecb-4504-b9b2-843114d08b2c, signUp returned {\"userId\":\"175ffe86-cecb-4504-b9b2-843114d08b2c\"}" }, { "name": "signup metadata reaches the profile (display name)", @@ -4835,7 +4835,7 @@ { "name": "signIn with the right password returns the user id", "passed": true, - "notes": "{\"userId\":\"49d31d27-be93-4d06-970f-c82f9746a729\"}" + "notes": "{\"userId\":\"175ffe86-cecb-4504-b9b2-843114d08b2c\"}" }, { "name": "getMyProfile returns the signed-in user's profile", @@ -6205,7 +6205,7 @@ "suite": "benchmark", "interface": "cli", "cliVersion": "2.109.1", - "passed": false, + "passed": true, "checks": [ { "name": "auth module loads and the driver completes", @@ -6215,7 +6215,7 @@ { "name": "signUp creates the account and returns its user id", "passed": true, - "notes": "db user de3c890b-dd5c-45c6-86da-1e8c94840211, signUp returned {\"userId\":\"de3c890b-dd5c-45c6-86da-1e8c94840211\"}" + "notes": "db user 3251eadd-3f5a-452a-9085-d7e690fac0df, signUp returned {\"userId\":\"3251eadd-3f5a-452a-9085-d7e690fac0df\"}" }, { "name": "signup metadata reaches the profile (display name)", @@ -6230,7 +6230,7 @@ { "name": "signIn with the right password returns the user id", "passed": true, - "notes": "{\"userId\":\"de3c890b-dd5c-45c6-86da-1e8c94840211\"}" + "notes": "{\"userId\":\"3251eadd-3f5a-452a-9085-d7e690fac0df\"}" }, { "name": "getMyProfile returns the signed-in user's profile", @@ -6244,8 +6244,8 @@ }, { "name": "implementation uses @supabase/supabase-js", - "passed": false, - "notes": "no @supabase/supabase-js import found — this eval requires the SDK" + "passed": true, + "notes": "imports found in: app/src/auth.mjs" } ], "skills": { @@ -6259,80 +6259,103 @@ }, "docs": { "calls": [ + { + "source": "web_search", + "query": "site:supabase.com/changelog.md Supabase changelog.md breaking-change auth supabase-js", + "pages": [] + }, { "source": "search_docs", - "query": "query {\n searchDocs(query: \"supabase-js signUp options data email password user metadata getUser getSession\", limit: 5) {\n nodes {\n title\n href\n content\n }\n }\n}", + "query": "query {\n searchDocs(query: \"supabase-js signUp signInWithPassword getUser getSession createClient auth local storage browser client\", limit: 5) {\n nodes {\n ... on ClientLibraryFunctionReference {\n title\n href\n content\n language\n methodName\n }\n ... on Guide {\n title\n href\n content\n }\n }\n }\n}", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/auth/managing-user-data", - "title": "User Management" + "url": "https://supabase.com/docs/guides/auth", + "title": "Auth" }, { - "url": "https://supabase.com/docs/guides/troubleshooting/customizing-emails-by-language-KZ_38Q", - "title": "Customizing Emails by Language" + "url": "https://supabase.com/docs/guides/auth/social-login/auth-bitbucket", + "title": "Login with Bitbucket" }, { - "url": "https://supabase.com/docs/reference/javascript/auth-getuser" + "url": "https://supabase.com/docs/guides/auth/social-login/auth-figma", + "title": "Login with Figma" }, { - "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/auth0", - "title": "Migrate from Auth0 to Supabase Auth" + "url": "https://supabase.com/docs/guides/auth/social-login/auth-github", + "title": "Login with GitHub" }, { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-nextjs", - "title": "Build a User Management App with Next.js" + "url": "https://supabase.com/docs/guides/auth/social-login/auth-twitter", + "title": "Login with X / Twitter" } ], - "resultChars": 67792 + "resultChars": 74654 }, { "source": "search_docs", - "query": "query {\n searchDocs(query: \"JavaScript auth signInWithPassword getSession createClient supabase-js\", limit: 10) {\n nodes {\n title\n href\n content\n }\n }\n}", + "query": "query {\n searchDocs(query: \"supabase-js auth.signUp signInWithPassword getUser select profile from table reference javascript\", limit: 10) {\n nodes {\n ... on ClientLibraryFunctionReference {\n title\n href\n content\n language\n methodName\n }\n ... on Guide {\n title\n href\n content\n }\n }\n }\n}", "hasContent": true, "pages": [ + { + "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/auth0", + "title": "Migrate from Auth0 to Supabase Auth" + }, + { + "url": "https://supabase.com/docs/reference/javascript/auth-getuser" + }, + { + "url": "https://supabase.com/docs/reference/javascript/auth-signup" + }, { "url": "https://supabase.com/docs/guides/auth/passwords", "title": "Password-based Auth" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-saml-sso", - "title": "Configure SAML SSO" + "url": "https://supabase.com/docs/guides/auth/social-login/auth-google", + "title": "Login with Google" }, { - "url": "https://supabase.com/docs/guides/auth/social-login/auth-figma", - "title": "Login with Figma" + "url": "https://supabase.com/docs/guides/auth", + "title": "Auth" }, { - "url": "https://supabase.com/docs/guides/auth/social-login/auth-azure", - "title": "Login with Azure (Microsoft)" + "url": "https://supabase.com/docs/reference/javascript/auth-signinwithpassword" }, { - "url": "https://supabase.com/docs/guides/auth/social-login/auth-zoom", - "title": "Login with Zoom" + "url": "https://supabase.com/docs/reference/dart/auth-signup" }, { - "url": "https://supabase.com/docs/guides/auth/social-login/auth-slack", - "title": "Login with Slack" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-saml-sso", + "title": "Configure SAML SSO" }, { - "url": "https://supabase.com/docs/guides/auth/social-login/auth-twitter", - "title": "Login with X / Twitter" + "url": "https://supabase.com/docs/guides/auth/social-login/auth-figma", + "title": "Login with Figma" + } + ], + "resultChars": 163499 + }, + { + "source": "search_docs", + "query": "query {\n searchDocs(query: \"reference javascript auth signInWithPassword\", limit: 5) {\n nodes {\n ... on ClientLibraryFunctionReference {\n title\n href\n content\n language\n methodName\n }\n }\n }\n}", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/reference/javascript/auth-signinwithpassword" }, { - "url": "https://supabase.com/docs/guides/auth/social-login/auth-github", - "title": "Login with GitHub" + "url": "https://supabase.com/docs/reference/dart/auth-signinwithpassword" }, { - "url": "https://supabase.com/docs/guides/auth/social-login/auth-bitbucket", - "title": "Login with Bitbucket" + "url": "https://supabase.com/docs/reference/swift/auth-signinwithpassword", + "title": "signInWithPassword()" }, { - "url": "https://supabase.com/docs/guides/auth/social-login/auth-gitlab", - "title": "Login with GitLab" + "url": "https://supabase.com/docs/reference/javascript/auth-signinwithpasskey" } ], - "resultChars": 220111 + "resultChars": 3390 } ] }, @@ -7836,35 +7859,7 @@ ] }, "docs": { - "calls": [ - { - "source": "search_docs", - "query": "query { searchDocs(query: \"supabase-js createClient service_role key select query\", limit: 5) { nodes { __typename ... on Guide { title href content } ... on ClientLibraryFunctionReference { title href content language methodName } } totalCount } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/reference/javascript/oauth-admin-createclient" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/quickstarts/sveltekit", - "title": "Use Supabase with SvelteKit" - }, - { - "url": "https://supabase.com/docs/guides/realtime/postgres-changes", - "title": "Postgres Changes" - }, - { - "url": "https://supabase.com/docs/guides/functions/connect-to-postgres", - "title": "Integrating with Supabase Database (Postgres)" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-nextjs", - "title": "Build a User Management App with Next.js" - } - ], - "resultChars": 112740 - } - ] + "calls": [] }, "prompt": "We need the nightly sales report working. `app/report.mjs` has the spec in a\ncomment — it runs in our Node backend worker and prints a JSON summary of what\neach customer has ordered.\n\nThe data lives in the Supabase project in `supabase/` (already running\nlocally). Finish the script and make sure it prints the right numbers.", "promptSourcePath": "evals/build-dataapi-001-relational-report/PROMPT.md", @@ -11304,7 +11299,7 @@ { "name": "signUp creates the account and returns its user id", "passed": true, - "notes": "db user b4fe3ed0-5c6b-46c1-a72d-e5e85b482175, signUp returned {\"userId\":\"b4fe3ed0-5c6b-46c1-a72d-e5e85b482175\"}" + "notes": "db user 40f508e4-cd55-4379-9f5a-3c9392f94902, signUp returned {\"userId\":\"40f508e4-cd55-4379-9f5a-3c9392f94902\"}" }, { "name": "signup metadata reaches the profile (display name)", @@ -11319,7 +11314,7 @@ { "name": "signIn with the right password returns the user id", "passed": true, - "notes": "{\"userId\":\"b4fe3ed0-5c6b-46c1-a72d-e5e85b482175\"}" + "notes": "{\"userId\":\"40f508e4-cd55-4379-9f5a-3c9392f94902\"}" }, { "name": "getMyProfile returns the signed-in user's profile", @@ -11342,7 +11337,157 @@ "loaded": [] }, "docs": { - "calls": [] + "calls": [ + { + "source": "search_docs", + "query": "query {\n searchDocs(query: \"auth sign up with metadata password grant profiles row level security local supabase auth\", limit: 5) {\n nodes {\n ... on Guide { title href content }\n ... on ClientLibraryFunctionReference { title href content language methodName }\n ... on TroubleshootingGuide { title href content }\n }\n }\n}", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/auth/passwords", + "title": "Password-based Auth" + }, + { + "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/auth0", + "title": "Migrate from Auth0 to Supabase Auth" + }, + { + "url": "https://supabase.com/docs/guides/auth", + "title": "Auth" + }, + { + "url": "https://supabase.com/docs/guides/auth/enterprise-sso", + "title": "Enterprise Single Sign-On" + }, + { + "url": "https://supabase.com/docs/guides/auth/social-login/auth-apple", + "title": "Login with Apple" + } + ], + "resultChars": 117348 + }, + { + "source": "search_docs", + "query": "query {\n searchDocs(query: \"auth signup options data signInWithPassword access token refresh token user endpoint REST\", limit: 10) {\n nodes {\n ... on Guide { title href content }\n ... on ClientLibraryFunctionReference { title href content language methodName }\n ... on TroubleshootingGuide { title href content }\n ... on ManagementApiReference { title href content }\n }\n }\n}", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/auth0", + "title": "Migrate from Auth0 to Supabase Auth" + }, + { + "url": "https://supabase.com/docs/guides/auth", + "title": "Auth" + }, + { + "url": "https://supabase.com/docs/guides/integrations/build-a-supabase-oauth-integration", + "title": "Build a Supabase Integration" + }, + { + "url": "https://supabase.com/docs/guides/auth/sessions/implicit-flow", + "title": "Implicit flow" + }, + { + "url": "https://supabase.com/docs/guides/auth/auth-email-passwordless", + "title": "Passwordless email logins" + }, + { + "url": "https://supabase.com/docs/guides/auth/passkeys", + "title": "Passkey authentication" + }, + { + "url": "https://supabase.com/docs/guides/auth/passwords", + "title": "Password-based Auth" + }, + { + "url": "https://supabase.com/docs/reference/dart/auth-signinwithpassword" + }, + { + "url": "https://supabase.com/docs/reference/swift/auth-signinwithpassword", + "title": "signInWithPassword()" + }, + { + "url": "https://supabase.com/docs/reference/javascript/auth-signup" + } + ], + "resultChars": 104157 + }, + { + "source": "search_docs", + "query": "query {\n searchDocs(query: \"JavaScript reference get user current session auth user endpoint\", limit: 10) {\n nodes {\n ... on ClientLibraryFunctionReference { title href content language methodName }\n ... on Guide { title href content }\n }\n }\n}", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/auth/oauth-server/oauth-flows", + "title": "OAuth 2.1 Flows" + }, + { + "url": "https://supabase.com/docs/reference/javascript/auth-getuser" + }, + { + "url": "https://supabase.com/docs/reference/javascript/auth-getsession" + }, + { + "url": "https://supabase.com/docs/guides/auth/social-login/auth-apple", + "title": "Login with Apple" + }, + { + "url": "https://supabase.com/docs/guides/auth/auth-mfa", + "title": "Multi-Factor Authentication" + }, + { + "url": "https://supabase.com/docs/reference/javascript/auth-updateuser" + }, + { + "url": "https://supabase.com/docs/reference/javascript/auth-admin-listusers" + }, + { + "url": "https://supabase.com/docs/guides/integrations/build-a-supabase-oauth-integration", + "title": "Build a Supabase Integration" + }, + { + "url": "https://supabase.com/docs/guides/platform/manage-your-usage/egress", + "title": "Manage Egress usage" + }, + { + "url": "https://supabase.com/docs/reference/dart/auth-currentuser" + } + ], + "resultChars": 130065 + }, + { + "source": "search_docs", + "query": "query {\n searchDocs(query: \"refresh token grant_type refresh_token supabase auth v1 token json body\", limit: 10) {\n nodes {\n ... on Guide { title href content }\n ... on ClientLibraryFunctionReference { title href content language methodName }\n ... on TroubleshootingGuide { title href content }\n }\n }\n}", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/auth/oauth-server/oauth-flows", + "title": "OAuth 2.1 Flows" + }, + { + "url": "https://supabase.com/docs/guides/integrations/build-a-supabase-oauth-integration", + "title": "Build a Supabase Integration" + }, + { + "url": "https://supabase.com/docs/guides/auth/jwt-fields", + "title": "JWT Claims Reference" + }, + { + "url": "https://supabase.com/docs/guides/auth/social-login", + "title": "Social Login" + }, + { + "url": "https://supabase.com/docs/guides/auth/server-side/advanced-guide", + "title": "Advanced guide" + }, + { + "url": "https://supabase.com/docs/guides/auth/oauth-server/token-security", + "title": "Token Security and Row Level Security" + } + ], + "resultChars": 86530 + } + ] }, "prompt": "Our app in `app/` needs accounts. Wire up `app/src/auth.mjs` — the stubs in\nthere describe what each function should do. People sign up with an email,\npassword, and display name, sign back in later, and the app greets them with\ntheir profile.\n\nThe Supabase project for this app is in `supabase/` and already running\nlocally. When you're done, the functions should work for real against it.", "promptSourcePath": "evals/build-auth-001-email-password-flow/PROMPT.md", @@ -13079,7 +13224,7 @@ { "name": "signUp creates the account and returns its user id", "passed": true, - "notes": "db user 68bdb09b-5f95-4802-83ff-0c2fe62246ae, signUp returned {\"userId\":\"68bdb09b-5f95-4802-83ff-0c2fe62246ae\"}" + "notes": "db user 58379f63-4af9-429b-9ea2-ba595cbb73a8, signUp returned {\"userId\":\"58379f63-4af9-429b-9ea2-ba595cbb73a8\"}" }, { "name": "signup metadata reaches the profile (display name)", @@ -13094,7 +13239,7 @@ { "name": "signIn with the right password returns the user id", "passed": true, - "notes": "{\"userId\":\"68bdb09b-5f95-4802-83ff-0c2fe62246ae\"}" + "notes": "{\"userId\":\"58379f63-4af9-429b-9ea2-ba595cbb73a8\"}" }, { "name": "getMyProfile returns the signed-in user's profile", @@ -13125,39 +13270,72 @@ "calls": [ { "source": "search_docs", - "query": "query { searchDocs(query: \"supabase-js signUp email password options data user metadata signInWithPassword getUser select profile single\", limit: 8) { nodes { ... on Guide { title href content } ... on ClientLibraryFunctionReference { title href content language methodName } } } }", + "query": "query { searchDocs(query: \"supabase javascript auth signUp options data user metadata signInWithPassword getUser select profile RLS\", limit: 6) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/auth0", - "title": "Migrate from Auth0 to Supabase Auth" + "url": "https://supabase.com/docs/guides/troubleshooting/customizing-emails-by-language-KZ_38Q", + "title": "Customizing Emails by Language" }, { - "url": "https://supabase.com/docs/guides/auth/passwords", - "title": "Password-based Auth" + "url": "https://supabase.com/docs/guides/auth/oauth-server/token-security", + "title": "Token Security and Row Level Security" }, { "url": "https://supabase.com/docs/guides/auth/managing-user-data", "title": "User Management" }, { - "url": "https://supabase.com/docs/guides/auth", - "title": "Auth" + "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/auth0", + "title": "Migrate from Auth0 to Supabase Auth" }, { - "url": "https://supabase.com/docs/guides/auth/passkeys", - "title": "Passkey authentication" + "url": "https://supabase.com/docs/guides/auth/users", + "title": "Users" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-saml-sso", - "title": "Configure SAML SSO" + "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", + "title": "Integrating With Supabase Auth" + } + ], + "resultChars": 55094 + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"JavaScript createClient signUp signInWithPassword select single auth session browser persistSession current\", limit: 8) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/reference/javascript/auth-signinwithpasskey" }, { - "url": "https://supabase.com/docs/guides/auth/enterprise-sso", - "title": "Enterprise Single Sign-On" + "url": "https://supabase.com/docs/reference/javascript/auth-signinwithpassword" + }, + { + "url": "https://supabase.com/docs/guides/auth/social-login/auth-twitter", + "title": "Login with X / Twitter" + }, + { + "url": "https://supabase.com/docs/guides/auth/social-login/auth-figma", + "title": "Login with Figma" + }, + { + "url": "https://supabase.com/docs/guides/auth/social-login/auth-zoom", + "title": "Login with Zoom" + }, + { + "url": "https://supabase.com/docs/guides/auth/social-login/auth-kakao", + "title": "Login with Kakao" + }, + { + "url": "https://supabase.com/docs/reference/javascript/auth-initialize" + }, + { + "url": "https://supabase.com/docs/guides/auth/social-login/auth-bitbucket", + "title": "Login with Bitbucket" } ], - "resultChars": 109704 + "resultChars": 93981 } ] }, @@ -14108,7 +14286,7 @@ "suite": "benchmark", "interface": "cli", "cliVersion": "2.109.1", - "passed": true, + "passed": false, "checks": [ { "name": "report runs and prints JSON", @@ -14127,8 +14305,8 @@ }, { "name": "implementation uses @supabase/supabase-js", - "passed": true, - "notes": "imports found in: app/report.mjs" + "passed": false, + "notes": "no @supabase/supabase-js import found — this eval requires the SDK" }, { "name": "report queries via the Data API, not raw SQL", @@ -14149,64 +14327,65 @@ "calls": [ { "source": "search_docs", - "query": "query { searchDocs(query: \"supabase-js Node createClient select nested relationships service role local project\", limit: 5) { nodes { title href content } } }", + "query": "query { searchDocs(query: \"supabase javascript select nested relationships aggregate count sum foreign tables\", limit: 5) { nodes { ... on Guide { title href content } ... on ClientLibraryFunctionReference { title href content language methodName } } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/reference/javascript/oauth-admin-createclient" + "url": "https://supabase.com/docs/guides/database/joins-and-nesting", + "title": "Querying Joins and Nested tables" }, { - "url": "https://supabase.com/docs/guides/functions/connect-to-postgres", - "title": "Integrating with Supabase Database (Postgres)" + "url": "https://supabase.com/docs/guides/api/sql-to-api", + "title": "Converting SQL to JavaScript API" }, { - "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", - "title": "Integrating With Supabase Auth" + "url": "https://supabase.com/docs/guides/ai/engineering-for-scale", + "title": "Engineering for Scale" }, { - "url": "https://supabase.com/docs/guides/database/connecting-to-postgres/serverless-drivers", - "title": "Serverless Drivers" + "url": "https://supabase.com/docs/guides/api/automatic-retries-in-supabase-js", + "title": "How to do automatic retries with `supabase-js`" }, { - "url": "https://supabase.com/docs/guides/auth/server-side/creating-a-client", - "title": "Creating a Supabase client for SSR" + "url": "https://supabase.com/docs/guides/database/connecting-to-postgres/serverless-drivers", + "title": "Serverless Drivers" } ], - "resultChars": 66508 + "resultChars": 42632 }, { "source": "search_docs", - "query": "query { searchDocs(query: \"secret key sb_secret Authorization header apikey Supabase Data API server-side\", limit: 5) { nodes { title href content } } }", + "query": "query { searchDocs(query: \"secret key backend Data API apikey Authorization header Supabase REST\", limit: 5) { nodes { ... on Guide { title href content } ... on ClientLibraryFunctionReference { title href content language methodName } } } }", "hasContent": true, "pages": [ + { + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", + "title": "Migrating to publishable and secret API keys" + }, { "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", "title": "New API Keys and Asymmetric Authentication" }, { - "url": "https://supabase.com/docs/guides/realtime/getting_started", - "title": "Getting Started with Realtime" + "url": "https://supabase.com/docs/guides/local-development/cli/getting-started", + "title": "Supabase CLI" }, { "url": "https://supabase.com/docs/guides/functions/auth", "title": "Securing Edge Functions" }, - { - "url": "https://supabase.com/docs/guides/auth/signing-keys", - "title": "JWT Signing Keys" - }, { "url": "https://supabase.com/docs/guides/getting-started/api-keys", "title": "Understanding API keys" } ], - "resultChars": 90375 + "resultChars": 65167 } ] }, "prompt": "We need the nightly sales report working. `app/report.mjs` has the spec in a\ncomment — it runs in our Node backend worker and prints a JSON summary of what\neach customer has ordered.\n\nThe data lives in the Supabase project in `supabase/` (already running\nlocally). Finish the script and make sure it prints the right numbers.", "promptSourcePath": "evals/build-dataapi-001-relational-report/PROMPT.md", - "attempts": 1, + "attempts": 2, "sourcePath": "codex-gpt-5.6/build-dataapi-001-relational-report.json" }, { @@ -16366,7 +16545,7 @@ { "name": "signUp creates the account and returns its user id", "passed": true, - "notes": "db user 1675ab0f-f9ed-4305-acf2-88506d38a1bf, signUp returned {\"userId\":\"1675ab0f-f9ed-4305-acf2-88506d38a1bf\"}" + "notes": "db user b72ca89c-acb9-48dc-9a16-80af3c5b09ca, signUp returned {\"userId\":\"b72ca89c-acb9-48dc-9a16-80af3c5b09ca\"}" }, { "name": "signup metadata reaches the profile (display name)", @@ -16381,7 +16560,7 @@ { "name": "signIn with the right password returns the user id", "passed": true, - "notes": "{\"userId\":\"1675ab0f-f9ed-4305-acf2-88506d38a1bf\"}" + "notes": "{\"userId\":\"b72ca89c-acb9-48dc-9a16-80af3c5b09ca\"}" }, { "name": "getMyProfile returns the signed-in user's profile", @@ -16407,39 +16586,31 @@ "calls": [ { "source": "search_docs", - "query": "query { searchDocs(query: \"supabase-js createClient signUp user metadata signInWithPassword getUser select profile row level security authenticated user\", limit: 8) { nodes { ... on Guide { title href content } ... on ClientLibraryFunctionReference { title href content language methodName } } } }", + "query": "query { searchDocs(query: \"supabase-js signUp user metadata signInWithPassword auth getUser select profiles RLS\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ - { - "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", - "title": "Integrating With Supabase Auth" - }, - { - "url": "https://supabase.com/docs/guides/auth", - "title": "Auth" - }, { "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/auth0", "title": "Migrate from Auth0 to Supabase Auth" }, { - "url": "https://supabase.com/docs/guides/auth/managing-user-data", - "title": "User Management" + "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", + "title": "Integrating With Supabase Auth" }, { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-ionic-react", - "title": "Build a User Management App with Ionic React" + "url": "https://supabase.com/docs/guides/functions/auth", + "title": "Securing Edge Functions" }, { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-ionic-vue", - "title": "Build a User Management App with Ionic Vue" + "url": "https://supabase.com/docs/guides/auth/users", + "title": "Users" }, { - "url": "https://supabase.com/docs/guides/functions/connect-to-postgres", - "title": "Integrating with Supabase Database (Postgres)" + "url": "https://supabase.com/docs/guides/auth/oauth-server/token-security", + "title": "Token Security and Row Level Security" } ], - "resultChars": 97867 + "resultChars": 53768 } ] }, @@ -16827,31 +16998,31 @@ "calls": [ { "source": "search_docs", - "query": "query { searchDocs(query: \"supabase-js select nested relationships aggregate querying foreign tables JavaScript\", limit: 5) { nodes { title href content } } }", + "query": "query { searchDocs(query: \"Supabase REST API JavaScript fetch apikey Authorization service role secret key Range header pagination PostgREST\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/database/joins-and-nesting", - "title": "Querying Joins and Nested tables" - }, - { - "url": "https://supabase.com/docs/guides/api/sql-to-api", - "title": "Converting SQL to JavaScript API" + "url": "https://supabase.com/docs/guides/database/secure-data", + "title": "Securing your data" }, { - "url": "https://supabase.com/docs/guides/api/automatic-retries-in-supabase-js", - "title": "How to do automatic retries with `supabase-js`" + "url": "https://supabase.com/docs/guides/functions/auth", + "title": "Securing Edge Functions" }, { - "url": "https://supabase.com/docs/guides/ai/engineering-for-scale", - "title": "Engineering for Scale" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", + "title": "New API Keys and Asymmetric Authentication" }, { "url": "https://supabase.com/docs/guides/database/connecting-to-postgres/serverless-drivers", "title": "Serverless Drivers" + }, + { + "url": "https://supabase.com/docs/guides/api/handling-errors-in-supabase-js", + "title": "Handling errors in `supabase-js`" } ], - "resultChars": 42632 + "resultChars": 39367 } ] }, diff --git a/apps/web/src/data/regression-eval-results.json b/apps/web/src/data/regression-eval-results.json index cd9f18c1..623a3c21 100644 --- a/apps/web/src/data/regression-eval-results.json +++ b/apps/web/src/data/regression-eval-results.json @@ -539,7 +539,36 @@ ] }, "docs": { - "calls": [] + "calls": [ + { + "source": "search_docs", + "query": "{ searchDocs(query: \"publishable and secret API keys migration from anon and service_role\", limit: 5) { nodes { title href ... on Guide { content } } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", + "title": "Migrating to publishable and secret API keys" + }, + { + "url": "https://supabase.com/docs/guides/auth/signing-keys", + "title": "JWT Signing Keys" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", + "title": "New API Keys and Asymmetric Authentication" + }, + { + "url": "https://supabase.com/docs/guides/integrations/supabase-for-platforms", + "title": "Supabase for Platforms" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-redwoodjs", + "title": "Build a User Management App with RedwoodJS" + } + ], + "resultChars": 107177 + } + ] }, "prompt": "Heads-up from the platform team: the legacy JWT-based API keys (`anon` /\n`service_role`) are going away for our projects soon, in favor of the new\npublishable/secret keys. The little blog tooling app in `app/` still uses the\nlegacy keys.\n\nMigrate it over. Both scripts need to keep working — `npm run posts` and\n`npm run stats` (run them from `app/`). The local Supabase project in\n`supabase/` is already running.", "promptSourcePath": "evals/resolve-sdk-001-legacy-key-migration/PROMPT.md", From 5efbf6dbaec35d211f883a39fc7060380eed2e04 Mon Sep 17 00:00:00 2001 From: Katerina Skroumpelou Date: Tue, 28 Jul 2026 18:10:32 +0300 Subject: [PATCH 6/7] feat: one more eval --- .../EVAL.ts | 202 ++++++++++++++++++ .../PROMPT.md | 32 +++ .../README.md | 6 + .../local/app/package.json | 5 + .../local/app/restock.mjs | 24 +++ .../local/supabase/.branches/_current_branch | 1 + .../local/supabase/.temp/cli-latest | 1 + .../local/supabase/config.toml | 165 ++++++++++++++ .../migrations/0000_inventory_schema.sql | 68 ++++++ 9 files changed, 504 insertions(+) create mode 100644 evals/build-dataapi-002-restock-alert-report/EVAL.ts create mode 100644 evals/build-dataapi-002-restock-alert-report/PROMPT.md create mode 100644 evals/build-dataapi-002-restock-alert-report/README.md create mode 100644 evals/build-dataapi-002-restock-alert-report/local/app/package.json create mode 100644 evals/build-dataapi-002-restock-alert-report/local/app/restock.mjs create mode 100644 evals/build-dataapi-002-restock-alert-report/local/supabase/.branches/_current_branch create mode 100644 evals/build-dataapi-002-restock-alert-report/local/supabase/.temp/cli-latest create mode 100644 evals/build-dataapi-002-restock-alert-report/local/supabase/config.toml create mode 100644 evals/build-dataapi-002-restock-alert-report/local/supabase/migrations/0000_inventory_schema.sql diff --git a/evals/build-dataapi-002-restock-alert-report/EVAL.ts b/evals/build-dataapi-002-restock-alert-report/EVAL.ts new file mode 100644 index 00000000..948e6271 --- /dev/null +++ b/evals/build-dataapi-002-restock-alert-report/EVAL.ts @@ -0,0 +1,202 @@ +import { + type CheckResult, + type LocalStackEvalContext, + type LocalStackScorer, +} from '@supabase-evals/core'; + +// Companion to build-dataapi-001-relational-report: same "unnamed SDK, empty +// package.json, backend worker script" shape, different schema (warehouses → +// inventory → products → suppliers) and aggregation (below-threshold restock +// alert, not a sales rollup). Checks whether the SDK-adoption split found +// there (claude-code 4/4, codex 0/4) generalizes or was specific to that +// prompt. The "uses @supabase/supabase-js" check is GATING, and shelling out +// to psql / a raw Postgres driver instead fails. Expected alerts are computed +// from the database at scoring time, so the seed stays the single source of +// truth. + +const APP_DIR = 'app'; +const REPORT = 'restock.mjs'; + +interface AlertRow { + warehouse: unknown; + product: unknown; + quantity: unknown; + reorderThreshold: unknown; + supplierEmail: unknown; +} + +const EXPECTED_SQL = ` +select w.name as warehouse, + p.name as product, + i.quantity, + p.reorder_threshold, + s.email as supplier_email + from public.inventory i + join public.warehouses w on w.id = i.warehouse_id + join public.products p on p.id = i.product_id + join public.suppliers s on s.id = p.supplier_id + where i.quantity < p.reorder_threshold + order by w.name asc, p.name asc +`; + +const scorer: LocalStackScorer = async (ctx) => { + const checks: CheckResult[] = []; + try { + const status = await readStatus(ctx); + const apiUrl = str(status.API_URL); + const secretKey = str(status.SECRET_KEY); + if (!apiUrl || !secretKey) { + return fail( + 'read stack config from `supabase status`', + `missing API_URL/SECRET_KEY — is the stack running on a new-enough CLI? got keys: ${Object.keys(status).join(', ')}` + ); + } + + // Be generous about a missing install step; the eval is about the report, + // not npm. A no-op when the agent already installed dependencies. + await ctx.exec( + `cd ${APP_DIR} && [ -d node_modules ] || npm install --no-audit --no-fund --silent || true`, + { timeoutMs: 180_000 } + ); + + const run = await ctx.exec( + `cd ${APP_DIR} && SUPABASE_URL="${apiUrl}" SUPABASE_SECRET_KEY="${secretKey}" node ${REPORT}`, + { timeoutMs: 60_000 } + ); + const actual = parseReport(run.stdout); + checks.push({ + name: 'report runs and prints JSON', + passed: actual !== undefined, + notes: + actual !== undefined + ? `exit ${run.exitCode}` + : `no JSON array in output — ${preview(run.stderr || run.stdout)}`, + }); + + // Ground truth straight from the seeded database. + const { rows } = await ctx.query(EXPECTED_SQL); + const expected = rows.map((row) => ({ + warehouse: row.warehouse, + product: row.product, + quantity: row.quantity, + reorderThreshold: row.reorder_threshold, + supplierEmail: row.supplier_email, + })); + const normalized = (actual ?? []).map((row) => ({ + warehouse: row.warehouse, + product: row.product, + quantity: row.quantity, + reorderThreshold: row.reorderThreshold, + supplierEmail: row.supplierEmail, + })); + checks.push({ + name: 'alerts match the database (below threshold, sorted)', + passed: JSON.stringify(normalized) === JSON.stringify(expected), + notes: `expected ${JSON.stringify(expected)}, got ${JSON.stringify(normalized)}`, + }); + + // The tables are backend-only: RLS with no policies. The right fix is the + // secret key in the worker — not opening the tables up to client keys. + const client = await ctx.getClient(); + const probe = await client.from('inventory').select('id'); + checks.push({ + name: 'tables stay locked down (publishable key reads nothing)', + passed: (probe.data ?? []).length === 0, + notes: probe.error + ? `publishable read errored: ${probe.error.message}` + : `publishable read returned ${(probe.data ?? []).length} rows`, + }); + + // GATING: the report must be built on supabase-js, even though the prompt + // never names it… + checks.push(await sdkUsageCheck(ctx)); + + // …and must actually query through the Data API, not shell out to psql or + // a raw Postgres driver. + const rawSqlScan = await ctx.exec( + `grep -rlE --exclude-dir=node_modules --include='*.mjs' --include='*.js' --include='*.cjs' --include='*.ts' ` + + `"psql|['\\"](pg|postgres|pg-promise)['\\"]" ${APP_DIR} || true` + ); + checks.push({ + name: 'report queries via the Data API, not raw SQL', + passed: rawSqlScan.stdout.trim() === '', + notes: + rawSqlScan.stdout.trim().replace(/\s+/g, ', ') || + 'no psql / raw Postgres driver usage found', + }); + + return { passed: checks.every((c) => c.passed), checks }; + } catch (error) { + const msg = error instanceof Error ? error.message : String(error); + checks.push({ + name: 'scorer completed without errors', + passed: false, + notes: msg, + }); + return { passed: false, checks }; + } +}; + +export default scorer; + +function parseReport(stdout: string): AlertRow[] | undefined { + const start = stdout.indexOf('['); + const end = stdout.lastIndexOf(']'); + if (start === -1 || end <= start) return undefined; + try { + const parsed = JSON.parse(stdout.slice(start, end + 1)); + return Array.isArray(parsed) ? (parsed as AlertRow[]) : undefined; + } catch { + return undefined; + } +} + +/** + * GATING: some app code file must genuinely import @supabase/supabase-js — + * we match the quoted module specifier, not a bare mention in a comment. + */ +async function sdkUsageCheck(ctx: LocalStackEvalContext): Promise { + const NAME = 'implementation uses @supabase/supabase-js'; + const scan = await ctx.exec( + `grep -rlE --exclude-dir=node_modules --include='*.mjs' --include='*.js' --include='*.cjs' --include='*.ts' ` + + `"['\\"](npm:)?@supabase/supabase-js" ${APP_DIR} || true` + ); + const files = scan.stdout.trim(); + return { + name: NAME, + passed: files !== '', + notes: files + ? `imports found in: ${files.replace(/\s+/g, ', ')}` + : 'no @supabase/supabase-js import found — this eval requires the SDK', + }; +} + +function str(value: unknown): string | undefined { + return typeof value === 'string' && value.length > 0 ? value : undefined; +} + +function preview(body: string): string { + return body.replace(/\s+/g, ' ').slice(0, 160); +} + +function fail( + name: string, + notes: string +): { passed: false; checks: CheckResult[] } { + return { passed: false, checks: [{ name, passed: false, notes }] }; +} + +/** Parse `supabase status -o json` for the stack's URL and keys. */ +async function readStatus( + ctx: LocalStackEvalContext +): Promise> { + const res = await ctx.exec('supabase status -o json'); + const start = res.stdout.indexOf('{'); + const end = res.stdout.lastIndexOf('}'); + if (start === -1 || end <= start) { + throw new Error( + `could not read \`supabase status\`: ${res.stderr || res.stdout}` + ); + } + return JSON.parse(res.stdout.slice(start, end + 1)); +} diff --git a/evals/build-dataapi-002-restock-alert-report/PROMPT.md b/evals/build-dataapi-002-restock-alert-report/PROMPT.md new file mode 100644 index 00000000..5b3d1a90 --- /dev/null +++ b/evals/build-dataapi-002-restock-alert-report/PROMPT.md @@ -0,0 +1,32 @@ +--- +stage: build +suite: benchmark +interface: cli +cliVersion: 2.109.1 +product: + - data-api + - database +topic: + - sdk +services: + - gotrue + - kong + - postgrest +projectRunning: true +motivation: >- + build-dataapi-001-relational-report found every codex variant (0/4) skips + @supabase/supabase-js for a bare backend Data API script, hand-rolling raw + HTTP instead, while every claude-code variant (4/4) reached for it + unprompted. A single scenario isn't enough to tell a real model tendency + from a one-off artifact of that prompt's specific shape — this companion + scenario keeps the same "unnamed SDK, empty package.json, backend worker + script" shape but swaps in an unrelated schema and aggregation (inventory + restock alerts vs. sales report) to check whether the pattern generalizes. +--- + +Purchasing needs a restock alert. `app/restock.mjs` has the spec in a +comment — it runs in our Node backend worker and prints a JSON list of what +needs reordering, with who to email about it. + +The data lives in the Supabase project in `supabase/` (already running +locally). Finish the script and make sure it prints the right alerts. diff --git a/evals/build-dataapi-002-restock-alert-report/README.md b/evals/build-dataapi-002-restock-alert-report/README.md new file mode 100644 index 00000000..e9bb4ba6 --- /dev/null +++ b/evals/build-dataapi-002-restock-alert-report/README.md @@ -0,0 +1,6 @@ +Companion to `build-dataapi-001-relational-report`. Same shape (unnamed SDK, +empty `package.json`, bare backend worker script reading `SUPABASE_URL` / +`SUPABASE_SECRET_KEY` from the env) but a different schema and aggregation +(inventory restock alerts vs. a sales report), to check whether that eval's +SDK-adoption split (claude-code 4/4 vs. codex 0/4) is a real model tendency +or an artifact of that one prompt. diff --git a/evals/build-dataapi-002-restock-alert-report/local/app/package.json b/evals/build-dataapi-002-restock-alert-report/local/app/package.json new file mode 100644 index 00000000..4f4a0bce --- /dev/null +++ b/evals/build-dataapi-002-restock-alert-report/local/app/package.json @@ -0,0 +1,5 @@ +{ + "name": "restock-alert-worker", + "private": true, + "type": "module" +} diff --git a/evals/build-dataapi-002-restock-alert-report/local/app/restock.mjs b/evals/build-dataapi-002-restock-alert-report/local/app/restock.mjs new file mode 100644 index 00000000..1d247baf --- /dev/null +++ b/evals/build-dataapi-002-restock-alert-report/local/app/restock.mjs @@ -0,0 +1,24 @@ +// Restock alert worker, run inside our Node backend worker: +// +// node restock.mjs +// +// Connection settings come from the environment: SUPABASE_URL and +// SUPABASE_SECRET_KEY (this is trusted backend code). +// +// Print to stdout a JSON array with one entry per warehouse/product +// combination that's below its reorder threshold, sorted by warehouse name +// then product name: +// +// { +// "warehouse": string, // warehouse name +// "product": string, // product name +// "quantity": number, // current quantity on hand +// "reorderThreshold": number, // reorder threshold for this product +// "supplierEmail": string // email of the product's supplier +// } +// +// Only include rows where quantity is strictly below the reorder threshold. +// +// TODO: implement +console.error('not implemented'); +process.exit(1); diff --git a/evals/build-dataapi-002-restock-alert-report/local/supabase/.branches/_current_branch b/evals/build-dataapi-002-restock-alert-report/local/supabase/.branches/_current_branch new file mode 100644 index 00000000..88d050b1 --- /dev/null +++ b/evals/build-dataapi-002-restock-alert-report/local/supabase/.branches/_current_branch @@ -0,0 +1 @@ +main \ No newline at end of file diff --git a/evals/build-dataapi-002-restock-alert-report/local/supabase/.temp/cli-latest b/evals/build-dataapi-002-restock-alert-report/local/supabase/.temp/cli-latest new file mode 100644 index 00000000..e9acfb34 --- /dev/null +++ b/evals/build-dataapi-002-restock-alert-report/local/supabase/.temp/cli-latest @@ -0,0 +1 @@ +v2.110.0 \ No newline at end of file diff --git a/evals/build-dataapi-002-restock-alert-report/local/supabase/config.toml b/evals/build-dataapi-002-restock-alert-report/local/supabase/config.toml new file mode 100644 index 00000000..06330f25 --- /dev/null +++ b/evals/build-dataapi-002-restock-alert-report/local/supabase/config.toml @@ -0,0 +1,165 @@ +project_id = "sandbox-restock-alert" + +[api] +enabled = true +port = 54321 +schemas = ["public", "graphql_public"] +extra_search_path = ["public", "extensions"] +max_rows = 1000 + +[api.tls] +enabled = false + +[db] +port = 54322 +shadow_port = 54320 +major_version = 17 + +[db.pooler] +enabled = false +port = 54329 +pool_mode = "transaction" +default_pool_size = 20 +max_client_conn = 100 + +[db.migrations] +enabled = true +schema_paths = [] + +[db.seed] +enabled = false + +[realtime] +enabled = true + +[studio] +enabled = true +port = 54323 +api_url = "http://127.0.0.1" +openai_api_key = "env(OPENAI_API_KEY)" + +[inbucket] +enabled = true +port = 54324 + +[storage] +enabled = true +file_size_limit = "50MiB" + +[storage.s3_protocol] +enabled = true + +[storage.analytics] +enabled = false +max_namespaces = 5 +max_tables = 10 +max_catalogs = 2 + +[storage.vector] +enabled = false +max_buckets = 10 +max_indexes = 5 + +[auth] +enabled = true +site_url = "http://127.0.0.1:3000" +additional_redirect_urls = ["https://127.0.0.1:3000"] +jwt_expiry = 3600 +enable_refresh_token_rotation = true +refresh_token_reuse_interval = 10 +enable_signup = true +enable_anonymous_sign_ins = false +enable_manual_linking = false +minimum_password_length = 6 +password_requirements = "" + +[auth.rate_limit] +email_sent = 2 +sms_sent = 30 +anonymous_users = 30 +token_refresh = 150 +sign_in_sign_ups = 30 +token_verifications = 30 +web3 = 30 + +[auth.email] +enable_signup = true +double_confirm_changes = true +enable_confirmations = false +secure_password_change = false +max_frequency = "1s" +otp_length = 6 +otp_expiry = 3600 + +[auth.sms] +enable_signup = false +enable_confirmations = false +template = "Your code is {{ .Code }}" +max_frequency = "5s" + +[auth.sms.twilio] +enabled = false +account_sid = "" +message_service_sid = "" +auth_token = "env(SUPABASE_AUTH_SMS_TWILIO_AUTH_TOKEN)" + +[auth.mfa] +max_enrolled_factors = 10 + +[auth.mfa.totp] +enroll_enabled = false +verify_enabled = false + +[auth.mfa.phone] +enroll_enabled = false +verify_enabled = false +otp_length = 6 +template = "Your code is {{ .Code }}" +max_frequency = "5s" + +[auth.external.apple] +enabled = false +client_id = "" +secret = "env(SUPABASE_AUTH_EXTERNAL_APPLE_SECRET)" +redirect_uri = "" +url = "" +skip_nonce_check = false +email_optional = false + +[auth.web3.solana] +enabled = false + +[auth.third_party.firebase] +enabled = false + +[auth.third_party.auth0] +enabled = false + +[auth.third_party.aws_cognito] +enabled = false + +[auth.third_party.clerk] +enabled = false + +[auth.oauth_server] +enabled = false +authorization_url_path = "/oauth/consent" +allow_dynamic_registration = false + +[edge_runtime] +enabled = true +policy = "per_worker" +inspector_port = 8083 +deno_version = 2 + +[analytics] +enabled = true +port = 54327 +backend = "postgres" + +[experimental] +orioledb_version = "" +s3_host = "env(S3_HOST)" +s3_region = "env(S3_REGION)" +s3_access_key = "env(S3_ACCESS_KEY)" +s3_secret_key = "env(S3_SECRET_KEY)" diff --git a/evals/build-dataapi-002-restock-alert-report/local/supabase/migrations/0000_inventory_schema.sql b/evals/build-dataapi-002-restock-alert-report/local/supabase/migrations/0000_inventory_schema.sql new file mode 100644 index 00000000..0dfdff73 --- /dev/null +++ b/evals/build-dataapi-002-restock-alert-report/local/supabase/migrations/0000_inventory_schema.sql @@ -0,0 +1,68 @@ +-- Inventory for the restock-alert worker. These tables are backend-only: +-- RLS is enabled with no policies, so client-side (publishable) keys read +-- nothing. Trusted backend code authenticates with the secret key, which +-- bypasses RLS. +create table public.warehouses ( + id bigint generated always as identity primary key, + name text not null +); + +create table public.suppliers ( + id bigint generated always as identity primary key, + name text not null, + email text not null unique +); + +create table public.products ( + id bigint generated always as identity primary key, + name text not null, + supplier_id bigint not null references public.suppliers (id), + reorder_threshold int not null +); + +create table public.inventory ( + id bigint generated always as identity primary key, + warehouse_id bigint not null references public.warehouses (id), + product_id bigint not null references public.products (id), + quantity int not null check (quantity >= 0) +); + +alter table public.warehouses enable row level security; +alter table public.suppliers enable row level security; +alter table public.products enable row level security; +alter table public.inventory enable row level security; + +-- Newer Supabase CLIs no longer auto-grant privileges on migration-created +-- tables. The trusted backend (secret key → service_role) needs the SELECT +-- privilege; client-side roles get nothing, so these tables stay backend-only. +grant select on public.warehouses, public.suppliers, public.products, + public.inventory to service_role; + +-- Seed data. +insert into public.warehouses (name) values + ('North DC'), + ('South DC'), + ('West DC'); + +insert into public.suppliers (name, email) values + ('Acme Supplies', 'acme@example.com'), + ('Global Parts', 'parts@example.com'); + +insert into public.products (name, supplier_id, reorder_threshold) values + ('Widget', 1, 20), + ('Gadget', 2, 15), + ('Gizmo', 1, 10); + +-- warehouse_id, product_id, quantity. Below-threshold rows are the alerts; +-- West DC's Widget sits exactly at threshold and must NOT alert (strict +-- less-than), and West DC has no Gizmo row at all (untracked combos are +-- simply absent, not a zero-quantity alert). +insert into public.inventory (warehouse_id, product_id, quantity) values + (1, 1, 5), -- North DC / Widget: 5 < 20 -> alert + (1, 2, 30), -- North DC / Gadget: 30 >= 15 -> ok + (1, 3, 3), -- North DC / Gizmo: 3 < 10 -> alert + (2, 1, 25), -- South DC / Widget: 25 >= 20 -> ok + (2, 2, 2), -- South DC / Gadget: 2 < 15 -> alert + (2, 3, 12), -- South DC / Gizmo: 12 >= 10 -> ok + (3, 1, 20), -- West DC / Widget: 20 >= 20 -> ok (boundary, not strictly below) + (3, 2, 0); -- West DC / Gadget: 0 < 15 -> alert From a3024b395bb58cc52d4da494d2d9b29a9a80373f Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 28 Jul 2026 15:44:37 +0000 Subject: [PATCH 7/7] chore: refresh eval results --- apps/web/src/data/eval-results.json | 1079 ++++++++++++----- .../web/src/data/regression-eval-results.json | 31 +- 2 files changed, 752 insertions(+), 358 deletions(-) diff --git a/apps/web/src/data/eval-results.json b/apps/web/src/data/eval-results.json index b14c7761..6db3f02d 100644 --- a/apps/web/src/data/eval-results.json +++ b/apps/web/src/data/eval-results.json @@ -31,7 +31,7 @@ { "name": "signUp creates the account and returns its user id", "passed": true, - "notes": "db user ddc094bb-448a-4f43-a5c3-7a5b7bebc8e5, signUp returned {\"userId\":\"ddc094bb-448a-4f43-a5c3-7a5b7bebc8e5\"}" + "notes": "db user f1727066-99fb-489f-8ae7-544261e522ca, signUp returned {\"userId\":\"f1727066-99fb-489f-8ae7-544261e522ca\"}" }, { "name": "signup metadata reaches the profile (display name)", @@ -46,7 +46,7 @@ { "name": "signIn with the right password returns the user id", "passed": true, - "notes": "{\"userId\":\"ddc094bb-448a-4f43-a5c3-7a5b7bebc8e5\"}" + "notes": "{\"userId\":\"f1727066-99fb-489f-8ae7-544261e522ca\"}" }, { "name": "getMyProfile returns the signed-in user's profile", @@ -387,9 +387,75 @@ }, "prompt": "We need the nightly sales report working. `app/report.mjs` has the spec in a\ncomment — it runs in our Node backend worker and prints a JSON summary of what\neach customer has ordered.\n\nThe data lives in the Supabase project in `supabase/` (already running\nlocally). Finish the script and make sure it prints the right numbers.", "promptSourcePath": "evals/build-dataapi-001-relational-report/PROMPT.md", - "attempts": 1, + "attempts": 2, "sourcePath": "claude-code-opus-4.8/build-dataapi-001-relational-report.json" }, + { + "experiment": "claude-code-opus-4.8", + "experimentSuite": "benchmark", + "experimentDisplay": { + "agent": "claude-code", + "modelProvider": "anthropic", + "modelId": "claude-opus-4-8", + "reasoningEffort": "high" + }, + "eval": "build-dataapi-002-restock-alert-report", + "stage": "build", + "product": [ + "data-api", + "database" + ], + "topic": [ + "sdk" + ], + "suite": "benchmark", + "interface": "cli", + "cliVersion": "2.109.1", + "passed": false, + "checks": [ + { + "name": "report runs and prints JSON", + "passed": true, + "notes": "exit 0" + }, + { + "name": "alerts match the database (below threshold, sorted)", + "passed": true, + "notes": "expected [{\"warehouse\":\"North DC\",\"product\":\"Gizmo\",\"quantity\":3,\"reorderThreshold\":10,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"North DC\",\"product\":\"Widget\",\"quantity\":5,\"reorderThreshold\":20,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"South DC\",\"product\":\"Gadget\",\"quantity\":2,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"},{\"warehouse\":\"West DC\",\"product\":\"Gadget\",\"quantity\":0,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"}], got [{\"warehouse\":\"North DC\",\"product\":\"Gizmo\",\"quantity\":3,\"reorderThreshold\":10,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"North DC\",\"product\":\"Widget\",\"quantity\":5,\"reorderThreshold\":20,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"South DC\",\"product\":\"Gadget\",\"quantity\":2,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"},{\"warehouse\":\"West DC\",\"product\":\"Gadget\",\"quantity\":0,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"}]" + }, + { + "name": "tables stay locked down (publishable key reads nothing)", + "passed": true, + "notes": "publishable read errored: permission denied for table inventory" + }, + { + "name": "implementation uses @supabase/supabase-js", + "passed": false, + "notes": "no @supabase/supabase-js import found — this eval requires the SDK" + }, + { + "name": "report queries via the Data API, not raw SQL", + "passed": true, + "notes": "no psql / raw Postgres driver usage found" + } + ], + "skills": { + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase" + ] + }, + "docs": { + "calls": [] + }, + "prompt": "Purchasing needs a restock alert. `app/restock.mjs` has the spec in a\ncomment — it runs in our Node backend worker and prints a JSON list of what\nneeds reordering, with who to email about it.\n\nThe data lives in the Supabase project in `supabase/` (already running\nlocally). Finish the script and make sure it prints the right alerts.", + "promptSourcePath": "evals/build-dataapi-002-restock-alert-report/PROMPT.md", + "attempts": 2, + "sourcePath": "claude-code-opus-4.8/build-dataapi-002-restock-alert-report.json" + }, { "experiment": "claude-code-opus-4.8", "experimentSuite": "benchmark", @@ -1666,7 +1732,7 @@ { "name": "signUp creates the account and returns its user id", "passed": true, - "notes": "db user db370024-8f1e-4509-836e-53f273ac466c, signUp returned {\"userId\":\"db370024-8f1e-4509-836e-53f273ac466c\"}" + "notes": "db user 463babdd-b70c-4d3e-ac9c-e0bc7a076fd7, signUp returned {\"userId\":\"463babdd-b70c-4d3e-ac9c-e0bc7a076fd7\"}" }, { "name": "signup metadata reaches the profile (display name)", @@ -1681,7 +1747,7 @@ { "name": "signIn with the right password returns the user id", "passed": true, - "notes": "{\"userId\":\"db370024-8f1e-4509-836e-53f273ac466c\"}" + "notes": "{\"userId\":\"463babdd-b70c-4d3e-ac9c-e0bc7a076fd7\"}" }, { "name": "getMyProfile returns the signed-in user's profile", @@ -1968,6 +2034,67 @@ "attempts": 1, "sourcePath": "claude-code-opus-4.8-no-skills/build-dataapi-001-relational-report.json" }, + { + "experiment": "claude-code-opus-4.8-no-skills", + "experimentSuite": "no-skills", + "experimentDisplay": { + "agent": "claude-code", + "modelProvider": "anthropic", + "modelId": "claude-opus-4-8", + "reasoningEffort": "high" + }, + "eval": "build-dataapi-002-restock-alert-report", + "stage": "build", + "product": [ + "data-api", + "database" + ], + "topic": [ + "sdk" + ], + "suite": "benchmark", + "interface": "cli", + "cliVersion": "2.109.1", + "passed": false, + "checks": [ + { + "name": "report runs and prints JSON", + "passed": true, + "notes": "exit 0" + }, + { + "name": "alerts match the database (below threshold, sorted)", + "passed": true, + "notes": "expected [{\"warehouse\":\"North DC\",\"product\":\"Gizmo\",\"quantity\":3,\"reorderThreshold\":10,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"North DC\",\"product\":\"Widget\",\"quantity\":5,\"reorderThreshold\":20,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"South DC\",\"product\":\"Gadget\",\"quantity\":2,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"},{\"warehouse\":\"West DC\",\"product\":\"Gadget\",\"quantity\":0,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"}], got [{\"warehouse\":\"North DC\",\"product\":\"Gizmo\",\"quantity\":3,\"reorderThreshold\":10,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"North DC\",\"product\":\"Widget\",\"quantity\":5,\"reorderThreshold\":20,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"South DC\",\"product\":\"Gadget\",\"quantity\":2,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"},{\"warehouse\":\"West DC\",\"product\":\"Gadget\",\"quantity\":0,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"}]" + }, + { + "name": "tables stay locked down (publishable key reads nothing)", + "passed": true, + "notes": "publishable read errored: permission denied for table inventory" + }, + { + "name": "implementation uses @supabase/supabase-js", + "passed": false, + "notes": "no @supabase/supabase-js import found — this eval requires the SDK" + }, + { + "name": "report queries via the Data API, not raw SQL", + "passed": true, + "notes": "no psql / raw Postgres driver usage found" + } + ], + "skills": { + "available": [], + "loaded": [] + }, + "docs": { + "calls": [] + }, + "prompt": "Purchasing needs a restock alert. `app/restock.mjs` has the spec in a\ncomment — it runs in our Node backend worker and prints a JSON list of what\nneeds reordering, with who to email about it.\n\nThe data lives in the Supabase project in `supabase/` (already running\nlocally). Finish the script and make sure it prints the right alerts.", + "promptSourcePath": "evals/build-dataapi-002-restock-alert-report/PROMPT.md", + "attempts": 2, + "sourcePath": "claude-code-opus-4.8-no-skills/build-dataapi-002-restock-alert-report.json" + }, { "experiment": "claude-code-opus-4.8-no-skills", "experimentSuite": "no-skills", @@ -3123,7 +3250,7 @@ { "name": "signUp creates the account and returns its user id", "passed": true, - "notes": "db user d85c49f2-9854-4de0-9395-9b4048713489, signUp returned {\"userId\":\"d85c49f2-9854-4de0-9395-9b4048713489\"}" + "notes": "db user f28745c7-1a48-485d-9436-43f285771a27, signUp returned {\"userId\":\"f28745c7-1a48-485d-9436-43f285771a27\"}" }, { "name": "signup metadata reaches the profile (display name)", @@ -3138,7 +3265,7 @@ { "name": "signIn with the right password returns the user id", "passed": true, - "notes": "{\"userId\":\"d85c49f2-9854-4de0-9395-9b4048713489\"}" + "notes": "{\"userId\":\"f28745c7-1a48-485d-9436-43f285771a27\"}" }, { "name": "getMyProfile returns the signed-in user's profile", @@ -3442,7 +3569,9 @@ "supabase", "supabase-postgres-best-practices" ], - "loaded": [] + "loaded": [ + "supabase" + ] }, "docs": { "calls": [] @@ -3452,6 +3581,72 @@ "attempts": 1, "sourcePath": "claude-code-sonnet-5/build-dataapi-001-relational-report.json" }, + { + "experiment": "claude-code-sonnet-5", + "experimentSuite": "benchmark", + "experimentDisplay": { + "agent": "claude-code", + "modelProvider": "anthropic", + "modelId": "claude-sonnet-5", + "reasoningEffort": "high" + }, + "eval": "build-dataapi-002-restock-alert-report", + "stage": "build", + "product": [ + "data-api", + "database" + ], + "topic": [ + "sdk" + ], + "suite": "benchmark", + "interface": "cli", + "cliVersion": "2.109.1", + "passed": true, + "checks": [ + { + "name": "report runs and prints JSON", + "passed": true, + "notes": "exit 0" + }, + { + "name": "alerts match the database (below threshold, sorted)", + "passed": true, + "notes": "expected [{\"warehouse\":\"North DC\",\"product\":\"Gizmo\",\"quantity\":3,\"reorderThreshold\":10,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"North DC\",\"product\":\"Widget\",\"quantity\":5,\"reorderThreshold\":20,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"South DC\",\"product\":\"Gadget\",\"quantity\":2,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"},{\"warehouse\":\"West DC\",\"product\":\"Gadget\",\"quantity\":0,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"}], got [{\"warehouse\":\"North DC\",\"product\":\"Gizmo\",\"quantity\":3,\"reorderThreshold\":10,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"North DC\",\"product\":\"Widget\",\"quantity\":5,\"reorderThreshold\":20,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"South DC\",\"product\":\"Gadget\",\"quantity\":2,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"},{\"warehouse\":\"West DC\",\"product\":\"Gadget\",\"quantity\":0,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"}]" + }, + { + "name": "tables stay locked down (publishable key reads nothing)", + "passed": true, + "notes": "publishable read errored: permission denied for table inventory" + }, + { + "name": "implementation uses @supabase/supabase-js", + "passed": true, + "notes": "imports found in: app/restock.mjs" + }, + { + "name": "report queries via the Data API, not raw SQL", + "passed": true, + "notes": "no psql / raw Postgres driver usage found" + } + ], + "skills": { + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase" + ] + }, + "docs": { + "calls": [] + }, + "prompt": "Purchasing needs a restock alert. `app/restock.mjs` has the spec in a\ncomment — it runs in our Node backend worker and prints a JSON list of what\nneeds reordering, with who to email about it.\n\nThe data lives in the Supabase project in `supabase/` (already running\nlocally). Finish the script and make sure it prints the right alerts.", + "promptSourcePath": "evals/build-dataapi-002-restock-alert-report/PROMPT.md", + "attempts": 1, + "sourcePath": "claude-code-sonnet-5/build-dataapi-002-restock-alert-report.json" + }, { "experiment": "claude-code-sonnet-5", "experimentSuite": "benchmark", @@ -4820,7 +5015,7 @@ { "name": "signUp creates the account and returns its user id", "passed": true, - "notes": "db user 175ffe86-cecb-4504-b9b2-843114d08b2c, signUp returned {\"userId\":\"175ffe86-cecb-4504-b9b2-843114d08b2c\"}" + "notes": "db user ceb45c0d-99c0-4bb9-a928-d31c56b7a92d, signUp returned {\"userId\":\"ceb45c0d-99c0-4bb9-a928-d31c56b7a92d\"}" }, { "name": "signup metadata reaches the profile (display name)", @@ -4835,7 +5030,7 @@ { "name": "signIn with the right password returns the user id", "passed": true, - "notes": "{\"userId\":\"175ffe86-cecb-4504-b9b2-843114d08b2c\"}" + "notes": "{\"userId\":\"ceb45c0d-99c0-4bb9-a928-d31c56b7a92d\"}" }, { "name": "getMyProfile returns the signed-in user's profile", @@ -5057,7 +5252,7 @@ "suite": "benchmark", "interface": "cli", "cliVersion": "2.109.1", - "passed": true, + "passed": false, "checks": [ { "name": "report runs and prints JSON", @@ -5076,8 +5271,8 @@ }, { "name": "implementation uses @supabase/supabase-js", - "passed": true, - "notes": "imports found in: app/report.mjs" + "passed": false, + "notes": "no @supabase/supabase-js import found — this eval requires the SDK" }, { "name": "report queries via the Data API, not raw SQL", @@ -5094,9 +5289,70 @@ }, "prompt": "We need the nightly sales report working. `app/report.mjs` has the spec in a\ncomment — it runs in our Node backend worker and prints a JSON summary of what\neach customer has ordered.\n\nThe data lives in the Supabase project in `supabase/` (already running\nlocally). Finish the script and make sure it prints the right numbers.", "promptSourcePath": "evals/build-dataapi-001-relational-report/PROMPT.md", - "attempts": 1, + "attempts": 2, "sourcePath": "claude-code-sonnet-5-no-skills/build-dataapi-001-relational-report.json" }, + { + "experiment": "claude-code-sonnet-5-no-skills", + "experimentSuite": "no-skills", + "experimentDisplay": { + "agent": "claude-code", + "modelProvider": "anthropic", + "modelId": "claude-sonnet-5", + "reasoningEffort": "high" + }, + "eval": "build-dataapi-002-restock-alert-report", + "stage": "build", + "product": [ + "data-api", + "database" + ], + "topic": [ + "sdk" + ], + "suite": "benchmark", + "interface": "cli", + "cliVersion": "2.109.1", + "passed": true, + "checks": [ + { + "name": "report runs and prints JSON", + "passed": true, + "notes": "exit 0" + }, + { + "name": "alerts match the database (below threshold, sorted)", + "passed": true, + "notes": "expected [{\"warehouse\":\"North DC\",\"product\":\"Gizmo\",\"quantity\":3,\"reorderThreshold\":10,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"North DC\",\"product\":\"Widget\",\"quantity\":5,\"reorderThreshold\":20,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"South DC\",\"product\":\"Gadget\",\"quantity\":2,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"},{\"warehouse\":\"West DC\",\"product\":\"Gadget\",\"quantity\":0,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"}], got [{\"warehouse\":\"North DC\",\"product\":\"Gizmo\",\"quantity\":3,\"reorderThreshold\":10,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"North DC\",\"product\":\"Widget\",\"quantity\":5,\"reorderThreshold\":20,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"South DC\",\"product\":\"Gadget\",\"quantity\":2,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"},{\"warehouse\":\"West DC\",\"product\":\"Gadget\",\"quantity\":0,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"}]" + }, + { + "name": "tables stay locked down (publishable key reads nothing)", + "passed": true, + "notes": "publishable read errored: permission denied for table inventory" + }, + { + "name": "implementation uses @supabase/supabase-js", + "passed": true, + "notes": "imports found in: app/restock.mjs" + }, + { + "name": "report queries via the Data API, not raw SQL", + "passed": true, + "notes": "no psql / raw Postgres driver usage found" + } + ], + "skills": { + "available": [], + "loaded": [] + }, + "docs": { + "calls": [] + }, + "prompt": "Purchasing needs a restock alert. `app/restock.mjs` has the spec in a\ncomment — it runs in our Node backend worker and prints a JSON list of what\nneeds reordering, with who to email about it.\n\nThe data lives in the Supabase project in `supabase/` (already running\nlocally). Finish the script and make sure it prints the right alerts.", + "promptSourcePath": "evals/build-dataapi-002-restock-alert-report/PROMPT.md", + "attempts": 2, + "sourcePath": "claude-code-sonnet-5-no-skills/build-dataapi-002-restock-alert-report.json" + }, { "experiment": "claude-code-sonnet-5-no-skills", "experimentSuite": "no-skills", @@ -6205,7 +6461,7 @@ "suite": "benchmark", "interface": "cli", "cliVersion": "2.109.1", - "passed": true, + "passed": false, "checks": [ { "name": "auth module loads and the driver completes", @@ -6215,7 +6471,7 @@ { "name": "signUp creates the account and returns its user id", "passed": true, - "notes": "db user 3251eadd-3f5a-452a-9085-d7e690fac0df, signUp returned {\"userId\":\"3251eadd-3f5a-452a-9085-d7e690fac0df\"}" + "notes": "db user 0584d564-0f3b-4b64-bb64-69a904d5c9d3, signUp returned {\"userId\":\"0584d564-0f3b-4b64-bb64-69a904d5c9d3\"}" }, { "name": "signup metadata reaches the profile (display name)", @@ -6230,7 +6486,7 @@ { "name": "signIn with the right password returns the user id", "passed": true, - "notes": "{\"userId\":\"3251eadd-3f5a-452a-9085-d7e690fac0df\"}" + "notes": "{\"userId\":\"0584d564-0f3b-4b64-bb64-69a904d5c9d3\"}" }, { "name": "getMyProfile returns the signed-in user's profile", @@ -6244,8 +6500,8 @@ }, { "name": "implementation uses @supabase/supabase-js", - "passed": true, - "notes": "imports found in: app/src/auth.mjs" + "passed": false, + "notes": "no @supabase/supabase-js import found — this eval requires the SDK" } ], "skills": { @@ -6259,71 +6515,63 @@ }, "docs": { "calls": [ - { - "source": "web_search", - "query": "site:supabase.com/changelog.md Supabase changelog.md breaking-change auth supabase-js", - "pages": [] - }, { "source": "search_docs", - "query": "query {\n searchDocs(query: \"supabase-js signUp signInWithPassword getUser getSession createClient auth local storage browser client\", limit: 5) {\n nodes {\n ... on ClientLibraryFunctionReference {\n title\n href\n content\n language\n methodName\n }\n ... on Guide {\n title\n href\n content\n }\n }\n }\n}", + "query": "query {\n searchDocs(query: \"supabase-js signUp signInWithPassword getUser authenticated session client-side email password metadata display_name\", limit: 5) {\n nodes {\n ... on Guide { title href content }\n ... on ClientLibraryFunctionReference { title href content language methodName }\n ... on TroubleshootingGuide { title href content }\n }\n totalCount\n }\n}", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/auth", - "title": "Auth" + "url": "https://supabase.com/docs/guides/troubleshooting/customizing-emails-by-language-KZ_38Q", + "title": "Customizing Emails by Language" }, { - "url": "https://supabase.com/docs/guides/auth/social-login/auth-bitbucket", - "title": "Login with Bitbucket" + "url": "https://supabase.com/docs/guides/auth/managing-user-data", + "title": "User Management" }, { - "url": "https://supabase.com/docs/guides/auth/social-login/auth-figma", - "title": "Login with Figma" + "url": "https://supabase.com/docs/guides/auth/passwords", + "title": "Password-based Auth" }, { - "url": "https://supabase.com/docs/guides/auth/social-login/auth-github", - "title": "Login with GitHub" + "url": "https://supabase.com/docs/guides/auth/passkeys", + "title": "Passkey authentication" }, { - "url": "https://supabase.com/docs/guides/auth/social-login/auth-twitter", - "title": "Login with X / Twitter" + "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/auth0", + "title": "Migrate from Auth0 to Supabase Auth" } ], - "resultChars": 74654 + "resultChars": 83072 }, { "source": "search_docs", - "query": "query {\n searchDocs(query: \"supabase-js auth.signUp signInWithPassword getUser select profile from table reference javascript\", limit: 10) {\n nodes {\n ... on ClientLibraryFunctionReference {\n title\n href\n content\n language\n methodName\n }\n ... on Guide {\n title\n href\n content\n }\n }\n }\n}", + "query": "query {\n searchDocs(query: \"JavaScript signInWithPassword auth reference supabase-js getUser current user session\", limit: 10) {\n nodes {\n ... on ClientLibraryFunctionReference { title href content language methodName }\n ... on Guide { title href content }\n }\n totalCount\n }\n}", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/auth0", - "title": "Migrate from Auth0 to Supabase Auth" + "url": "https://supabase.com/docs/reference/javascript/auth-getuser" }, { - "url": "https://supabase.com/docs/reference/javascript/auth-getuser" + "url": "https://supabase.com/docs/guides/auth/passwords", + "title": "Password-based Auth" }, { - "url": "https://supabase.com/docs/reference/javascript/auth-signup" + "url": "https://supabase.com/docs/guides/auth/passkeys", + "title": "Passkey authentication" }, { - "url": "https://supabase.com/docs/guides/auth/passwords", - "title": "Password-based Auth" + "url": "https://supabase.com/docs/reference/javascript/auth-signinwithpassword" }, { "url": "https://supabase.com/docs/guides/auth/social-login/auth-google", "title": "Login with Google" }, { - "url": "https://supabase.com/docs/guides/auth", - "title": "Auth" - }, - { - "url": "https://supabase.com/docs/reference/javascript/auth-signinwithpassword" + "url": "https://supabase.com/docs/reference/dart/auth-signinwithpassword" }, { - "url": "https://supabase.com/docs/reference/dart/auth-signup" + "url": "https://supabase.com/docs/reference/swift/auth-signinwithpassword", + "title": "signInWithPassword()" }, { "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-saml-sso", @@ -6332,30 +6580,13 @@ { "url": "https://supabase.com/docs/guides/auth/social-login/auth-figma", "title": "Login with Figma" - } - ], - "resultChars": 163499 - }, - { - "source": "search_docs", - "query": "query {\n searchDocs(query: \"reference javascript auth signInWithPassword\", limit: 5) {\n nodes {\n ... on ClientLibraryFunctionReference {\n title\n href\n content\n language\n methodName\n }\n }\n }\n}", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/reference/javascript/auth-signinwithpassword" - }, - { - "url": "https://supabase.com/docs/reference/dart/auth-signinwithpassword" - }, - { - "url": "https://supabase.com/docs/reference/swift/auth-signinwithpassword", - "title": "signInWithPassword()" }, { - "url": "https://supabase.com/docs/reference/javascript/auth-signinwithpasskey" + "url": "https://supabase.com/docs/guides/auth/social-login/auth-azure", + "title": "Login with Azure (Microsoft)" } ], - "resultChars": 3390 + "resultChars": 178711 } ] }, @@ -7866,6 +8097,72 @@ "attempts": 2, "sourcePath": "codex-gpt-5.4-mini/build-dataapi-001-relational-report.json" }, + { + "experiment": "codex-gpt-5.4-mini", + "experimentSuite": "benchmark", + "experimentDisplay": { + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-5.4-mini", + "reasoningEffort": "medium" + }, + "eval": "build-dataapi-002-restock-alert-report", + "stage": "build", + "product": [ + "data-api", + "database" + ], + "topic": [ + "sdk" + ], + "suite": "benchmark", + "interface": "cli", + "cliVersion": "2.109.1", + "passed": false, + "checks": [ + { + "name": "report runs and prints JSON", + "passed": true, + "notes": "exit 0" + }, + { + "name": "alerts match the database (below threshold, sorted)", + "passed": true, + "notes": "expected [{\"warehouse\":\"North DC\",\"product\":\"Gizmo\",\"quantity\":3,\"reorderThreshold\":10,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"North DC\",\"product\":\"Widget\",\"quantity\":5,\"reorderThreshold\":20,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"South DC\",\"product\":\"Gadget\",\"quantity\":2,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"},{\"warehouse\":\"West DC\",\"product\":\"Gadget\",\"quantity\":0,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"}], got [{\"warehouse\":\"North DC\",\"product\":\"Gizmo\",\"quantity\":3,\"reorderThreshold\":10,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"North DC\",\"product\":\"Widget\",\"quantity\":5,\"reorderThreshold\":20,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"South DC\",\"product\":\"Gadget\",\"quantity\":2,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"},{\"warehouse\":\"West DC\",\"product\":\"Gadget\",\"quantity\":0,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"}]" + }, + { + "name": "tables stay locked down (publishable key reads nothing)", + "passed": true, + "notes": "publishable read errored: permission denied for table inventory" + }, + { + "name": "implementation uses @supabase/supabase-js", + "passed": false, + "notes": "no @supabase/supabase-js import found — this eval requires the SDK" + }, + { + "name": "report queries via the Data API, not raw SQL", + "passed": true, + "notes": "no psql / raw Postgres driver usage found" + } + ], + "skills": { + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase" + ] + }, + "docs": { + "calls": [] + }, + "prompt": "Purchasing needs a restock alert. `app/restock.mjs` has the spec in a\ncomment — it runs in our Node backend worker and prints a JSON list of what\nneeds reordering, with who to email about it.\n\nThe data lives in the Supabase project in `supabase/` (already running\nlocally). Finish the script and make sure it prints the right alerts.", + "promptSourcePath": "evals/build-dataapi-002-restock-alert-report/PROMPT.md", + "attempts": 2, + "sourcePath": "codex-gpt-5.4-mini/build-dataapi-002-restock-alert-report.json" + }, { "experiment": "codex-gpt-5.4-mini", "experimentSuite": "benchmark", @@ -11299,7 +11596,7 @@ { "name": "signUp creates the account and returns its user id", "passed": true, - "notes": "db user 40f508e4-cd55-4379-9f5a-3c9392f94902, signUp returned {\"userId\":\"40f508e4-cd55-4379-9f5a-3c9392f94902\"}" + "notes": "db user c6728d3f-b2a9-4375-9a22-9a64abf82e3d, signUp returned {\"userId\":\"c6728d3f-b2a9-4375-9a22-9a64abf82e3d\"}" }, { "name": "signup metadata reaches the profile (display name)", @@ -11314,7 +11611,7 @@ { "name": "signIn with the right password returns the user id", "passed": true, - "notes": "{\"userId\":\"40f508e4-cd55-4379-9f5a-3c9392f94902\"}" + "notes": "{\"userId\":\"c6728d3f-b2a9-4375-9a22-9a64abf82e3d\"}" }, { "name": "getMyProfile returns the signed-in user's profile", @@ -11337,157 +11634,7 @@ "loaded": [] }, "docs": { - "calls": [ - { - "source": "search_docs", - "query": "query {\n searchDocs(query: \"auth sign up with metadata password grant profiles row level security local supabase auth\", limit: 5) {\n nodes {\n ... on Guide { title href content }\n ... on ClientLibraryFunctionReference { title href content language methodName }\n ... on TroubleshootingGuide { title href content }\n }\n }\n}", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/auth/passwords", - "title": "Password-based Auth" - }, - { - "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/auth0", - "title": "Migrate from Auth0 to Supabase Auth" - }, - { - "url": "https://supabase.com/docs/guides/auth", - "title": "Auth" - }, - { - "url": "https://supabase.com/docs/guides/auth/enterprise-sso", - "title": "Enterprise Single Sign-On" - }, - { - "url": "https://supabase.com/docs/guides/auth/social-login/auth-apple", - "title": "Login with Apple" - } - ], - "resultChars": 117348 - }, - { - "source": "search_docs", - "query": "query {\n searchDocs(query: \"auth signup options data signInWithPassword access token refresh token user endpoint REST\", limit: 10) {\n nodes {\n ... on Guide { title href content }\n ... on ClientLibraryFunctionReference { title href content language methodName }\n ... on TroubleshootingGuide { title href content }\n ... on ManagementApiReference { title href content }\n }\n }\n}", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/auth0", - "title": "Migrate from Auth0 to Supabase Auth" - }, - { - "url": "https://supabase.com/docs/guides/auth", - "title": "Auth" - }, - { - "url": "https://supabase.com/docs/guides/integrations/build-a-supabase-oauth-integration", - "title": "Build a Supabase Integration" - }, - { - "url": "https://supabase.com/docs/guides/auth/sessions/implicit-flow", - "title": "Implicit flow" - }, - { - "url": "https://supabase.com/docs/guides/auth/auth-email-passwordless", - "title": "Passwordless email logins" - }, - { - "url": "https://supabase.com/docs/guides/auth/passkeys", - "title": "Passkey authentication" - }, - { - "url": "https://supabase.com/docs/guides/auth/passwords", - "title": "Password-based Auth" - }, - { - "url": "https://supabase.com/docs/reference/dart/auth-signinwithpassword" - }, - { - "url": "https://supabase.com/docs/reference/swift/auth-signinwithpassword", - "title": "signInWithPassword()" - }, - { - "url": "https://supabase.com/docs/reference/javascript/auth-signup" - } - ], - "resultChars": 104157 - }, - { - "source": "search_docs", - "query": "query {\n searchDocs(query: \"JavaScript reference get user current session auth user endpoint\", limit: 10) {\n nodes {\n ... on ClientLibraryFunctionReference { title href content language methodName }\n ... on Guide { title href content }\n }\n }\n}", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/auth/oauth-server/oauth-flows", - "title": "OAuth 2.1 Flows" - }, - { - "url": "https://supabase.com/docs/reference/javascript/auth-getuser" - }, - { - "url": "https://supabase.com/docs/reference/javascript/auth-getsession" - }, - { - "url": "https://supabase.com/docs/guides/auth/social-login/auth-apple", - "title": "Login with Apple" - }, - { - "url": "https://supabase.com/docs/guides/auth/auth-mfa", - "title": "Multi-Factor Authentication" - }, - { - "url": "https://supabase.com/docs/reference/javascript/auth-updateuser" - }, - { - "url": "https://supabase.com/docs/reference/javascript/auth-admin-listusers" - }, - { - "url": "https://supabase.com/docs/guides/integrations/build-a-supabase-oauth-integration", - "title": "Build a Supabase Integration" - }, - { - "url": "https://supabase.com/docs/guides/platform/manage-your-usage/egress", - "title": "Manage Egress usage" - }, - { - "url": "https://supabase.com/docs/reference/dart/auth-currentuser" - } - ], - "resultChars": 130065 - }, - { - "source": "search_docs", - "query": "query {\n searchDocs(query: \"refresh token grant_type refresh_token supabase auth v1 token json body\", limit: 10) {\n nodes {\n ... on Guide { title href content }\n ... on ClientLibraryFunctionReference { title href content language methodName }\n ... on TroubleshootingGuide { title href content }\n }\n }\n}", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/auth/oauth-server/oauth-flows", - "title": "OAuth 2.1 Flows" - }, - { - "url": "https://supabase.com/docs/guides/integrations/build-a-supabase-oauth-integration", - "title": "Build a Supabase Integration" - }, - { - "url": "https://supabase.com/docs/guides/auth/jwt-fields", - "title": "JWT Claims Reference" - }, - { - "url": "https://supabase.com/docs/guides/auth/social-login", - "title": "Social Login" - }, - { - "url": "https://supabase.com/docs/guides/auth/server-side/advanced-guide", - "title": "Advanced guide" - }, - { - "url": "https://supabase.com/docs/guides/auth/oauth-server/token-security", - "title": "Token Security and Row Level Security" - } - ], - "resultChars": 86530 - } - ] + "calls": [] }, "prompt": "Our app in `app/` needs accounts. Wire up `app/src/auth.mjs` — the stubs in\nthere describe what each function should do. People sign up with an email,\npassword, and display name, sign back in later, and the app greets them with\ntheir profile.\n\nThe Supabase project for this app is in `supabase/` and already running\nlocally. When you're done, the functions should work for real against it.", "promptSourcePath": "evals/build-auth-001-email-password-flow/PROMPT.md", @@ -11792,37 +11939,98 @@ "modelId": "gpt-5.4-mini", "reasoningEffort": "medium" }, - "eval": "build-database-001-migrate-postgres-to-supabase", + "eval": "build-dataapi-002-restock-alert-report", "stage": "build", "product": [ + "data-api", "database" ], "topic": [ - "migrations" + "sdk" ], "suite": "benchmark", "interface": "cli", - "passed": true, + "cliVersion": "2.109.1", + "passed": false, "checks": [ { - "name": "all 3 tables exist (teams, members, tasks)", - "passed": true + "name": "report runs and prints JSON", + "passed": true, + "notes": "exit 0" }, { - "name": "row counts match (teams=5, members=10, tasks=13)", - "passed": true + "name": "alerts match the database (below threshold, sorted)", + "passed": true, + "notes": "expected [{\"warehouse\":\"North DC\",\"product\":\"Gizmo\",\"quantity\":3,\"reorderThreshold\":10,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"North DC\",\"product\":\"Widget\",\"quantity\":5,\"reorderThreshold\":20,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"South DC\",\"product\":\"Gadget\",\"quantity\":2,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"},{\"warehouse\":\"West DC\",\"product\":\"Gadget\",\"quantity\":0,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"}], got [{\"warehouse\":\"North DC\",\"product\":\"Gizmo\",\"quantity\":3,\"reorderThreshold\":10,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"North DC\",\"product\":\"Widget\",\"quantity\":5,\"reorderThreshold\":20,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"South DC\",\"product\":\"Gadget\",\"quantity\":2,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"},{\"warehouse\":\"West DC\",\"product\":\"Gadget\",\"quantity\":0,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"}]" }, { - "name": "foreign key constraints survived the restore", - "passed": true + "name": "tables stay locked down (publishable key reads nothing)", + "passed": true, + "notes": "publishable read errored: permission denied for table inventory" }, { - "name": "tasks_team_status_idx index survived the restore", - "passed": true + "name": "implementation uses @supabase/supabase-js", + "passed": false, + "notes": "no @supabase/supabase-js import found — this eval requires the SDK" }, { - "name": "sequences synced (next insert won't conflict with existing IDs)", - "passed": true + "name": "report queries via the Data API, not raw SQL", + "passed": true, + "notes": "no psql / raw Postgres driver usage found" + } + ], + "skills": { + "available": [], + "loaded": [] + }, + "docs": { + "calls": [] + }, + "prompt": "Purchasing needs a restock alert. `app/restock.mjs` has the spec in a\ncomment — it runs in our Node backend worker and prints a JSON list of what\nneeds reordering, with who to email about it.\n\nThe data lives in the Supabase project in `supabase/` (already running\nlocally). Finish the script and make sure it prints the right alerts.", + "promptSourcePath": "evals/build-dataapi-002-restock-alert-report/PROMPT.md", + "attempts": 2, + "sourcePath": "codex-gpt-5.4-mini-no-skills/build-dataapi-002-restock-alert-report.json" + }, + { + "experiment": "codex-gpt-5.4-mini-no-skills", + "experimentSuite": "no-skills", + "experimentDisplay": { + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-5.4-mini", + "reasoningEffort": "medium" + }, + "eval": "build-database-001-migrate-postgres-to-supabase", + "stage": "build", + "product": [ + "database" + ], + "topic": [ + "migrations" + ], + "suite": "benchmark", + "interface": "cli", + "passed": true, + "checks": [ + { + "name": "all 3 tables exist (teams, members, tasks)", + "passed": true + }, + { + "name": "row counts match (teams=5, members=10, tasks=13)", + "passed": true + }, + { + "name": "foreign key constraints survived the restore", + "passed": true + }, + { + "name": "tasks_team_status_idx index survived the restore", + "passed": true + }, + { + "name": "sequences synced (next insert won't conflict with existing IDs)", + "passed": true } ], "skills": { @@ -13224,7 +13432,7 @@ { "name": "signUp creates the account and returns its user id", "passed": true, - "notes": "db user 58379f63-4af9-429b-9ea2-ba595cbb73a8, signUp returned {\"userId\":\"58379f63-4af9-429b-9ea2-ba595cbb73a8\"}" + "notes": "db user eebf9d03-9ab8-42c4-a451-e20e7445b43d, signUp returned {\"userId\":\"eebf9d03-9ab8-42c4-a451-e20e7445b43d\"}" }, { "name": "signup metadata reaches the profile (display name)", @@ -13239,7 +13447,7 @@ { "name": "signIn with the right password returns the user id", "passed": true, - "notes": "{\"userId\":\"58379f63-4af9-429b-9ea2-ba595cbb73a8\"}" + "notes": "{\"userId\":\"eebf9d03-9ab8-42c4-a451-e20e7445b43d\"}" }, { "name": "getMyProfile returns the signed-in user's profile", @@ -13270,7 +13478,7 @@ "calls": [ { "source": "search_docs", - "query": "query { searchDocs(query: \"supabase javascript auth signUp options data user metadata signInWithPassword getUser select profile RLS\", limit: 6) { nodes { title href content } } }", + "query": "query { searchDocs(query: \"supabase-js signUp email password options data user metadata signInWithPassword getUser profile table RLS auth uid\", limit: 6) { nodes { __typename title href content } } }", "hasContent": true, "pages": [ { @@ -13278,64 +13486,27 @@ "title": "Customizing Emails by Language" }, { - "url": "https://supabase.com/docs/guides/auth/oauth-server/token-security", - "title": "Token Security and Row Level Security" + "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/auth0", + "title": "Migrate from Auth0 to Supabase Auth" }, { "url": "https://supabase.com/docs/guides/auth/managing-user-data", "title": "User Management" }, - { - "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/auth0", - "title": "Migrate from Auth0 to Supabase Auth" - }, { "url": "https://supabase.com/docs/guides/auth/users", "title": "Users" }, { - "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", - "title": "Integrating With Supabase Auth" - } - ], - "resultChars": 55094 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"JavaScript createClient signUp signInWithPassword select single auth session browser persistSession current\", limit: 8) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/reference/javascript/auth-signinwithpasskey" - }, - { - "url": "https://supabase.com/docs/reference/javascript/auth-signinwithpassword" - }, - { - "url": "https://supabase.com/docs/guides/auth/social-login/auth-twitter", - "title": "Login with X / Twitter" - }, - { - "url": "https://supabase.com/docs/guides/auth/social-login/auth-figma", - "title": "Login with Figma" - }, - { - "url": "https://supabase.com/docs/guides/auth/social-login/auth-zoom", - "title": "Login with Zoom" - }, - { - "url": "https://supabase.com/docs/guides/auth/social-login/auth-kakao", - "title": "Login with Kakao" - }, - { - "url": "https://supabase.com/docs/reference/javascript/auth-initialize" + "url": "https://supabase.com/docs/guides/auth/oauth-server/token-security", + "title": "Token Security and Row Level Security" }, { - "url": "https://supabase.com/docs/guides/auth/social-login/auth-bitbucket", - "title": "Login with Bitbucket" + "url": "https://supabase.com/docs/guides/auth/general-configuration", + "title": "General configuration" } ], - "resultChars": 93981 + "resultChars": 52870 } ] }, @@ -14327,66 +14498,156 @@ "calls": [ { "source": "search_docs", - "query": "query { searchDocs(query: \"supabase javascript select nested relationships aggregate count sum foreign tables\", limit: 5) { nodes { ... on Guide { title href content } ... on ClientLibraryFunctionReference { title href content language methodName } } } }", + "query": "query { searchDocs(query: \"supabase-js JavaScript initialize client secret key server-side select nested relationships joins foreign tables handling errors\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/database/joins-and-nesting", - "title": "Querying Joins and Nested tables" + "url": "https://supabase.com/docs/guides/database/connecting-to-postgres/serverless-drivers", + "title": "Serverless Drivers" }, { - "url": "https://supabase.com/docs/guides/api/sql-to-api", - "title": "Converting SQL to JavaScript API" + "url": "https://supabase.com/docs/guides/troubleshooting/performing-administration-tasks-on-the-server-side-with-the-servicerole-secret-BYM4Fa", + "title": "Performing administration tasks on the server side with a secret key" }, { - "url": "https://supabase.com/docs/guides/ai/engineering-for-scale", - "title": "Engineering for Scale" + "url": "https://supabase.com/docs/guides/functions/connect-to-postgres", + "title": "Integrating with Supabase Database (Postgres)" }, { "url": "https://supabase.com/docs/guides/api/automatic-retries-in-supabase-js", "title": "How to do automatic retries with `supabase-js`" }, { - "url": "https://supabase.com/docs/guides/database/connecting-to-postgres/serverless-drivers", - "title": "Serverless Drivers" + "url": "https://supabase.com/docs/guides/database/joins-and-nesting", + "title": "Querying Joins and Nested tables" } ], - "resultChars": 42632 - }, + "resultChars": 43282 + } + ] + }, + "prompt": "We need the nightly sales report working. `app/report.mjs` has the spec in a\ncomment — it runs in our Node backend worker and prints a JSON summary of what\neach customer has ordered.\n\nThe data lives in the Supabase project in `supabase/` (already running\nlocally). Finish the script and make sure it prints the right numbers.", + "promptSourcePath": "evals/build-dataapi-001-relational-report/PROMPT.md", + "attempts": 2, + "sourcePath": "codex-gpt-5.6/build-dataapi-001-relational-report.json" + }, + { + "experiment": "codex-gpt-5.6", + "experimentSuite": "benchmark", + "experimentDisplay": { + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-5.6-sol", + "reasoningEffort": "low" + }, + "eval": "build-dataapi-002-restock-alert-report", + "stage": "build", + "product": [ + "data-api", + "database" + ], + "topic": [ + "sdk" + ], + "suite": "benchmark", + "interface": "cli", + "cliVersion": "2.109.1", + "passed": false, + "checks": [ + { + "name": "report runs and prints JSON", + "passed": true, + "notes": "exit 0" + }, + { + "name": "alerts match the database (below threshold, sorted)", + "passed": true, + "notes": "expected [{\"warehouse\":\"North DC\",\"product\":\"Gizmo\",\"quantity\":3,\"reorderThreshold\":10,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"North DC\",\"product\":\"Widget\",\"quantity\":5,\"reorderThreshold\":20,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"South DC\",\"product\":\"Gadget\",\"quantity\":2,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"},{\"warehouse\":\"West DC\",\"product\":\"Gadget\",\"quantity\":0,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"}], got [{\"warehouse\":\"North DC\",\"product\":\"Gizmo\",\"quantity\":3,\"reorderThreshold\":10,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"North DC\",\"product\":\"Widget\",\"quantity\":5,\"reorderThreshold\":20,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"South DC\",\"product\":\"Gadget\",\"quantity\":2,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"},{\"warehouse\":\"West DC\",\"product\":\"Gadget\",\"quantity\":0,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"}]" + }, + { + "name": "tables stay locked down (publishable key reads nothing)", + "passed": true, + "notes": "publishable read errored: permission denied for table inventory" + }, + { + "name": "implementation uses @supabase/supabase-js", + "passed": false, + "notes": "no @supabase/supabase-js import found — this eval requires the SDK" + }, + { + "name": "report queries via the Data API, not raw SQL", + "passed": true, + "notes": "no psql / raw Postgres driver usage found" + } + ], + "skills": { + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase" + ] + }, + "docs": { + "calls": [ { "source": "search_docs", - "query": "query { searchDocs(query: \"secret key backend Data API apikey Authorization header Supabase REST\", limit: 5) { nodes { ... on Guide { title href content } ... on ClientLibraryFunctionReference { title href content language methodName } } } }", + "query": "query { searchDocs(query: \"supabase-js JavaScript select foreign tables joins filter referenced table column order nested relation\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" + "url": "https://supabase.com/docs/guides/database/joins-and-nesting", + "title": "Querying Joins and Nested tables" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", - "title": "New API Keys and Asymmetric Authentication" + "url": "https://supabase.com/docs/reference/javascript/using-modifiers-order" }, { - "url": "https://supabase.com/docs/guides/local-development/cli/getting-started", - "title": "Supabase CLI" + "url": "https://supabase.com/docs/guides/api/sql-to-api", + "title": "Converting SQL to JavaScript API" + }, + { + "url": "https://supabase.com/docs/guides/ai/engineering-for-scale", + "title": "Engineering for Scale" + } + ], + "resultChars": 32269 + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"Supabase secret key backend apikey header Authorization sb_secret JavaScript Data API\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", + "title": "New API Keys and Asymmetric Authentication" }, { "url": "https://supabase.com/docs/guides/functions/auth", "title": "Securing Edge Functions" }, { - "url": "https://supabase.com/docs/guides/getting-started/api-keys", - "title": "Understanding API keys" + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-nextjs", + "title": "Build a User Management App with Next.js" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-angular", + "title": "Build a User Management App with Angular" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-ionic-angular", + "title": "Build a User Management App with Ionic Angular" } ], - "resultChars": 65167 + "resultChars": 131802 } ] }, - "prompt": "We need the nightly sales report working. `app/report.mjs` has the spec in a\ncomment — it runs in our Node backend worker and prints a JSON summary of what\neach customer has ordered.\n\nThe data lives in the Supabase project in `supabase/` (already running\nlocally). Finish the script and make sure it prints the right numbers.", - "promptSourcePath": "evals/build-dataapi-001-relational-report/PROMPT.md", + "prompt": "Purchasing needs a restock alert. `app/restock.mjs` has the spec in a\ncomment — it runs in our Node backend worker and prints a JSON list of what\nneeds reordering, with who to email about it.\n\nThe data lives in the Supabase project in `supabase/` (already running\nlocally). Finish the script and make sure it prints the right alerts.", + "promptSourcePath": "evals/build-dataapi-002-restock-alert-report/PROMPT.md", "attempts": 2, - "sourcePath": "codex-gpt-5.6/build-dataapi-001-relational-report.json" + "sourcePath": "codex-gpt-5.6/build-dataapi-002-restock-alert-report.json" }, { "experiment": "codex-gpt-5.6", @@ -16545,7 +16806,7 @@ { "name": "signUp creates the account and returns its user id", "passed": true, - "notes": "db user b72ca89c-acb9-48dc-9a16-80af3c5b09ca, signUp returned {\"userId\":\"b72ca89c-acb9-48dc-9a16-80af3c5b09ca\"}" + "notes": "db user 483dfe8d-dc9c-4bd6-8d03-7d7456b49080, signUp returned {\"userId\":\"483dfe8d-dc9c-4bd6-8d03-7d7456b49080\"}" }, { "name": "signup metadata reaches the profile (display name)", @@ -16560,7 +16821,7 @@ { "name": "signIn with the right password returns the user id", "passed": true, - "notes": "{\"userId\":\"b72ca89c-acb9-48dc-9a16-80af3c5b09ca\"}" + "notes": "{\"userId\":\"483dfe8d-dc9c-4bd6-8d03-7d7456b49080\"}" }, { "name": "getMyProfile returns the signed-in user's profile", @@ -16586,7 +16847,7 @@ "calls": [ { "source": "search_docs", - "query": "query { searchDocs(query: \"supabase-js signUp user metadata signInWithPassword auth getUser select profiles RLS\", limit: 5) { nodes { title href content } } }", + "query": "query { searchDocs(query: \"supabase-js createClient auth signUp email password options data signInWithPassword getUser select single profile\", limit: 5) { nodes { ... on Guide { title href content } ... on ClientLibraryFunctionReference { title href content language methodName } } } }", "hasContent": true, "pages": [ { @@ -16594,23 +16855,67 @@ "title": "Migrate from Auth0 to Supabase Auth" }, { - "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", - "title": "Integrating With Supabase Auth" + "url": "https://supabase.com/docs/guides/auth/passwords", + "title": "Password-based Auth" }, { - "url": "https://supabase.com/docs/guides/functions/auth", - "title": "Securing Edge Functions" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-saml-sso", + "title": "Configure SAML SSO" }, { - "url": "https://supabase.com/docs/guides/auth/users", - "title": "Users" + "url": "https://supabase.com/docs/guides/auth", + "title": "Auth" }, { - "url": "https://supabase.com/docs/guides/auth/oauth-server/token-security", - "title": "Token Security and Row Level Security" + "url": "https://supabase.com/docs/guides/auth/auth-email-passwordless", + "title": "Passwordless email logins" + } + ], + "resultChars": 102589 + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"JavaScript signUp email password options data user metadata signInWithPassword select single\", limit: 10) { nodes { ... on ClientLibraryFunctionReference { title href content language methodName } ... on Guide { title href content } } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/reference/javascript/auth-signinwithpassword" + }, + { + "url": "https://supabase.com/docs/reference/dart/auth-signinwithpassword" + }, + { + "url": "https://supabase.com/docs/guides/auth/passwords", + "title": "Password-based Auth" + }, + { + "url": "https://supabase.com/docs/reference/swift/auth-signinwithpassword", + "title": "signInWithPassword()" + }, + { + "url": "https://supabase.com/docs/guides/auth/auth-email-passwordless", + "title": "Passwordless email logins" + }, + { + "url": "https://supabase.com/docs/reference/csharp/auth-signinwithpassword", + "title": "SignIn(email, password)" + }, + { + "url": "https://supabase.com/docs/reference/javascript/auth-signinwithpasskey" + }, + { + "url": "https://supabase.com/docs/reference/swift/auth-signinwithpasskey", + "title": "signInWithPasskey()" + }, + { + "url": "https://supabase.com/docs/guides/auth/passkeys", + "title": "Passkey authentication" + }, + { + "url": "https://supabase.com/docs/reference/dart/auth-signinwithpasskey" } ], - "resultChars": 53768 + "resultChars": 75745 } ] }, @@ -16998,31 +17303,31 @@ "calls": [ { "source": "search_docs", - "query": "query { searchDocs(query: \"Supabase REST API JavaScript fetch apikey Authorization service role secret key Range header pagination PostgREST\", limit: 5) { nodes { title href content } } }", + "query": "query { searchDocs(query: \"Supabase secret key REST API apikey Authorization header sb_secret backend\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/database/secure-data", - "title": "Securing your data" + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", + "title": "Migrating to publishable and secret API keys" }, { "url": "https://supabase.com/docs/guides/functions/auth", "title": "Securing Edge Functions" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", - "title": "New API Keys and Asymmetric Authentication" + "url": "https://supabase.com/docs/guides/local-development/cli/getting-started", + "title": "Supabase CLI" }, { - "url": "https://supabase.com/docs/guides/database/connecting-to-postgres/serverless-drivers", - "title": "Serverless Drivers" + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", + "title": "New API Keys and Asymmetric Authentication" }, { - "url": "https://supabase.com/docs/guides/api/handling-errors-in-supabase-js", - "title": "Handling errors in `supabase-js`" + "url": "https://supabase.com/docs/guides/api/creating-routes", + "title": "Creating API Routes" } ], - "resultChars": 39367 + "resultChars": 57476 } ] }, @@ -17031,6 +17336,124 @@ "attempts": 2, "sourcePath": "codex-gpt-5.6-no-skills/build-dataapi-001-relational-report.json" }, + { + "experiment": "codex-gpt-5.6-no-skills", + "experimentSuite": "no-skills", + "experimentDisplay": { + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-5.6-sol", + "reasoningEffort": "low" + }, + "eval": "build-dataapi-002-restock-alert-report", + "stage": "build", + "product": [ + "data-api", + "database" + ], + "topic": [ + "sdk" + ], + "suite": "benchmark", + "interface": "cli", + "cliVersion": "2.109.1", + "passed": false, + "checks": [ + { + "name": "report runs and prints JSON", + "passed": true, + "notes": "exit 0" + }, + { + "name": "alerts match the database (below threshold, sorted)", + "passed": true, + "notes": "expected [{\"warehouse\":\"North DC\",\"product\":\"Gizmo\",\"quantity\":3,\"reorderThreshold\":10,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"North DC\",\"product\":\"Widget\",\"quantity\":5,\"reorderThreshold\":20,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"South DC\",\"product\":\"Gadget\",\"quantity\":2,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"},{\"warehouse\":\"West DC\",\"product\":\"Gadget\",\"quantity\":0,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"}], got [{\"warehouse\":\"North DC\",\"product\":\"Gizmo\",\"quantity\":3,\"reorderThreshold\":10,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"North DC\",\"product\":\"Widget\",\"quantity\":5,\"reorderThreshold\":20,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"South DC\",\"product\":\"Gadget\",\"quantity\":2,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"},{\"warehouse\":\"West DC\",\"product\":\"Gadget\",\"quantity\":0,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"}]" + }, + { + "name": "tables stay locked down (publishable key reads nothing)", + "passed": true, + "notes": "publishable read errored: permission denied for table inventory" + }, + { + "name": "implementation uses @supabase/supabase-js", + "passed": false, + "notes": "no @supabase/supabase-js import found — this eval requires the SDK" + }, + { + "name": "report queries via the Data API, not raw SQL", + "passed": true, + "notes": "no psql / raw Postgres driver usage found" + } + ], + "skills": { + "available": [], + "loaded": [] + }, + "docs": { + "calls": [ + { + "source": "search_docs", + "query": "query { searchDocs(query: \"supabase-js nested select foreign key relationships inner join filtering related table service_role secret key createClient\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/reference/javascript/oauth-admin-createclient" + }, + { + "url": "https://supabase.com/docs/reference/javascript/oauth-admin-getclient" + }, + { + "url": "https://supabase.com/docs/guides/functions/connect-to-postgres", + "title": "Integrating with Supabase Database (Postgres)" + }, + { + "url": "https://supabase.com/docs/reference/javascript/oauth-admin-updateclient" + }, + { + "url": "https://supabase.com/docs/reference/javascript/oauth-admin-deleteclient" + } + ], + "resultChars": 8555 + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"secret key apikey Authorization header Supabase Data API REST server-side\", limit: 6) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/realtime/getting_started", + "title": "Getting Started with Realtime" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", + "title": "New API Keys and Asymmetric Authentication" + }, + { + "url": "https://supabase.com/docs/guides/functions/auth", + "title": "Securing Edge Functions" + }, + { + "url": "https://supabase.com/docs/guides/api/creating-routes", + "title": "Creating API Routes" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/api-keys", + "title": "Understanding API keys" + }, + { + "url": "https://supabase.com/docs/guides/auth/signing-keys", + "title": "JWT Signing Keys" + } + ], + "resultChars": 96290 + } + ] + }, + "prompt": "Purchasing needs a restock alert. `app/restock.mjs` has the spec in a\ncomment — it runs in our Node backend worker and prints a JSON list of what\nneeds reordering, with who to email about it.\n\nThe data lives in the Supabase project in `supabase/` (already running\nlocally). Finish the script and make sure it prints the right alerts.", + "promptSourcePath": "evals/build-dataapi-002-restock-alert-report/PROMPT.md", + "attempts": 2, + "sourcePath": "codex-gpt-5.6-no-skills/build-dataapi-002-restock-alert-report.json" + }, { "experiment": "codex-gpt-5.6-no-skills", "experimentSuite": "no-skills", diff --git a/apps/web/src/data/regression-eval-results.json b/apps/web/src/data/regression-eval-results.json index 623a3c21..cd9f18c1 100644 --- a/apps/web/src/data/regression-eval-results.json +++ b/apps/web/src/data/regression-eval-results.json @@ -539,36 +539,7 @@ ] }, "docs": { - "calls": [ - { - "source": "search_docs", - "query": "{ searchDocs(query: \"publishable and secret API keys migration from anon and service_role\", limit: 5) { nodes { title href ... on Guide { content } } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" - }, - { - "url": "https://supabase.com/docs/guides/auth/signing-keys", - "title": "JWT Signing Keys" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", - "title": "New API Keys and Asymmetric Authentication" - }, - { - "url": "https://supabase.com/docs/guides/integrations/supabase-for-platforms", - "title": "Supabase for Platforms" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-redwoodjs", - "title": "Build a User Management App with RedwoodJS" - } - ], - "resultChars": 107177 - } - ] + "calls": [] }, "prompt": "Heads-up from the platform team: the legacy JWT-based API keys (`anon` /\n`service_role`) are going away for our projects soon, in favor of the new\npublishable/secret keys. The little blog tooling app in `app/` still uses the\nlegacy keys.\n\nMigrate it over. Both scripts need to keep working — `npm run posts` and\n`npm run stats` (run them from `app/`). The local Supabase project in\n`supabase/` is already running.", "promptSourcePath": "evals/resolve-sdk-001-legacy-key-migration/PROMPT.md",