diff --git a/apps/web/src/data/eval-results.json b/apps/web/src/data/eval-results.json index cd6b8942..a4ec4315 100644 --- a/apps/web/src/data/eval-results.json +++ b/apps/web/src/data/eval-results.json @@ -383,6 +383,80 @@ "attempts": 1, "sourcePath": "claude-code-opus-4.8/build-functions-005-dual-auth-user-secret.json" }, + { + "experiment": "claude-code-opus-4.8", + "experimentSuite": "benchmark", + "experimentDisplay": { + "agent": "claude-code", + "modelProvider": "anthropic", + "modelId": "claude-opus-4-8", + "reasoningEffort": "high" + }, + "eval": "build-rls-004-approved-profiles-visibility", + "stage": "build", + "product": [ + "database", + "auth" + ], + "topic": [ + "rls", + "security" + ], + "suite": "benchmark", + "interface": "mcp", + "passed": false, + "checks": [ + { + "name": "RLS enabled on profiles", + "passed": true + }, + { + "name": "user A can select own profile", + "passed": true + }, + { + "name": "user A can update own profile", + "passed": true + }, + { + "name": "unapproved user C can still select own profile", + "passed": true + }, + { + "name": "unapproved user C can still update own profile", + "passed": true + }, + { + "name": "unrelated user B cannot select user A's approved profile", + "passed": false + }, + { + "name": "unrelated user A cannot select user B's approved profile", + "passed": false + }, + { + "name": "user B cannot update user A's profile", + "passed": true + }, + { + "name": "user B cannot delete user A's profile", + "passed": true + } + ], + "skills": { + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase" + ] + }, + "prompt": "Add RLS policies to our profiles table. It's for a meetup app: users should be able to see approved profiles to find people to meet up with, and manage their own.", + "promptSourcePath": "evals/build-rls-004-approved-profiles-visibility/PROMPT.md", + "attempts": 2, + "sourcePath": "claude-code-opus-4.8/build-rls-004-approved-profiles-visibility.json" + }, { "experiment": "claude-code-opus-4.8", "experimentSuite": "benchmark", @@ -1533,6 +1607,75 @@ "attempts": 2, "sourcePath": "claude-code-opus-4.8-no-skills/build-functions-005-dual-auth-user-secret.json" }, + { + "experiment": "claude-code-opus-4.8-no-skills", + "experimentSuite": "no-skills", + "experimentDisplay": { + "agent": "claude-code", + "modelProvider": "anthropic", + "modelId": "claude-opus-4-8", + "reasoningEffort": "high" + }, + "eval": "build-rls-004-approved-profiles-visibility", + "stage": "build", + "product": [ + "database", + "auth" + ], + "topic": [ + "rls", + "security" + ], + "suite": "benchmark", + "interface": "mcp", + "passed": false, + "checks": [ + { + "name": "RLS enabled on profiles", + "passed": true + }, + { + "name": "user A can select own profile", + "passed": true + }, + { + "name": "user A can update own profile", + "passed": true + }, + { + "name": "unapproved user C can still select own profile", + "passed": true + }, + { + "name": "unapproved user C can still update own profile", + "passed": true + }, + { + "name": "unrelated user B cannot select user A's approved profile", + "passed": false + }, + { + "name": "unrelated user A cannot select user B's approved profile", + "passed": false + }, + { + "name": "user B cannot update user A's profile", + "passed": true + }, + { + "name": "user B cannot delete user A's profile", + "passed": true + } + ], + "skills": { + "available": [], + "loaded": [] + }, + "prompt": "Add RLS policies to our profiles table. It's for a meetup app: users should be able to see approved profiles to find people to meet up with, and manage their own.", + "promptSourcePath": "evals/build-rls-004-approved-profiles-visibility/PROMPT.md", + "attempts": 2, + "sourcePath": "claude-code-opus-4.8-no-skills/build-rls-004-approved-profiles-visibility.json" + }, { "experiment": "claude-code-opus-4.8-no-skills", "experimentSuite": "no-skills", @@ -2648,6 +2791,78 @@ "attempts": 1, "sourcePath": "claude-code-sonnet-5/build-functions-005-dual-auth-user-secret.json" }, + { + "experiment": "claude-code-sonnet-5", + "experimentSuite": "benchmark", + "experimentDisplay": { + "agent": "claude-code", + "modelProvider": "anthropic", + "modelId": "claude-sonnet-5", + "reasoningEffort": "high" + }, + "eval": "build-rls-004-approved-profiles-visibility", + "stage": "build", + "product": [ + "database", + "auth" + ], + "topic": [ + "rls", + "security" + ], + "suite": "benchmark", + "interface": "mcp", + "passed": false, + "checks": [ + { + "name": "RLS enabled on profiles", + "passed": true + }, + { + "name": "user A can select own profile", + "passed": false + }, + { + "name": "user A can update own profile", + "passed": false + }, + { + "name": "unapproved user C can still select own profile", + "passed": false + }, + { + "name": "unapproved user C can still update own profile", + "passed": false + }, + { + "name": "unrelated user B cannot select user A's approved profile", + "passed": true + }, + { + "name": "unrelated user A cannot select user B's approved profile", + "passed": true + }, + { + "name": "user B cannot update user A's profile", + "passed": true + }, + { + "name": "user B cannot delete user A's profile", + "passed": true + } + ], + "skills": { + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [] + }, + "prompt": "Add RLS policies to our profiles table. It's for a meetup app: users should be able to see approved profiles to find people to meet up with, and manage their own.", + "promptSourcePath": "evals/build-rls-004-approved-profiles-visibility/PROMPT.md", + "attempts": 2, + "sourcePath": "claude-code-sonnet-5/build-rls-004-approved-profiles-visibility.json" + }, { "experiment": "claude-code-sonnet-5", "experimentSuite": "benchmark", @@ -3798,6 +4013,75 @@ "attempts": 2, "sourcePath": "claude-code-sonnet-5-no-skills/build-functions-005-dual-auth-user-secret.json" }, + { + "experiment": "claude-code-sonnet-5-no-skills", + "experimentSuite": "no-skills", + "experimentDisplay": { + "agent": "claude-code", + "modelProvider": "anthropic", + "modelId": "claude-sonnet-5", + "reasoningEffort": "high" + }, + "eval": "build-rls-004-approved-profiles-visibility", + "stage": "build", + "product": [ + "database", + "auth" + ], + "topic": [ + "rls", + "security" + ], + "suite": "benchmark", + "interface": "mcp", + "passed": false, + "checks": [ + { + "name": "RLS enabled on profiles", + "passed": true + }, + { + "name": "user A can select own profile", + "passed": false + }, + { + "name": "user A can update own profile", + "passed": false + }, + { + "name": "unapproved user C can still select own profile", + "passed": false + }, + { + "name": "unapproved user C can still update own profile", + "passed": false + }, + { + "name": "unrelated user B cannot select user A's approved profile", + "passed": true + }, + { + "name": "unrelated user A cannot select user B's approved profile", + "passed": true + }, + { + "name": "user B cannot update user A's profile", + "passed": true + }, + { + "name": "user B cannot delete user A's profile", + "passed": true + } + ], + "skills": { + "available": [], + "loaded": [] + }, + "prompt": "Add RLS policies to our profiles table. It's for a meetup app: users should be able to see approved profiles to find people to meet up with, and manage their own.", + "promptSourcePath": "evals/build-rls-004-approved-profiles-visibility/PROMPT.md", + "attempts": 2, + "sourcePath": "claude-code-sonnet-5-no-skills/build-rls-004-approved-profiles-visibility.json" + }, { "experiment": "claude-code-sonnet-5-no-skills", "experimentSuite": "no-skills", @@ -4914,6 +5198,80 @@ "attempts": 2, "sourcePath": "codex-gpt-5.4-mini/build-functions-005-dual-auth-user-secret.json" }, + { + "experiment": "codex-gpt-5.4-mini", + "experimentSuite": "benchmark", + "experimentDisplay": { + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-5.4-mini", + "reasoningEffort": "medium" + }, + "eval": "build-rls-004-approved-profiles-visibility", + "stage": "build", + "product": [ + "database", + "auth" + ], + "topic": [ + "rls", + "security" + ], + "suite": "benchmark", + "interface": "mcp", + "passed": false, + "checks": [ + { + "name": "RLS enabled on profiles", + "passed": true + }, + { + "name": "user A can select own profile", + "passed": true + }, + { + "name": "user A can update own profile", + "passed": true + }, + { + "name": "unapproved user C can still select own profile", + "passed": true + }, + { + "name": "unapproved user C can still update own profile", + "passed": true + }, + { + "name": "unrelated user B cannot select user A's approved profile", + "passed": false + }, + { + "name": "unrelated user A cannot select user B's approved profile", + "passed": false + }, + { + "name": "user B cannot update user A's profile", + "passed": true + }, + { + "name": "user B cannot delete user A's profile", + "passed": true + } + ], + "skills": { + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase" + ] + }, + "prompt": "Add RLS policies to our profiles table. It's for a meetup app: users should be able to see approved profiles to find people to meet up with, and manage their own.", + "promptSourcePath": "evals/build-rls-004-approved-profiles-visibility/PROMPT.md", + "attempts": 2, + "sourcePath": "codex-gpt-5.4-mini/build-rls-004-approved-profiles-visibility.json" + }, { "experiment": "codex-gpt-5.4-mini", "experimentSuite": "benchmark", @@ -6071,6 +6429,75 @@ "attempts": 2, "sourcePath": "codex-gpt-5.4-mini-no-skills/build-functions-005-dual-auth-user-secret.json" }, + { + "experiment": "codex-gpt-5.4-mini-no-skills", + "experimentSuite": "no-skills", + "experimentDisplay": { + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-5.4-mini", + "reasoningEffort": "medium" + }, + "eval": "build-rls-004-approved-profiles-visibility", + "stage": "build", + "product": [ + "database", + "auth" + ], + "topic": [ + "rls", + "security" + ], + "suite": "benchmark", + "interface": "mcp", + "passed": false, + "checks": [ + { + "name": "RLS enabled on profiles", + "passed": true + }, + { + "name": "user A can select own profile", + "passed": true + }, + { + "name": "user A can update own profile", + "passed": true + }, + { + "name": "unapproved user C can still select own profile", + "passed": true + }, + { + "name": "unapproved user C can still update own profile", + "passed": true + }, + { + "name": "unrelated user B cannot select user A's approved profile", + "passed": false + }, + { + "name": "unrelated user A cannot select user B's approved profile", + "passed": false + }, + { + "name": "user B cannot update user A's profile", + "passed": true + }, + { + "name": "user B cannot delete user A's profile", + "passed": true + } + ], + "skills": { + "available": [], + "loaded": [] + }, + "prompt": "Add RLS policies to our profiles table. It's for a meetup app: users should be able to see approved profiles to find people to meet up with, and manage their own.", + "promptSourcePath": "evals/build-rls-004-approved-profiles-visibility/PROMPT.md", + "attempts": 2, + "sourcePath": "codex-gpt-5.4-mini-no-skills/build-rls-004-approved-profiles-visibility.json" + }, { "experiment": "codex-gpt-5.4-mini-no-skills", "experimentSuite": "no-skills", @@ -7189,6 +7616,81 @@ "attempts": 2, "sourcePath": "codex-gpt-5.5/build-functions-005-dual-auth-user-secret.json" }, + { + "experiment": "codex-gpt-5.5", + "experimentSuite": "benchmark", + "experimentDisplay": { + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-5.5", + "reasoningEffort": "medium" + }, + "eval": "build-rls-004-approved-profiles-visibility", + "stage": "build", + "product": [ + "database", + "auth" + ], + "topic": [ + "rls", + "security" + ], + "suite": "benchmark", + "interface": "mcp", + "passed": false, + "checks": [ + { + "name": "RLS enabled on profiles", + "passed": true + }, + { + "name": "user A can select own profile", + "passed": true + }, + { + "name": "user A can update own profile", + "passed": true + }, + { + "name": "unapproved user C can still select own profile", + "passed": true + }, + { + "name": "unapproved user C can still update own profile", + "passed": true + }, + { + "name": "unrelated user B cannot select user A's approved profile", + "passed": false + }, + { + "name": "unrelated user A cannot select user B's approved profile", + "passed": false + }, + { + "name": "user B cannot update user A's profile", + "passed": true + }, + { + "name": "user B cannot delete user A's profile", + "passed": true + } + ], + "skills": { + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase", + "supabase-postgres-best-practices" + ] + }, + "prompt": "Add RLS policies to our profiles table. It's for a meetup app: users should be able to see approved profiles to find people to meet up with, and manage their own.", + "promptSourcePath": "evals/build-rls-004-approved-profiles-visibility/PROMPT.md", + "attempts": 2, + "sourcePath": "codex-gpt-5.5/build-rls-004-approved-profiles-visibility.json" + }, { "experiment": "codex-gpt-5.5", "experimentSuite": "benchmark", @@ -8343,6 +8845,75 @@ "attempts": 2, "sourcePath": "codex-gpt-5.5-no-skills/build-functions-005-dual-auth-user-secret.json" }, + { + "experiment": "codex-gpt-5.5-no-skills", + "experimentSuite": "no-skills", + "experimentDisplay": { + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-5.5", + "reasoningEffort": "medium" + }, + "eval": "build-rls-004-approved-profiles-visibility", + "stage": "build", + "product": [ + "database", + "auth" + ], + "topic": [ + "rls", + "security" + ], + "suite": "benchmark", + "interface": "mcp", + "passed": false, + "checks": [ + { + "name": "RLS enabled on profiles", + "passed": true + }, + { + "name": "user A can select own profile", + "passed": true + }, + { + "name": "user A can update own profile", + "passed": true + }, + { + "name": "unapproved user C can still select own profile", + "passed": true + }, + { + "name": "unapproved user C can still update own profile", + "passed": true + }, + { + "name": "unrelated user B cannot select user A's approved profile", + "passed": false + }, + { + "name": "unrelated user A cannot select user B's approved profile", + "passed": false + }, + { + "name": "user B cannot update user A's profile", + "passed": true + }, + { + "name": "user B cannot delete user A's profile", + "passed": true + } + ], + "skills": { + "available": [], + "loaded": [] + }, + "prompt": "Add RLS policies to our profiles table. It's for a meetup app: users should be able to see approved profiles to find people to meet up with, and manage their own.", + "promptSourcePath": "evals/build-rls-004-approved-profiles-visibility/PROMPT.md", + "attempts": 2, + "sourcePath": "codex-gpt-5.5-no-skills/build-rls-004-approved-profiles-visibility.json" + }, { "experiment": "codex-gpt-5.5-no-skills", "experimentSuite": "no-skills", diff --git a/evals/build-rls-004-approved-profiles-visibility/EVAL.ts b/evals/build-rls-004-approved-profiles-visibility/EVAL.ts new file mode 100644 index 00000000..ec008265 --- /dev/null +++ b/evals/build-rls-004-approved-profiles-visibility/EVAL.ts @@ -0,0 +1,144 @@ +import type { CheckResult, SupabaseClient, ToolScorer } from "@supabase-evals/core"; +import { stripIndent } from "common-tags"; + +const PASSWORD = "secret123"; + +const scorer: ToolScorer = async (ctx) => { + try { + const userA = await signUpUser(ctx.client, "profiles-a@example.com"); + if ("failure" in userA) return { passed: false, checks: [userA.failure] }; + const userB = await signUpUser(ctx.getClient(), "profiles-b@example.com"); + if ("failure" in userB) return { passed: false, checks: [userB.failure] }; + const userC = await signUpUser(ctx.getClient(), "profiles-c@example.com"); + if ("failure" in userC) return { passed: false, checks: [userC.failure] }; + + const clientA = userA.client; + const clientB = userB.client; + const clientC = userC.client; + + // SET ROLE service_role so a policy-protected is_approved column (e.g. a trigger that + // only lets service_role set it) doesn't silently reset our seeded approval state. + await ctx.query(stripIndent` + SET ROLE service_role; + INSERT INTO profiles (user_id, display_name, phone_number, is_approved) VALUES + ('${userA.id}', 'Alice', '555-0101', true), + ('${userB.id}', 'Bob', '555-0102', true), + ('${userC.id}', 'Cara', '555-0103', false); + RESET ROLE; + `); + + const { rows: seeded } = await ctx.query(stripIndent` + SELECT user_id, is_approved FROM profiles + WHERE user_id IN ('${userA.id}', '${userB.id}', '${userC.id}'); + `); + const seededCorrectly = + seeded.find((r) => r.user_id === userA.id)?.is_approved === true && + seeded.find((r) => r.user_id === userB.id)?.is_approved === true && + seeded.find((r) => r.user_id === userC.id)?.is_approved === false; + if (!seededCorrectly) { + return { + passed: false, + checks: [ + { + name: "seed data has expected approval states", + passed: false, + notes: `unexpected is_approved values after seeding: ${JSON.stringify(seeded)}`, + }, + ], + }; + } + + const { rows: rls } = await ctx.query( + `SELECT relrowsecurity FROM pg_class WHERE relname = 'profiles';`, + ); + + const aOwnSelect = await clientA.from("profiles").select("id").eq("user_id", userA.id); + const aOwnUpdate = await clientA + .from("profiles") + .update({ bio: "hello" }) + .eq("user_id", userA.id) + .select("bio"); + const cOwnSelect = await clientC.from("profiles").select("id").eq("user_id", userC.id); + const cOwnUpdate = await clientC + .from("profiles") + .update({ bio: "not approved yet" }) + .eq("user_id", userC.id) + .select("bio"); + const bReadsA = await clientB.from("profiles").select("id").eq("user_id", userA.id); + const aReadsB = await clientA.from("profiles").select("id").eq("user_id", userB.id); + const bUpdatesA = await clientB + .from("profiles") + .update({ display_name: "hijacked" }) + .eq("user_id", userA.id) + .select("id"); + const bDeletesA = await clientB.from("profiles").delete().eq("user_id", userA.id).select("id"); + + const checks: CheckResult[] = [ + { name: "RLS enabled on profiles", passed: rls[0]?.relrowsecurity === true }, + { + name: "user A can select own profile", + passed: !aOwnSelect.error && aOwnSelect.data?.length === 1, + notes: aOwnSelect.error?.message, + }, + { + name: "user A can update own profile", + passed: !aOwnUpdate.error && aOwnUpdate.data?.[0]?.bio === "hello", + notes: aOwnUpdate.error?.message, + }, + { + name: "unapproved user C can still select own profile", + passed: !cOwnSelect.error && cOwnSelect.data?.length === 1, + notes: cOwnSelect.error?.message, + }, + { + name: "unapproved user C can still update own profile", + passed: !cOwnUpdate.error && cOwnUpdate.data?.[0]?.bio === "not approved yet", + notes: cOwnUpdate.error?.message, + }, + { + name: "unrelated user B cannot select user A's approved profile", + passed: Boolean(bReadsA.error) || bReadsA.data?.length === 0, + }, + { + name: "unrelated user A cannot select user B's approved profile", + passed: Boolean(aReadsB.error) || aReadsB.data?.length === 0, + }, + { + name: "user B cannot update user A's profile", + passed: Boolean(bUpdatesA.error) || !bUpdatesA.data || bUpdatesA.data.length === 0, + }, + { + name: "user B cannot delete user A's profile", + passed: Boolean(bDeletesA.error) || !bDeletesA.data || bDeletesA.data.length === 0, + }, + ]; + + return { passed: checks.every((check) => check.passed), checks }; + } catch (error) { + const msg = error instanceof Error ? error.message : String(error); + return { + passed: false, + checks: [{ name: "scorer evaluated profiles RLS behavior", passed: false, notes: msg }], + }; + } +}; + +export default scorer; + +/** Signs up a fresh auth user on the given client so tests can exercise per-user RLS. */ +async function signUpUser( + client: SupabaseClient, + email: string, +): Promise<{ client: SupabaseClient; id: string } | { failure: CheckResult }> { + const { data, error } = await client.auth.signUp({ email, password: PASSWORD }); + if (error || !data.user?.id || !data.session) { + return { + failure: { + name: `created auth session for ${email}`, + passed: false, + notes: error?.message ?? "missing session", + }, + }; + } + return { client, id: data.user.id }; +} diff --git a/evals/build-rls-004-approved-profiles-visibility/PROMPT.md b/evals/build-rls-004-approved-profiles-visibility/PROMPT.md new file mode 100644 index 00000000..684057a7 --- /dev/null +++ b/evals/build-rls-004-approved-profiles-visibility/PROMPT.md @@ -0,0 +1,14 @@ +--- +stage: build +suite: benchmark +interface: mcp +product: + - database + - auth +topic: + - rls + - security +motivation: AI-756, AI-676 +--- + +Add RLS policies to our profiles table. It's for a meetup app: users should be able to see approved profiles to find people to meet up with, and manage their own. diff --git a/evals/build-rls-004-approved-profiles-visibility/remote/project.sql b/evals/build-rls-004-approved-profiles-visibility/remote/project.sql new file mode 100644 index 00000000..3720187b --- /dev/null +++ b/evals/build-rls-004-approved-profiles-visibility/remote/project.sql @@ -0,0 +1,13 @@ +CREATE TABLE profiles ( + id uuid PRIMARY KEY DEFAULT gen_random_uuid(), + user_id uuid NOT NULL, + display_name text, + bio text, + phone_number text, + is_approved boolean NOT NULL DEFAULT false +); + +ALTER TABLE profiles ENABLE ROW LEVEL SECURITY; + +GRANT SELECT, INSERT, UPDATE, DELETE ON profiles TO authenticated; +GRANT ALL ON profiles TO service_role; diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index 0ccdbc3d..6de441b7 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -245,7 +245,13 @@ export interface ToolScoringContext { client: SupabaseClient; /** Create a fresh independent Supabase client (useful for multi-user RLS tests). */ getClient: () => SupabaseClient; - /** Run a SQL query in-process against the project database. */ + /** + * Run a SQL query in-process against the project database, as the + * `postgres` superuser (bypasses RLS and grants, but not triggers). If a + * table gates a column on `current_setting('role') = 'service_role'` (a + * common anti-self-approval pattern), seed writes to it need + * `SET ROLE service_role` first or the write can be silently altered. + */ query: (sql: string) => Promise<{ rows: Record[] }>; /** Invoke a deployed edge function in-process. */ invokeFunction: (