diff --git a/.gitignore b/.gitignore index 932a6aea..23951d46 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,8 @@ node_modules/ dist/ .env .env*.local +# eval seed data may include a .env with well-known local demo keys +!evals/*/local/**/.env .DS_Store results/*/ .sync-tmp/ diff --git a/apps/web/src/data/eval-results.json b/apps/web/src/data/eval-results.json index 556790c2..6db3f02d 100644 --- a/apps/web/src/data/eval-results.json +++ b/apps/web/src/data/eval-results.json @@ -1,4 +1,86 @@ [ + { + "experiment": "claude-code-opus-4.8", + "experimentSuite": "benchmark", + "experimentDisplay": { + "agent": "claude-code", + "modelProvider": "anthropic", + "modelId": "claude-opus-4-8", + "reasoningEffort": "high" + }, + "eval": "build-auth-001-email-password-flow", + "stage": "build", + "product": [ + "auth", + "database" + ], + "topic": [ + "sdk", + "rls" + ], + "suite": "benchmark", + "interface": "cli", + "cliVersion": "2.109.1", + "passed": true, + "checks": [ + { + "name": "auth module loads and the driver completes", + "passed": true, + "notes": "driver produced a result" + }, + { + "name": "signUp creates the account and returns its user id", + "passed": true, + "notes": "db user f1727066-99fb-489f-8ae7-544261e522ca, signUp returned {\"userId\":\"f1727066-99fb-489f-8ae7-544261e522ca\"}" + }, + { + "name": "signup metadata reaches the profile (display name)", + "passed": true, + "notes": "profiles.display_name = \"Alex Doe\"" + }, + { + "name": "wrong password is rejected gracefully (no throw, no session)", + "passed": true, + "notes": "{\"error\":\"Invalid login credentials\"}" + }, + { + "name": "signIn with the right password returns the user id", + "passed": true, + "notes": "{\"userId\":\"f1727066-99fb-489f-8ae7-544261e522ca\"}" + }, + { + "name": "getMyProfile returns the signed-in user's profile", + "passed": true, + "notes": "{\"displayName\":\"Alex Doe\",\"plan\":\"free\"}" + }, + { + "name": "app code does not use the secret / service-role key", + "passed": true, + "notes": "no secret-key references found" + }, + { + "name": "implementation uses @supabase/supabase-js", + "passed": true, + "notes": "imports found in: app/src/auth.mjs" + } + ], + "skills": { + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase" + ] + }, + "docs": { + "calls": [] + }, + "prompt": "Our app in `app/` needs accounts. Wire up `app/src/auth.mjs` — the stubs in\nthere describe what each function should do. People sign up with an email,\npassword, and display name, sign back in later, and the app greets them with\ntheir profile.\n\nThe Supabase project for this app is in `supabase/` and already running\nlocally. When you're done, the functions should work for real against it.", + "promptSourcePath": "evals/build-auth-001-email-password-flow/PROMPT.md", + "attempts": 1, + "sourcePath": "claude-code-opus-4.8/build-auth-001-email-password-flow.json" + }, { "experiment": "claude-code-opus-4.8", "experimentSuite": "benchmark", @@ -242,6 +324,138 @@ "attempts": 1, "sourcePath": "claude-code-opus-4.8/build-cli-003-pg-cron-queue-workflow.json" }, + { + "experiment": "claude-code-opus-4.8", + "experimentSuite": "benchmark", + "experimentDisplay": { + "agent": "claude-code", + "modelProvider": "anthropic", + "modelId": "claude-opus-4-8", + "reasoningEffort": "high" + }, + "eval": "build-dataapi-001-relational-report", + "stage": "build", + "product": [ + "data-api", + "database" + ], + "topic": [ + "sdk" + ], + "suite": "benchmark", + "interface": "cli", + "cliVersion": "2.109.1", + "passed": true, + "checks": [ + { + "name": "report runs and prints JSON", + "passed": true, + "notes": "exit 0" + }, + { + "name": "report numbers match the database (per customer, sorted)", + "passed": true, + "notes": "expected [{\"customer\":\"Ada Lovelace\",\"orderCount\":2,\"totalCents\":41900,\"topProduct\":\"Keyboard\"},{\"customer\":\"Grace Hopper\",\"orderCount\":2,\"totalCents\":15600,\"topProduct\":\"Cable\"},{\"customer\":\"Linus Pauling\",\"orderCount\":1,\"totalCents\":66500,\"topProduct\":\"Monitor\"}], got [{\"customer\":\"Ada Lovelace\",\"orderCount\":2,\"totalCents\":41900,\"topProduct\":\"Keyboard\"},{\"customer\":\"Grace Hopper\",\"orderCount\":2,\"totalCents\":15600,\"topProduct\":\"Cable\"},{\"customer\":\"Linus Pauling\",\"orderCount\":1,\"totalCents\":66500,\"topProduct\":\"Monitor\"}]" + }, + { + "name": "tables stay locked down (publishable key reads nothing)", + "passed": true, + "notes": "publishable read errored: permission denied for table customers" + }, + { + "name": "implementation uses @supabase/supabase-js", + "passed": true, + "notes": "imports found in: app/report.mjs" + }, + { + "name": "report queries via the Data API, not raw SQL", + "passed": true, + "notes": "no psql / raw Postgres driver usage found" + } + ], + "skills": { + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase" + ] + }, + "docs": { + "calls": [] + }, + "prompt": "We need the nightly sales report working. `app/report.mjs` has the spec in a\ncomment — it runs in our Node backend worker and prints a JSON summary of what\neach customer has ordered.\n\nThe data lives in the Supabase project in `supabase/` (already running\nlocally). Finish the script and make sure it prints the right numbers.", + "promptSourcePath": "evals/build-dataapi-001-relational-report/PROMPT.md", + "attempts": 2, + "sourcePath": "claude-code-opus-4.8/build-dataapi-001-relational-report.json" + }, + { + "experiment": "claude-code-opus-4.8", + "experimentSuite": "benchmark", + "experimentDisplay": { + "agent": "claude-code", + "modelProvider": "anthropic", + "modelId": "claude-opus-4-8", + "reasoningEffort": "high" + }, + "eval": "build-dataapi-002-restock-alert-report", + "stage": "build", + "product": [ + "data-api", + "database" + ], + "topic": [ + "sdk" + ], + "suite": "benchmark", + "interface": "cli", + "cliVersion": "2.109.1", + "passed": false, + "checks": [ + { + "name": "report runs and prints JSON", + "passed": true, + "notes": "exit 0" + }, + { + "name": "alerts match the database (below threshold, sorted)", + "passed": true, + "notes": "expected [{\"warehouse\":\"North DC\",\"product\":\"Gizmo\",\"quantity\":3,\"reorderThreshold\":10,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"North DC\",\"product\":\"Widget\",\"quantity\":5,\"reorderThreshold\":20,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"South DC\",\"product\":\"Gadget\",\"quantity\":2,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"},{\"warehouse\":\"West DC\",\"product\":\"Gadget\",\"quantity\":0,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"}], got [{\"warehouse\":\"North DC\",\"product\":\"Gizmo\",\"quantity\":3,\"reorderThreshold\":10,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"North DC\",\"product\":\"Widget\",\"quantity\":5,\"reorderThreshold\":20,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"South DC\",\"product\":\"Gadget\",\"quantity\":2,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"},{\"warehouse\":\"West DC\",\"product\":\"Gadget\",\"quantity\":0,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"}]" + }, + { + "name": "tables stay locked down (publishable key reads nothing)", + "passed": true, + "notes": "publishable read errored: permission denied for table inventory" + }, + { + "name": "implementation uses @supabase/supabase-js", + "passed": false, + "notes": "no @supabase/supabase-js import found — this eval requires the SDK" + }, + { + "name": "report queries via the Data API, not raw SQL", + "passed": true, + "notes": "no psql / raw Postgres driver usage found" + } + ], + "skills": { + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase" + ] + }, + "docs": { + "calls": [] + }, + "prompt": "Purchasing needs a restock alert. `app/restock.mjs` has the spec in a\ncomment — it runs in our Node backend worker and prints a JSON list of what\nneeds reordering, with who to email about it.\n\nThe data lives in the Supabase project in `supabase/` (already running\nlocally). Finish the script and make sure it prints the right alerts.", + "promptSourcePath": "evals/build-dataapi-002-restock-alert-report/PROMPT.md", + "attempts": 2, + "sourcePath": "claude-code-opus-4.8/build-dataapi-002-restock-alert-report.json" + }, { "experiment": "claude-code-opus-4.8", "experimentSuite": "benchmark", @@ -1486,6 +1700,83 @@ "attempts": 1, "sourcePath": "claude-code-opus-4.8/resolve-security-002-rls-cross-tenant-leak.json" }, + { + "experiment": "claude-code-opus-4.8-no-skills", + "experimentSuite": "no-skills", + "experimentDisplay": { + "agent": "claude-code", + "modelProvider": "anthropic", + "modelId": "claude-opus-4-8", + "reasoningEffort": "high" + }, + "eval": "build-auth-001-email-password-flow", + "stage": "build", + "product": [ + "auth", + "database" + ], + "topic": [ + "sdk", + "rls" + ], + "suite": "benchmark", + "interface": "cli", + "cliVersion": "2.109.1", + "passed": true, + "checks": [ + { + "name": "auth module loads and the driver completes", + "passed": true, + "notes": "driver produced a result" + }, + { + "name": "signUp creates the account and returns its user id", + "passed": true, + "notes": "db user 463babdd-b70c-4d3e-ac9c-e0bc7a076fd7, signUp returned {\"userId\":\"463babdd-b70c-4d3e-ac9c-e0bc7a076fd7\"}" + }, + { + "name": "signup metadata reaches the profile (display name)", + "passed": true, + "notes": "profiles.display_name = \"Alex Doe\"" + }, + { + "name": "wrong password is rejected gracefully (no throw, no session)", + "passed": true, + "notes": "{\"error\":\"Invalid login credentials\"}" + }, + { + "name": "signIn with the right password returns the user id", + "passed": true, + "notes": "{\"userId\":\"463babdd-b70c-4d3e-ac9c-e0bc7a076fd7\"}" + }, + { + "name": "getMyProfile returns the signed-in user's profile", + "passed": true, + "notes": "{\"displayName\":\"Alex Doe\",\"plan\":\"free\"}" + }, + { + "name": "app code does not use the secret / service-role key", + "passed": true, + "notes": "no secret-key references found" + }, + { + "name": "implementation uses @supabase/supabase-js", + "passed": true, + "notes": "imports found in: app/src/auth.mjs" + } + ], + "skills": { + "available": [], + "loaded": [] + }, + "docs": { + "calls": [] + }, + "prompt": "Our app in `app/` needs accounts. Wire up `app/src/auth.mjs` — the stubs in\nthere describe what each function should do. People sign up with an email,\npassword, and display name, sign back in later, and the app greets them with\ntheir profile.\n\nThe Supabase project for this app is in `supabase/` and already running\nlocally. When you're done, the functions should work for real against it.", + "promptSourcePath": "evals/build-auth-001-email-password-flow/PROMPT.md", + "attempts": 1, + "sourcePath": "claude-code-opus-4.8-no-skills/build-auth-001-email-password-flow.json" + }, { "experiment": "claude-code-opus-4.8-no-skills", "experimentSuite": "no-skills", @@ -1691,37 +1982,44 @@ "modelId": "claude-opus-4-8", "reasoningEffort": "high" }, - "eval": "build-database-001-migrate-postgres-to-supabase", + "eval": "build-dataapi-001-relational-report", "stage": "build", "product": [ + "data-api", "database" ], "topic": [ - "migrations" + "sdk" ], "suite": "benchmark", "interface": "cli", + "cliVersion": "2.109.1", "passed": true, "checks": [ { - "name": "all 3 tables exist (teams, members, tasks)", - "passed": true + "name": "report runs and prints JSON", + "passed": true, + "notes": "exit 0" }, { - "name": "row counts match (teams=5, members=10, tasks=13)", - "passed": true + "name": "report numbers match the database (per customer, sorted)", + "passed": true, + "notes": "expected [{\"customer\":\"Ada Lovelace\",\"orderCount\":2,\"totalCents\":41900,\"topProduct\":\"Keyboard\"},{\"customer\":\"Grace Hopper\",\"orderCount\":2,\"totalCents\":15600,\"topProduct\":\"Cable\"},{\"customer\":\"Linus Pauling\",\"orderCount\":1,\"totalCents\":66500,\"topProduct\":\"Monitor\"}], got [{\"customer\":\"Ada Lovelace\",\"orderCount\":2,\"totalCents\":41900,\"topProduct\":\"Keyboard\"},{\"customer\":\"Grace Hopper\",\"orderCount\":2,\"totalCents\":15600,\"topProduct\":\"Cable\"},{\"customer\":\"Linus Pauling\",\"orderCount\":1,\"totalCents\":66500,\"topProduct\":\"Monitor\"}]" }, { - "name": "foreign key constraints survived the restore", - "passed": true + "name": "tables stay locked down (publishable key reads nothing)", + "passed": true, + "notes": "publishable read errored: permission denied for table customers" }, { - "name": "tasks_team_status_idx index survived the restore", - "passed": true + "name": "implementation uses @supabase/supabase-js", + "passed": true, + "notes": "imports found in: app/report.mjs" }, { - "name": "sequences synced (next insert won't conflict with existing IDs)", - "passed": true + "name": "report queries via the Data API, not raw SQL", + "passed": true, + "notes": "no psql / raw Postgres driver usage found" } ], "skills": { @@ -1731,10 +2029,10 @@ "docs": { "calls": [] }, - "prompt": "I have an existing Postgres database I want to migrate to Supabase. There's a binary dump at `source.dump` in the current directory.\n\nCan you set up a local Supabase project and restore the dump into it?", - "promptSourcePath": "evals/build-database-001-migrate-postgres-to-supabase/PROMPT.md", - "attempts": 2, - "sourcePath": "claude-code-opus-4.8-no-skills/build-database-001-migrate-postgres-to-supabase.json" + "prompt": "We need the nightly sales report working. `app/report.mjs` has the spec in a\ncomment — it runs in our Node backend worker and prints a JSON summary of what\neach customer has ordered.\n\nThe data lives in the Supabase project in `supabase/` (already running\nlocally). Finish the script and make sure it prints the right numbers.", + "promptSourcePath": "evals/build-dataapi-001-relational-report/PROMPT.md", + "attempts": 1, + "sourcePath": "claude-code-opus-4.8-no-skills/build-dataapi-001-relational-report.json" }, { "experiment": "claude-code-opus-4.8-no-skills", @@ -1745,21 +2043,136 @@ "modelId": "claude-opus-4-8", "reasoningEffort": "high" }, - "eval": "build-functions-004-service-role-bypass", + "eval": "build-dataapi-002-restock-alert-report", "stage": "build", "product": [ - "edge-functions", - "auth", + "data-api", "database" ], "topic": [ - "rls", - "security", "sdk" ], "suite": "benchmark", - "interface": "mcp", - "passed": true, + "interface": "cli", + "cliVersion": "2.109.1", + "passed": false, + "checks": [ + { + "name": "report runs and prints JSON", + "passed": true, + "notes": "exit 0" + }, + { + "name": "alerts match the database (below threshold, sorted)", + "passed": true, + "notes": "expected [{\"warehouse\":\"North DC\",\"product\":\"Gizmo\",\"quantity\":3,\"reorderThreshold\":10,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"North DC\",\"product\":\"Widget\",\"quantity\":5,\"reorderThreshold\":20,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"South DC\",\"product\":\"Gadget\",\"quantity\":2,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"},{\"warehouse\":\"West DC\",\"product\":\"Gadget\",\"quantity\":0,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"}], got [{\"warehouse\":\"North DC\",\"product\":\"Gizmo\",\"quantity\":3,\"reorderThreshold\":10,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"North DC\",\"product\":\"Widget\",\"quantity\":5,\"reorderThreshold\":20,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"South DC\",\"product\":\"Gadget\",\"quantity\":2,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"},{\"warehouse\":\"West DC\",\"product\":\"Gadget\",\"quantity\":0,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"}]" + }, + { + "name": "tables stay locked down (publishable key reads nothing)", + "passed": true, + "notes": "publishable read errored: permission denied for table inventory" + }, + { + "name": "implementation uses @supabase/supabase-js", + "passed": false, + "notes": "no @supabase/supabase-js import found — this eval requires the SDK" + }, + { + "name": "report queries via the Data API, not raw SQL", + "passed": true, + "notes": "no psql / raw Postgres driver usage found" + } + ], + "skills": { + "available": [], + "loaded": [] + }, + "docs": { + "calls": [] + }, + "prompt": "Purchasing needs a restock alert. `app/restock.mjs` has the spec in a\ncomment — it runs in our Node backend worker and prints a JSON list of what\nneeds reordering, with who to email about it.\n\nThe data lives in the Supabase project in `supabase/` (already running\nlocally). Finish the script and make sure it prints the right alerts.", + "promptSourcePath": "evals/build-dataapi-002-restock-alert-report/PROMPT.md", + "attempts": 2, + "sourcePath": "claude-code-opus-4.8-no-skills/build-dataapi-002-restock-alert-report.json" + }, + { + "experiment": "claude-code-opus-4.8-no-skills", + "experimentSuite": "no-skills", + "experimentDisplay": { + "agent": "claude-code", + "modelProvider": "anthropic", + "modelId": "claude-opus-4-8", + "reasoningEffort": "high" + }, + "eval": "build-database-001-migrate-postgres-to-supabase", + "stage": "build", + "product": [ + "database" + ], + "topic": [ + "migrations" + ], + "suite": "benchmark", + "interface": "cli", + "passed": true, + "checks": [ + { + "name": "all 3 tables exist (teams, members, tasks)", + "passed": true + }, + { + "name": "row counts match (teams=5, members=10, tasks=13)", + "passed": true + }, + { + "name": "foreign key constraints survived the restore", + "passed": true + }, + { + "name": "tasks_team_status_idx index survived the restore", + "passed": true + }, + { + "name": "sequences synced (next insert won't conflict with existing IDs)", + "passed": true + } + ], + "skills": { + "available": [], + "loaded": [] + }, + "docs": { + "calls": [] + }, + "prompt": "I have an existing Postgres database I want to migrate to Supabase. There's a binary dump at `source.dump` in the current directory.\n\nCan you set up a local Supabase project and restore the dump into it?", + "promptSourcePath": "evals/build-database-001-migrate-postgres-to-supabase/PROMPT.md", + "attempts": 2, + "sourcePath": "claude-code-opus-4.8-no-skills/build-database-001-migrate-postgres-to-supabase.json" + }, + { + "experiment": "claude-code-opus-4.8-no-skills", + "experimentSuite": "no-skills", + "experimentDisplay": { + "agent": "claude-code", + "modelProvider": "anthropic", + "modelId": "claude-opus-4-8", + "reasoningEffort": "high" + }, + "eval": "build-functions-004-service-role-bypass", + "stage": "build", + "product": [ + "edge-functions", + "auth", + "database" + ], + "topic": [ + "rls", + "security", + "sdk" + ], + "suite": "benchmark", + "interface": "mcp", + "passed": true, "checks": [ { "name": "rejects missing auth", @@ -2805,6 +3218,88 @@ "attempts": 1, "sourcePath": "claude-code-opus-4.8-no-skills/resolve-security-002-rls-cross-tenant-leak.json" }, + { + "experiment": "claude-code-sonnet-5", + "experimentSuite": "benchmark", + "experimentDisplay": { + "agent": "claude-code", + "modelProvider": "anthropic", + "modelId": "claude-sonnet-5", + "reasoningEffort": "high" + }, + "eval": "build-auth-001-email-password-flow", + "stage": "build", + "product": [ + "auth", + "database" + ], + "topic": [ + "sdk", + "rls" + ], + "suite": "benchmark", + "interface": "cli", + "cliVersion": "2.109.1", + "passed": true, + "checks": [ + { + "name": "auth module loads and the driver completes", + "passed": true, + "notes": "driver produced a result" + }, + { + "name": "signUp creates the account and returns its user id", + "passed": true, + "notes": "db user f28745c7-1a48-485d-9436-43f285771a27, signUp returned {\"userId\":\"f28745c7-1a48-485d-9436-43f285771a27\"}" + }, + { + "name": "signup metadata reaches the profile (display name)", + "passed": true, + "notes": "profiles.display_name = \"Alex Doe\"" + }, + { + "name": "wrong password is rejected gracefully (no throw, no session)", + "passed": true, + "notes": "{\"error\":\"Invalid login credentials\"}" + }, + { + "name": "signIn with the right password returns the user id", + "passed": true, + "notes": "{\"userId\":\"f28745c7-1a48-485d-9436-43f285771a27\"}" + }, + { + "name": "getMyProfile returns the signed-in user's profile", + "passed": true, + "notes": "{\"displayName\":\"Alex Doe\",\"plan\":\"free\"}" + }, + { + "name": "app code does not use the secret / service-role key", + "passed": true, + "notes": "no secret-key references found" + }, + { + "name": "implementation uses @supabase/supabase-js", + "passed": true, + "notes": "imports found in: app/src/auth.mjs" + } + ], + "skills": { + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase" + ] + }, + "docs": { + "calls": [] + }, + "prompt": "Our app in `app/` needs accounts. Wire up `app/src/auth.mjs` — the stubs in\nthere describe what each function should do. People sign up with an email,\npassword, and display name, sign back in later, and the app greets them with\ntheir profile.\n\nThe Supabase project for this app is in `supabase/` and already running\nlocally. When you're done, the functions should work for real against it.", + "promptSourcePath": "evals/build-auth-001-email-password-flow/PROMPT.md", + "attempts": 1, + "sourcePath": "claude-code-sonnet-5/build-auth-001-email-password-flow.json" + }, { "experiment": "claude-code-sonnet-5", "experimentSuite": "benchmark", @@ -3029,37 +3524,44 @@ "modelId": "claude-sonnet-5", "reasoningEffort": "high" }, - "eval": "build-database-001-migrate-postgres-to-supabase", + "eval": "build-dataapi-001-relational-report", "stage": "build", "product": [ + "data-api", "database" ], "topic": [ - "migrations" + "sdk" ], "suite": "benchmark", "interface": "cli", + "cliVersion": "2.109.1", "passed": true, "checks": [ { - "name": "all 3 tables exist (teams, members, tasks)", - "passed": true + "name": "report runs and prints JSON", + "passed": true, + "notes": "exit 0" }, { - "name": "row counts match (teams=5, members=10, tasks=13)", - "passed": true + "name": "report numbers match the database (per customer, sorted)", + "passed": true, + "notes": "expected [{\"customer\":\"Ada Lovelace\",\"orderCount\":2,\"totalCents\":41900,\"topProduct\":\"Keyboard\"},{\"customer\":\"Grace Hopper\",\"orderCount\":2,\"totalCents\":15600,\"topProduct\":\"Cable\"},{\"customer\":\"Linus Pauling\",\"orderCount\":1,\"totalCents\":66500,\"topProduct\":\"Monitor\"}], got [{\"customer\":\"Ada Lovelace\",\"orderCount\":2,\"totalCents\":41900,\"topProduct\":\"Keyboard\"},{\"customer\":\"Grace Hopper\",\"orderCount\":2,\"totalCents\":15600,\"topProduct\":\"Cable\"},{\"customer\":\"Linus Pauling\",\"orderCount\":1,\"totalCents\":66500,\"topProduct\":\"Monitor\"}]" }, { - "name": "foreign key constraints survived the restore", - "passed": true + "name": "tables stay locked down (publishable key reads nothing)", + "passed": true, + "notes": "publishable read errored: permission denied for table customers" }, { - "name": "tasks_team_status_idx index survived the restore", - "passed": true + "name": "implementation uses @supabase/supabase-js", + "passed": true, + "notes": "imports found in: app/report.mjs" }, { - "name": "sequences synced (next insert won't conflict with existing IDs)", - "passed": true + "name": "report queries via the Data API, not raw SQL", + "passed": true, + "notes": "no psql / raw Postgres driver usage found" } ], "skills": { @@ -3074,10 +3576,10 @@ "docs": { "calls": [] }, - "prompt": "I have an existing Postgres database I want to migrate to Supabase. There's a binary dump at `source.dump` in the current directory.\n\nCan you set up a local Supabase project and restore the dump into it?", - "promptSourcePath": "evals/build-database-001-migrate-postgres-to-supabase/PROMPT.md", + "prompt": "We need the nightly sales report working. `app/report.mjs` has the spec in a\ncomment — it runs in our Node backend worker and prints a JSON summary of what\neach customer has ordered.\n\nThe data lives in the Supabase project in `supabase/` (already running\nlocally). Finish the script and make sure it prints the right numbers.", + "promptSourcePath": "evals/build-dataapi-001-relational-report/PROMPT.md", "attempts": 1, - "sourcePath": "claude-code-sonnet-5/build-database-001-migrate-postgres-to-supabase.json" + "sourcePath": "claude-code-sonnet-5/build-dataapi-001-relational-report.json" }, { "experiment": "claude-code-sonnet-5", @@ -3088,46 +3590,44 @@ "modelId": "claude-sonnet-5", "reasoningEffort": "high" }, - "eval": "build-functions-004-service-role-bypass", + "eval": "build-dataapi-002-restock-alert-report", "stage": "build", "product": [ - "edge-functions", - "auth", + "data-api", "database" ], "topic": [ - "rls", - "security", "sdk" ], "suite": "benchmark", - "interface": "mcp", + "interface": "cli", + "cliVersion": "2.109.1", "passed": true, "checks": [ { - "name": "rejects missing auth", + "name": "report runs and prints JSON", "passed": true, - "notes": "status=401" + "notes": "exit 0" }, { - "name": "user A reads own note", + "name": "alerts match the database (below threshold, sorted)", "passed": true, - "notes": "status=200" + "notes": "expected [{\"warehouse\":\"North DC\",\"product\":\"Gizmo\",\"quantity\":3,\"reorderThreshold\":10,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"North DC\",\"product\":\"Widget\",\"quantity\":5,\"reorderThreshold\":20,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"South DC\",\"product\":\"Gadget\",\"quantity\":2,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"},{\"warehouse\":\"West DC\",\"product\":\"Gadget\",\"quantity\":0,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"}], got [{\"warehouse\":\"North DC\",\"product\":\"Gizmo\",\"quantity\":3,\"reorderThreshold\":10,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"North DC\",\"product\":\"Widget\",\"quantity\":5,\"reorderThreshold\":20,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"South DC\",\"product\":\"Gadget\",\"quantity\":2,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"},{\"warehouse\":\"West DC\",\"product\":\"Gadget\",\"quantity\":0,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"}]" }, { - "name": "reads only with the caller's JWT", + "name": "tables stay locked down (publishable key reads nothing)", "passed": true, - "notes": "bearer_tokens=2, all_match=true" + "notes": "publishable read errored: permission denied for table inventory" }, { - "name": "user A cannot force-read user B note", + "name": "implementation uses @supabase/supabase-js", "passed": true, - "notes": "status=200" + "notes": "imports found in: app/restock.mjs" }, { - "name": "user B cannot force-read user A note", + "name": "report queries via the Data API, not raw SQL", "passed": true, - "notes": "status=200" + "notes": "no psql / raw Postgres driver usage found" } ], "skills": { @@ -3142,10 +3642,10 @@ "docs": { "calls": [] }, - "prompt": "I built an Edge Function called `private-notes` for showing a user's saved\nprivate notes.\n\nCan you check whether there's any way one user could see another user's notes?\n\nPlease fix and deploy it if needed.", - "promptSourcePath": "evals/build-functions-004-service-role-bypass/PROMPT.md", + "prompt": "Purchasing needs a restock alert. `app/restock.mjs` has the spec in a\ncomment — it runs in our Node backend worker and prints a JSON list of what\nneeds reordering, with who to email about it.\n\nThe data lives in the Supabase project in `supabase/` (already running\nlocally). Finish the script and make sure it prints the right alerts.", + "promptSourcePath": "evals/build-dataapi-002-restock-alert-report/PROMPT.md", "attempts": 1, - "sourcePath": "claude-code-sonnet-5/build-functions-004-service-role-bypass.json" + "sourcePath": "claude-code-sonnet-5/build-dataapi-002-restock-alert-report.json" }, { "experiment": "claude-code-sonnet-5", @@ -3156,57 +3656,184 @@ "modelId": "claude-sonnet-5", "reasoningEffort": "high" }, - "eval": "build-functions-005-dual-auth-user-secret", + "eval": "build-database-001-migrate-postgres-to-supabase", "stage": "build", "product": [ - "edge-functions", - "auth", "database" ], "topic": [ - "sdk", - "rls", - "security" + "migrations" ], "suite": "benchmark", "interface": "cli", - "cliVersion": "2.109.1", "passed": true, "checks": [ { - "name": "seed rows present", - "passed": true, - "notes": "found 2/2 seeded rows" - }, - { - "name": "rejects request with no credentials", - "passed": true, - "notes": "status 401: {\"message\":\"Invalid credentials\",\"code\":\"INVALID_CREDENTIALS\"}" - }, - { - "name": "user with JWT reads only their own rows", - "passed": true, - "notes": "status 200: {\"data\":[{\"user_id\":\"6f3de9df-d36e-4fe3-afa6-585831ee3ab6\",\"metric\":\"steps_a_mrnza8oa\",\"value\":111}]}" + "name": "all 3 tables exist (teams, members, tasks)", + "passed": true }, { - "name": "user cannot read another user's rows by passing user_id", - "passed": true, - "notes": "status 200: {\"data\":[{\"user_id\":\"6f3de9df-d36e-4fe3-afa6-585831ee3ab6\",\"metric\":\"steps_a_mrnza8oa\",\"value\":111}]}" + "name": "row counts match (teams=5, members=10, tasks=13)", + "passed": true }, { - "name": "service key bypasses RLS to read the target user's rows", - "passed": true, - "notes": "status 200: {\"data\":[{\"user_id\":\"547d02a8-3075-460e-b74b-3566ca328be3\",\"metric\":\"steps_b_mrnza8oa\",\"value\":222}]}" + "name": "foreign key constraints survived the restore", + "passed": true }, { - "name": "non-service key is not granted service access", - "passed": true, - "notes": "status 401: {\"message\":\"Invalid credentials\",\"code\":\"INVALID_CREDENTIALS\"}" + "name": "tasks_team_status_idx index survived the restore", + "passed": true }, { - "name": "rejects an unverified (forged) user token", - "passed": true, - "notes": "status 504: { \"message\":\"The upstream server is timing out\" }" + "name": "sequences synced (next insert won't conflict with existing IDs)", + "passed": true + } + ], + "skills": { + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase" + ] + }, + "docs": { + "calls": [] + }, + "prompt": "I have an existing Postgres database I want to migrate to Supabase. There's a binary dump at `source.dump` in the current directory.\n\nCan you set up a local Supabase project and restore the dump into it?", + "promptSourcePath": "evals/build-database-001-migrate-postgres-to-supabase/PROMPT.md", + "attempts": 1, + "sourcePath": "claude-code-sonnet-5/build-database-001-migrate-postgres-to-supabase.json" + }, + { + "experiment": "claude-code-sonnet-5", + "experimentSuite": "benchmark", + "experimentDisplay": { + "agent": "claude-code", + "modelProvider": "anthropic", + "modelId": "claude-sonnet-5", + "reasoningEffort": "high" + }, + "eval": "build-functions-004-service-role-bypass", + "stage": "build", + "product": [ + "edge-functions", + "auth", + "database" + ], + "topic": [ + "rls", + "security", + "sdk" + ], + "suite": "benchmark", + "interface": "mcp", + "passed": true, + "checks": [ + { + "name": "rejects missing auth", + "passed": true, + "notes": "status=401" + }, + { + "name": "user A reads own note", + "passed": true, + "notes": "status=200" + }, + { + "name": "reads only with the caller's JWT", + "passed": true, + "notes": "bearer_tokens=2, all_match=true" + }, + { + "name": "user A cannot force-read user B note", + "passed": true, + "notes": "status=200" + }, + { + "name": "user B cannot force-read user A note", + "passed": true, + "notes": "status=200" + } + ], + "skills": { + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase" + ] + }, + "docs": { + "calls": [] + }, + "prompt": "I built an Edge Function called `private-notes` for showing a user's saved\nprivate notes.\n\nCan you check whether there's any way one user could see another user's notes?\n\nPlease fix and deploy it if needed.", + "promptSourcePath": "evals/build-functions-004-service-role-bypass/PROMPT.md", + "attempts": 1, + "sourcePath": "claude-code-sonnet-5/build-functions-004-service-role-bypass.json" + }, + { + "experiment": "claude-code-sonnet-5", + "experimentSuite": "benchmark", + "experimentDisplay": { + "agent": "claude-code", + "modelProvider": "anthropic", + "modelId": "claude-sonnet-5", + "reasoningEffort": "high" + }, + "eval": "build-functions-005-dual-auth-user-secret", + "stage": "build", + "product": [ + "edge-functions", + "auth", + "database" + ], + "topic": [ + "sdk", + "rls", + "security" + ], + "suite": "benchmark", + "interface": "cli", + "cliVersion": "2.109.1", + "passed": true, + "checks": [ + { + "name": "seed rows present", + "passed": true, + "notes": "found 2/2 seeded rows" + }, + { + "name": "rejects request with no credentials", + "passed": true, + "notes": "status 401: {\"message\":\"Invalid credentials\",\"code\":\"INVALID_CREDENTIALS\"}" + }, + { + "name": "user with JWT reads only their own rows", + "passed": true, + "notes": "status 200: {\"data\":[{\"user_id\":\"6f3de9df-d36e-4fe3-afa6-585831ee3ab6\",\"metric\":\"steps_a_mrnza8oa\",\"value\":111}]}" + }, + { + "name": "user cannot read another user's rows by passing user_id", + "passed": true, + "notes": "status 200: {\"data\":[{\"user_id\":\"6f3de9df-d36e-4fe3-afa6-585831ee3ab6\",\"metric\":\"steps_a_mrnza8oa\",\"value\":111}]}" + }, + { + "name": "service key bypasses RLS to read the target user's rows", + "passed": true, + "notes": "status 200: {\"data\":[{\"user_id\":\"547d02a8-3075-460e-b74b-3566ca328be3\",\"metric\":\"steps_b_mrnza8oa\",\"value\":222}]}" + }, + { + "name": "non-service key is not granted service access", + "passed": true, + "notes": "status 401: {\"message\":\"Invalid credentials\",\"code\":\"INVALID_CREDENTIALS\"}" + }, + { + "name": "rejects an unverified (forged) user token", + "passed": true, + "notes": "status 504: { \"message\":\"The upstream server is timing out\" }" }, { "name": "a user token in the apikey slot is not treated as the service key", @@ -4356,6 +4983,83 @@ "attempts": 1, "sourcePath": "claude-code-sonnet-5/resolve-security-002-rls-cross-tenant-leak.json" }, + { + "experiment": "claude-code-sonnet-5-no-skills", + "experimentSuite": "no-skills", + "experimentDisplay": { + "agent": "claude-code", + "modelProvider": "anthropic", + "modelId": "claude-sonnet-5", + "reasoningEffort": "high" + }, + "eval": "build-auth-001-email-password-flow", + "stage": "build", + "product": [ + "auth", + "database" + ], + "topic": [ + "sdk", + "rls" + ], + "suite": "benchmark", + "interface": "cli", + "cliVersion": "2.109.1", + "passed": true, + "checks": [ + { + "name": "auth module loads and the driver completes", + "passed": true, + "notes": "driver produced a result" + }, + { + "name": "signUp creates the account and returns its user id", + "passed": true, + "notes": "db user ceb45c0d-99c0-4bb9-a928-d31c56b7a92d, signUp returned {\"userId\":\"ceb45c0d-99c0-4bb9-a928-d31c56b7a92d\"}" + }, + { + "name": "signup metadata reaches the profile (display name)", + "passed": true, + "notes": "profiles.display_name = \"Alex Doe\"" + }, + { + "name": "wrong password is rejected gracefully (no throw, no session)", + "passed": true, + "notes": "{\"error\":\"Invalid login credentials\"}" + }, + { + "name": "signIn with the right password returns the user id", + "passed": true, + "notes": "{\"userId\":\"ceb45c0d-99c0-4bb9-a928-d31c56b7a92d\"}" + }, + { + "name": "getMyProfile returns the signed-in user's profile", + "passed": true, + "notes": "{\"displayName\":\"Alex Doe\",\"plan\":\"free\"}" + }, + { + "name": "app code does not use the secret / service-role key", + "passed": true, + "notes": "no secret-key references found" + }, + { + "name": "implementation uses @supabase/supabase-js", + "passed": true, + "notes": "imports found in: app/src/auth.mjs" + } + ], + "skills": { + "available": [], + "loaded": [] + }, + "docs": { + "calls": [] + }, + "prompt": "Our app in `app/` needs accounts. Wire up `app/src/auth.mjs` — the stubs in\nthere describe what each function should do. People sign up with an email,\npassword, and display name, sign back in later, and the app greets them with\ntheir profile.\n\nThe Supabase project for this app is in `supabase/` and already running\nlocally. When you're done, the functions should work for real against it.", + "promptSourcePath": "evals/build-auth-001-email-password-flow/PROMPT.md", + "attempts": 1, + "sourcePath": "claude-code-sonnet-5-no-skills/build-auth-001-email-password-flow.json" + }, { "experiment": "claude-code-sonnet-5-no-skills", "experimentSuite": "no-skills", @@ -4536,37 +5240,44 @@ "modelId": "claude-sonnet-5", "reasoningEffort": "high" }, - "eval": "build-database-001-migrate-postgres-to-supabase", + "eval": "build-dataapi-001-relational-report", "stage": "build", "product": [ + "data-api", "database" ], "topic": [ - "migrations" + "sdk" ], "suite": "benchmark", "interface": "cli", - "passed": true, + "cliVersion": "2.109.1", + "passed": false, "checks": [ { - "name": "all 3 tables exist (teams, members, tasks)", - "passed": true + "name": "report runs and prints JSON", + "passed": true, + "notes": "exit 0" }, { - "name": "row counts match (teams=5, members=10, tasks=13)", - "passed": true + "name": "report numbers match the database (per customer, sorted)", + "passed": true, + "notes": "expected [{\"customer\":\"Ada Lovelace\",\"orderCount\":2,\"totalCents\":41900,\"topProduct\":\"Keyboard\"},{\"customer\":\"Grace Hopper\",\"orderCount\":2,\"totalCents\":15600,\"topProduct\":\"Cable\"},{\"customer\":\"Linus Pauling\",\"orderCount\":1,\"totalCents\":66500,\"topProduct\":\"Monitor\"}], got [{\"customer\":\"Ada Lovelace\",\"orderCount\":2,\"totalCents\":41900,\"topProduct\":\"Keyboard\"},{\"customer\":\"Grace Hopper\",\"orderCount\":2,\"totalCents\":15600,\"topProduct\":\"Cable\"},{\"customer\":\"Linus Pauling\",\"orderCount\":1,\"totalCents\":66500,\"topProduct\":\"Monitor\"}]" }, { - "name": "foreign key constraints survived the restore", - "passed": true + "name": "tables stay locked down (publishable key reads nothing)", + "passed": true, + "notes": "publishable read errored: permission denied for table customers" }, { - "name": "tasks_team_status_idx index survived the restore", - "passed": true + "name": "implementation uses @supabase/supabase-js", + "passed": false, + "notes": "no @supabase/supabase-js import found — this eval requires the SDK" }, { - "name": "sequences synced (next insert won't conflict with existing IDs)", - "passed": true + "name": "report queries via the Data API, not raw SQL", + "passed": true, + "notes": "no psql / raw Postgres driver usage found" } ], "skills": { @@ -4576,10 +5287,10 @@ "docs": { "calls": [] }, - "prompt": "I have an existing Postgres database I want to migrate to Supabase. There's a binary dump at `source.dump` in the current directory.\n\nCan you set up a local Supabase project and restore the dump into it?", - "promptSourcePath": "evals/build-database-001-migrate-postgres-to-supabase/PROMPT.md", - "attempts": 1, - "sourcePath": "claude-code-sonnet-5-no-skills/build-database-001-migrate-postgres-to-supabase.json" + "prompt": "We need the nightly sales report working. `app/report.mjs` has the spec in a\ncomment — it runs in our Node backend worker and prints a JSON summary of what\neach customer has ordered.\n\nThe data lives in the Supabase project in `supabase/` (already running\nlocally). Finish the script and make sure it prints the right numbers.", + "promptSourcePath": "evals/build-dataapi-001-relational-report/PROMPT.md", + "attempts": 2, + "sourcePath": "claude-code-sonnet-5-no-skills/build-dataapi-001-relational-report.json" }, { "experiment": "claude-code-sonnet-5-no-skills", @@ -4590,46 +5301,44 @@ "modelId": "claude-sonnet-5", "reasoningEffort": "high" }, - "eval": "build-functions-004-service-role-bypass", + "eval": "build-dataapi-002-restock-alert-report", "stage": "build", "product": [ - "edge-functions", - "auth", + "data-api", "database" ], "topic": [ - "rls", - "security", "sdk" ], "suite": "benchmark", - "interface": "mcp", + "interface": "cli", + "cliVersion": "2.109.1", "passed": true, "checks": [ { - "name": "rejects missing auth", + "name": "report runs and prints JSON", "passed": true, - "notes": "status=401" + "notes": "exit 0" }, { - "name": "user A reads own note", + "name": "alerts match the database (below threshold, sorted)", "passed": true, - "notes": "status=200" + "notes": "expected [{\"warehouse\":\"North DC\",\"product\":\"Gizmo\",\"quantity\":3,\"reorderThreshold\":10,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"North DC\",\"product\":\"Widget\",\"quantity\":5,\"reorderThreshold\":20,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"South DC\",\"product\":\"Gadget\",\"quantity\":2,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"},{\"warehouse\":\"West DC\",\"product\":\"Gadget\",\"quantity\":0,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"}], got [{\"warehouse\":\"North DC\",\"product\":\"Gizmo\",\"quantity\":3,\"reorderThreshold\":10,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"North DC\",\"product\":\"Widget\",\"quantity\":5,\"reorderThreshold\":20,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"South DC\",\"product\":\"Gadget\",\"quantity\":2,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"},{\"warehouse\":\"West DC\",\"product\":\"Gadget\",\"quantity\":0,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"}]" }, { - "name": "reads only with the caller's JWT", + "name": "tables stay locked down (publishable key reads nothing)", "passed": true, - "notes": "bearer_tokens=2, all_match=true" + "notes": "publishable read errored: permission denied for table inventory" }, { - "name": "user A cannot force-read user B note", + "name": "implementation uses @supabase/supabase-js", "passed": true, - "notes": "status=200" + "notes": "imports found in: app/restock.mjs" }, { - "name": "user B cannot force-read user A note", + "name": "report queries via the Data API, not raw SQL", "passed": true, - "notes": "status=200" + "notes": "no psql / raw Postgres driver usage found" } ], "skills": { @@ -4639,10 +5348,127 @@ "docs": { "calls": [] }, - "prompt": "I built an Edge Function called `private-notes` for showing a user's saved\nprivate notes.\n\nCan you check whether there's any way one user could see another user's notes?\n\nPlease fix and deploy it if needed.", - "promptSourcePath": "evals/build-functions-004-service-role-bypass/PROMPT.md", - "attempts": 1, - "sourcePath": "claude-code-sonnet-5-no-skills/build-functions-004-service-role-bypass.json" + "prompt": "Purchasing needs a restock alert. `app/restock.mjs` has the spec in a\ncomment — it runs in our Node backend worker and prints a JSON list of what\nneeds reordering, with who to email about it.\n\nThe data lives in the Supabase project in `supabase/` (already running\nlocally). Finish the script and make sure it prints the right alerts.", + "promptSourcePath": "evals/build-dataapi-002-restock-alert-report/PROMPT.md", + "attempts": 2, + "sourcePath": "claude-code-sonnet-5-no-skills/build-dataapi-002-restock-alert-report.json" + }, + { + "experiment": "claude-code-sonnet-5-no-skills", + "experimentSuite": "no-skills", + "experimentDisplay": { + "agent": "claude-code", + "modelProvider": "anthropic", + "modelId": "claude-sonnet-5", + "reasoningEffort": "high" + }, + "eval": "build-database-001-migrate-postgres-to-supabase", + "stage": "build", + "product": [ + "database" + ], + "topic": [ + "migrations" + ], + "suite": "benchmark", + "interface": "cli", + "passed": true, + "checks": [ + { + "name": "all 3 tables exist (teams, members, tasks)", + "passed": true + }, + { + "name": "row counts match (teams=5, members=10, tasks=13)", + "passed": true + }, + { + "name": "foreign key constraints survived the restore", + "passed": true + }, + { + "name": "tasks_team_status_idx index survived the restore", + "passed": true + }, + { + "name": "sequences synced (next insert won't conflict with existing IDs)", + "passed": true + } + ], + "skills": { + "available": [], + "loaded": [] + }, + "docs": { + "calls": [] + }, + "prompt": "I have an existing Postgres database I want to migrate to Supabase. There's a binary dump at `source.dump` in the current directory.\n\nCan you set up a local Supabase project and restore the dump into it?", + "promptSourcePath": "evals/build-database-001-migrate-postgres-to-supabase/PROMPT.md", + "attempts": 1, + "sourcePath": "claude-code-sonnet-5-no-skills/build-database-001-migrate-postgres-to-supabase.json" + }, + { + "experiment": "claude-code-sonnet-5-no-skills", + "experimentSuite": "no-skills", + "experimentDisplay": { + "agent": "claude-code", + "modelProvider": "anthropic", + "modelId": "claude-sonnet-5", + "reasoningEffort": "high" + }, + "eval": "build-functions-004-service-role-bypass", + "stage": "build", + "product": [ + "edge-functions", + "auth", + "database" + ], + "topic": [ + "rls", + "security", + "sdk" + ], + "suite": "benchmark", + "interface": "mcp", + "passed": true, + "checks": [ + { + "name": "rejects missing auth", + "passed": true, + "notes": "status=401" + }, + { + "name": "user A reads own note", + "passed": true, + "notes": "status=200" + }, + { + "name": "reads only with the caller's JWT", + "passed": true, + "notes": "bearer_tokens=2, all_match=true" + }, + { + "name": "user A cannot force-read user B note", + "passed": true, + "notes": "status=200" + }, + { + "name": "user B cannot force-read user A note", + "passed": true, + "notes": "status=200" + } + ], + "skills": { + "available": [], + "loaded": [] + }, + "docs": { + "calls": [] + }, + "prompt": "I built an Edge Function called `private-notes` for showing a user's saved\nprivate notes.\n\nCan you check whether there's any way one user could see another user's notes?\n\nPlease fix and deploy it if needed.", + "promptSourcePath": "evals/build-functions-004-service-role-bypass/PROMPT.md", + "attempts": 1, + "sourcePath": "claude-code-sonnet-5-no-skills/build-functions-004-service-role-bypass.json" }, { "experiment": "claude-code-sonnet-5-no-skills", @@ -5613,6 +6439,162 @@ "attempts": 1, "sourcePath": "claude-code-sonnet-5-no-skills/resolve-security-002-rls-cross-tenant-leak.json" }, + { + "experiment": "codex-gpt-5.4-mini", + "experimentSuite": "benchmark", + "experimentDisplay": { + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-5.4-mini", + "reasoningEffort": "medium" + }, + "eval": "build-auth-001-email-password-flow", + "stage": "build", + "product": [ + "auth", + "database" + ], + "topic": [ + "sdk", + "rls" + ], + "suite": "benchmark", + "interface": "cli", + "cliVersion": "2.109.1", + "passed": false, + "checks": [ + { + "name": "auth module loads and the driver completes", + "passed": true, + "notes": "driver produced a result" + }, + { + "name": "signUp creates the account and returns its user id", + "passed": true, + "notes": "db user 0584d564-0f3b-4b64-bb64-69a904d5c9d3, signUp returned {\"userId\":\"0584d564-0f3b-4b64-bb64-69a904d5c9d3\"}" + }, + { + "name": "signup metadata reaches the profile (display name)", + "passed": true, + "notes": "profiles.display_name = \"Alex Doe\"" + }, + { + "name": "wrong password is rejected gracefully (no throw, no session)", + "passed": true, + "notes": "{\"error\":\"Invalid login credentials\"}" + }, + { + "name": "signIn with the right password returns the user id", + "passed": true, + "notes": "{\"userId\":\"0584d564-0f3b-4b64-bb64-69a904d5c9d3\"}" + }, + { + "name": "getMyProfile returns the signed-in user's profile", + "passed": true, + "notes": "{\"displayName\":\"Alex Doe\",\"plan\":\"free\"}" + }, + { + "name": "app code does not use the secret / service-role key", + "passed": true, + "notes": "no secret-key references found" + }, + { + "name": "implementation uses @supabase/supabase-js", + "passed": false, + "notes": "no @supabase/supabase-js import found — this eval requires the SDK" + } + ], + "skills": { + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase" + ] + }, + "docs": { + "calls": [ + { + "source": "search_docs", + "query": "query {\n searchDocs(query: \"supabase-js signUp signInWithPassword getUser authenticated session client-side email password metadata display_name\", limit: 5) {\n nodes {\n ... on Guide { title href content }\n ... on ClientLibraryFunctionReference { title href content language methodName }\n ... on TroubleshootingGuide { title href content }\n }\n totalCount\n }\n}", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/troubleshooting/customizing-emails-by-language-KZ_38Q", + "title": "Customizing Emails by Language" + }, + { + "url": "https://supabase.com/docs/guides/auth/managing-user-data", + "title": "User Management" + }, + { + "url": "https://supabase.com/docs/guides/auth/passwords", + "title": "Password-based Auth" + }, + { + "url": "https://supabase.com/docs/guides/auth/passkeys", + "title": "Passkey authentication" + }, + { + "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/auth0", + "title": "Migrate from Auth0 to Supabase Auth" + } + ], + "resultChars": 83072 + }, + { + "source": "search_docs", + "query": "query {\n searchDocs(query: \"JavaScript signInWithPassword auth reference supabase-js getUser current user session\", limit: 10) {\n nodes {\n ... on ClientLibraryFunctionReference { title href content language methodName }\n ... on Guide { title href content }\n }\n totalCount\n }\n}", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/reference/javascript/auth-getuser" + }, + { + "url": "https://supabase.com/docs/guides/auth/passwords", + "title": "Password-based Auth" + }, + { + "url": "https://supabase.com/docs/guides/auth/passkeys", + "title": "Passkey authentication" + }, + { + "url": "https://supabase.com/docs/reference/javascript/auth-signinwithpassword" + }, + { + "url": "https://supabase.com/docs/guides/auth/social-login/auth-google", + "title": "Login with Google" + }, + { + "url": "https://supabase.com/docs/reference/dart/auth-signinwithpassword" + }, + { + "url": "https://supabase.com/docs/reference/swift/auth-signinwithpassword", + "title": "signInWithPassword()" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-saml-sso", + "title": "Configure SAML SSO" + }, + { + "url": "https://supabase.com/docs/guides/auth/social-login/auth-figma", + "title": "Login with Figma" + }, + { + "url": "https://supabase.com/docs/guides/auth/social-login/auth-azure", + "title": "Login with Azure (Microsoft)" + } + ], + "resultChars": 178711 + } + ] + }, + "prompt": "Our app in `app/` needs accounts. Wire up `app/src/auth.mjs` — the stubs in\nthere describe what each function should do. People sign up with an email,\npassword, and display name, sign back in later, and the app greets them with\ntheir profile.\n\nThe Supabase project for this app is in `supabase/` and already running\nlocally. When you're done, the functions should work for real against it.", + "promptSourcePath": "evals/build-auth-001-email-password-flow/PROMPT.md", + "attempts": 2, + "sourcePath": "codex-gpt-5.4-mini/build-auth-001-email-password-flow.json" + }, { "experiment": "codex-gpt-5.4-mini", "experimentSuite": "benchmark", @@ -7058,37 +8040,44 @@ "modelId": "gpt-5.4-mini", "reasoningEffort": "medium" }, - "eval": "build-database-001-migrate-postgres-to-supabase", + "eval": "build-dataapi-001-relational-report", "stage": "build", "product": [ + "data-api", "database" ], "topic": [ - "migrations" + "sdk" ], "suite": "benchmark", "interface": "cli", - "passed": true, + "cliVersion": "2.109.1", + "passed": false, "checks": [ { - "name": "all 3 tables exist (teams, members, tasks)", - "passed": true + "name": "report runs and prints JSON", + "passed": true, + "notes": "exit 0" }, { - "name": "row counts match (teams=5, members=10, tasks=13)", - "passed": true + "name": "report numbers match the database (per customer, sorted)", + "passed": true, + "notes": "expected [{\"customer\":\"Ada Lovelace\",\"orderCount\":2,\"totalCents\":41900,\"topProduct\":\"Keyboard\"},{\"customer\":\"Grace Hopper\",\"orderCount\":2,\"totalCents\":15600,\"topProduct\":\"Cable\"},{\"customer\":\"Linus Pauling\",\"orderCount\":1,\"totalCents\":66500,\"topProduct\":\"Monitor\"}], got [{\"customer\":\"Ada Lovelace\",\"orderCount\":2,\"totalCents\":41900,\"topProduct\":\"Keyboard\"},{\"customer\":\"Grace Hopper\",\"orderCount\":2,\"totalCents\":15600,\"topProduct\":\"Cable\"},{\"customer\":\"Linus Pauling\",\"orderCount\":1,\"totalCents\":66500,\"topProduct\":\"Monitor\"}]" }, { - "name": "foreign key constraints survived the restore", - "passed": true + "name": "tables stay locked down (publishable key reads nothing)", + "passed": true, + "notes": "publishable read errored: permission denied for table customers" }, { - "name": "tasks_team_status_idx index survived the restore", - "passed": true + "name": "implementation uses @supabase/supabase-js", + "passed": false, + "notes": "no @supabase/supabase-js import found — this eval requires the SDK" }, { - "name": "sequences synced (next insert won't conflict with existing IDs)", - "passed": true + "name": "report queries via the Data API, not raw SQL", + "passed": true, + "notes": "no psql / raw Postgres driver usage found" } ], "skills": { @@ -7103,8 +8092,133 @@ "docs": { "calls": [] }, - "prompt": "I have an existing Postgres database I want to migrate to Supabase. There's a binary dump at `source.dump` in the current directory.\n\nCan you set up a local Supabase project and restore the dump into it?", - "promptSourcePath": "evals/build-database-001-migrate-postgres-to-supabase/PROMPT.md", + "prompt": "We need the nightly sales report working. `app/report.mjs` has the spec in a\ncomment — it runs in our Node backend worker and prints a JSON summary of what\neach customer has ordered.\n\nThe data lives in the Supabase project in `supabase/` (already running\nlocally). Finish the script and make sure it prints the right numbers.", + "promptSourcePath": "evals/build-dataapi-001-relational-report/PROMPT.md", + "attempts": 2, + "sourcePath": "codex-gpt-5.4-mini/build-dataapi-001-relational-report.json" + }, + { + "experiment": "codex-gpt-5.4-mini", + "experimentSuite": "benchmark", + "experimentDisplay": { + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-5.4-mini", + "reasoningEffort": "medium" + }, + "eval": "build-dataapi-002-restock-alert-report", + "stage": "build", + "product": [ + "data-api", + "database" + ], + "topic": [ + "sdk" + ], + "suite": "benchmark", + "interface": "cli", + "cliVersion": "2.109.1", + "passed": false, + "checks": [ + { + "name": "report runs and prints JSON", + "passed": true, + "notes": "exit 0" + }, + { + "name": "alerts match the database (below threshold, sorted)", + "passed": true, + "notes": "expected [{\"warehouse\":\"North DC\",\"product\":\"Gizmo\",\"quantity\":3,\"reorderThreshold\":10,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"North DC\",\"product\":\"Widget\",\"quantity\":5,\"reorderThreshold\":20,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"South DC\",\"product\":\"Gadget\",\"quantity\":2,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"},{\"warehouse\":\"West DC\",\"product\":\"Gadget\",\"quantity\":0,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"}], got [{\"warehouse\":\"North DC\",\"product\":\"Gizmo\",\"quantity\":3,\"reorderThreshold\":10,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"North DC\",\"product\":\"Widget\",\"quantity\":5,\"reorderThreshold\":20,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"South DC\",\"product\":\"Gadget\",\"quantity\":2,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"},{\"warehouse\":\"West DC\",\"product\":\"Gadget\",\"quantity\":0,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"}]" + }, + { + "name": "tables stay locked down (publishable key reads nothing)", + "passed": true, + "notes": "publishable read errored: permission denied for table inventory" + }, + { + "name": "implementation uses @supabase/supabase-js", + "passed": false, + "notes": "no @supabase/supabase-js import found — this eval requires the SDK" + }, + { + "name": "report queries via the Data API, not raw SQL", + "passed": true, + "notes": "no psql / raw Postgres driver usage found" + } + ], + "skills": { + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase" + ] + }, + "docs": { + "calls": [] + }, + "prompt": "Purchasing needs a restock alert. `app/restock.mjs` has the spec in a\ncomment — it runs in our Node backend worker and prints a JSON list of what\nneeds reordering, with who to email about it.\n\nThe data lives in the Supabase project in `supabase/` (already running\nlocally). Finish the script and make sure it prints the right alerts.", + "promptSourcePath": "evals/build-dataapi-002-restock-alert-report/PROMPT.md", + "attempts": 2, + "sourcePath": "codex-gpt-5.4-mini/build-dataapi-002-restock-alert-report.json" + }, + { + "experiment": "codex-gpt-5.4-mini", + "experimentSuite": "benchmark", + "experimentDisplay": { + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-5.4-mini", + "reasoningEffort": "medium" + }, + "eval": "build-database-001-migrate-postgres-to-supabase", + "stage": "build", + "product": [ + "database" + ], + "topic": [ + "migrations" + ], + "suite": "benchmark", + "interface": "cli", + "passed": true, + "checks": [ + { + "name": "all 3 tables exist (teams, members, tasks)", + "passed": true + }, + { + "name": "row counts match (teams=5, members=10, tasks=13)", + "passed": true + }, + { + "name": "foreign key constraints survived the restore", + "passed": true + }, + { + "name": "tasks_team_status_idx index survived the restore", + "passed": true + }, + { + "name": "sequences synced (next insert won't conflict with existing IDs)", + "passed": true + } + ], + "skills": { + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase" + ] + }, + "docs": { + "calls": [] + }, + "prompt": "I have an existing Postgres database I want to migrate to Supabase. There's a binary dump at `source.dump` in the current directory.\n\nCan you set up a local Supabase project and restore the dump into it?", + "promptSourcePath": "evals/build-database-001-migrate-postgres-to-supabase/PROMPT.md", "attempts": 2, "sourcePath": "codex-gpt-5.4-mini/build-database-001-migrate-postgres-to-supabase.json" }, @@ -10450,6 +11564,83 @@ "attempts": 1, "sourcePath": "codex-gpt-5.4-mini/resolve-security-002-rls-cross-tenant-leak.json" }, + { + "experiment": "codex-gpt-5.4-mini-no-skills", + "experimentSuite": "no-skills", + "experimentDisplay": { + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-5.4-mini", + "reasoningEffort": "medium" + }, + "eval": "build-auth-001-email-password-flow", + "stage": "build", + "product": [ + "auth", + "database" + ], + "topic": [ + "sdk", + "rls" + ], + "suite": "benchmark", + "interface": "cli", + "cliVersion": "2.109.1", + "passed": false, + "checks": [ + { + "name": "auth module loads and the driver completes", + "passed": true, + "notes": "driver produced a result" + }, + { + "name": "signUp creates the account and returns its user id", + "passed": true, + "notes": "db user c6728d3f-b2a9-4375-9a22-9a64abf82e3d, signUp returned {\"userId\":\"c6728d3f-b2a9-4375-9a22-9a64abf82e3d\"}" + }, + { + "name": "signup metadata reaches the profile (display name)", + "passed": true, + "notes": "profiles.display_name = \"Alex Doe\"" + }, + { + "name": "wrong password is rejected gracefully (no throw, no session)", + "passed": true, + "notes": "{\"error\":\"Invalid login credentials\"}" + }, + { + "name": "signIn with the right password returns the user id", + "passed": true, + "notes": "{\"userId\":\"c6728d3f-b2a9-4375-9a22-9a64abf82e3d\"}" + }, + { + "name": "getMyProfile returns the signed-in user's profile", + "passed": true, + "notes": "{\"displayName\":\"Alex Doe\",\"plan\":\"free\"}" + }, + { + "name": "app code does not use the secret / service-role key", + "passed": true, + "notes": "no secret-key references found" + }, + { + "name": "implementation uses @supabase/supabase-js", + "passed": false, + "notes": "no @supabase/supabase-js import found — this eval requires the SDK" + } + ], + "skills": { + "available": [], + "loaded": [] + }, + "docs": { + "calls": [] + }, + "prompt": "Our app in `app/` needs accounts. Wire up `app/src/auth.mjs` — the stubs in\nthere describe what each function should do. People sign up with an email,\npassword, and display name, sign back in later, and the app greets them with\ntheir profile.\n\nThe Supabase project for this app is in `supabase/` and already running\nlocally. When you're done, the functions should work for real against it.", + "promptSourcePath": "evals/build-auth-001-email-password-flow/PROMPT.md", + "attempts": 2, + "sourcePath": "codex-gpt-5.4-mini-no-skills/build-auth-001-email-password-flow.json" + }, { "experiment": "codex-gpt-5.4-mini-no-skills", "experimentSuite": "no-skills", @@ -10687,37 +11878,44 @@ "modelId": "gpt-5.4-mini", "reasoningEffort": "medium" }, - "eval": "build-database-001-migrate-postgres-to-supabase", + "eval": "build-dataapi-001-relational-report", "stage": "build", "product": [ + "data-api", "database" ], "topic": [ - "migrations" + "sdk" ], "suite": "benchmark", "interface": "cli", - "passed": true, + "cliVersion": "2.109.1", + "passed": false, "checks": [ { - "name": "all 3 tables exist (teams, members, tasks)", - "passed": true + "name": "report runs and prints JSON", + "passed": true, + "notes": "exit 0" }, { - "name": "row counts match (teams=5, members=10, tasks=13)", - "passed": true + "name": "report numbers match the database (per customer, sorted)", + "passed": true, + "notes": "expected [{\"customer\":\"Ada Lovelace\",\"orderCount\":2,\"totalCents\":41900,\"topProduct\":\"Keyboard\"},{\"customer\":\"Grace Hopper\",\"orderCount\":2,\"totalCents\":15600,\"topProduct\":\"Cable\"},{\"customer\":\"Linus Pauling\",\"orderCount\":1,\"totalCents\":66500,\"topProduct\":\"Monitor\"}], got [{\"customer\":\"Ada Lovelace\",\"orderCount\":2,\"totalCents\":41900,\"topProduct\":\"Keyboard\"},{\"customer\":\"Grace Hopper\",\"orderCount\":2,\"totalCents\":15600,\"topProduct\":\"Cable\"},{\"customer\":\"Linus Pauling\",\"orderCount\":1,\"totalCents\":66500,\"topProduct\":\"Monitor\"}]" }, { - "name": "foreign key constraints survived the restore", - "passed": true + "name": "tables stay locked down (publishable key reads nothing)", + "passed": true, + "notes": "publishable read errored: permission denied for table customers" }, { - "name": "tasks_team_status_idx index survived the restore", - "passed": true + "name": "implementation uses @supabase/supabase-js", + "passed": false, + "notes": "no @supabase/supabase-js import found — this eval requires the SDK" }, { - "name": "sequences synced (next insert won't conflict with existing IDs)", - "passed": true + "name": "report queries via the Data API, not raw SQL", + "passed": true, + "notes": "no psql / raw Postgres driver usage found" } ], "skills": { @@ -10727,10 +11925,10 @@ "docs": { "calls": [] }, - "prompt": "I have an existing Postgres database I want to migrate to Supabase. There's a binary dump at `source.dump` in the current directory.\n\nCan you set up a local Supabase project and restore the dump into it?", - "promptSourcePath": "evals/build-database-001-migrate-postgres-to-supabase/PROMPT.md", - "attempts": 1, - "sourcePath": "codex-gpt-5.4-mini-no-skills/build-database-001-migrate-postgres-to-supabase.json" + "prompt": "We need the nightly sales report working. `app/report.mjs` has the spec in a\ncomment — it runs in our Node backend worker and prints a JSON summary of what\neach customer has ordered.\n\nThe data lives in the Supabase project in `supabase/` (already running\nlocally). Finish the script and make sure it prints the right numbers.", + "promptSourcePath": "evals/build-dataapi-001-relational-report/PROMPT.md", + "attempts": 2, + "sourcePath": "codex-gpt-5.4-mini-no-skills/build-dataapi-001-relational-report.json" }, { "experiment": "codex-gpt-5.4-mini-no-skills", @@ -10741,46 +11939,44 @@ "modelId": "gpt-5.4-mini", "reasoningEffort": "medium" }, - "eval": "build-functions-004-service-role-bypass", + "eval": "build-dataapi-002-restock-alert-report", "stage": "build", "product": [ - "edge-functions", - "auth", + "data-api", "database" ], "topic": [ - "rls", - "security", "sdk" ], "suite": "benchmark", - "interface": "mcp", - "passed": true, + "interface": "cli", + "cliVersion": "2.109.1", + "passed": false, "checks": [ { - "name": "rejects missing auth", + "name": "report runs and prints JSON", "passed": true, - "notes": "status=401" + "notes": "exit 0" }, { - "name": "user A reads own note", + "name": "alerts match the database (below threshold, sorted)", "passed": true, - "notes": "status=200" + "notes": "expected [{\"warehouse\":\"North DC\",\"product\":\"Gizmo\",\"quantity\":3,\"reorderThreshold\":10,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"North DC\",\"product\":\"Widget\",\"quantity\":5,\"reorderThreshold\":20,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"South DC\",\"product\":\"Gadget\",\"quantity\":2,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"},{\"warehouse\":\"West DC\",\"product\":\"Gadget\",\"quantity\":0,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"}], got [{\"warehouse\":\"North DC\",\"product\":\"Gizmo\",\"quantity\":3,\"reorderThreshold\":10,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"North DC\",\"product\":\"Widget\",\"quantity\":5,\"reorderThreshold\":20,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"South DC\",\"product\":\"Gadget\",\"quantity\":2,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"},{\"warehouse\":\"West DC\",\"product\":\"Gadget\",\"quantity\":0,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"}]" }, { - "name": "reads only with the caller's JWT", + "name": "tables stay locked down (publishable key reads nothing)", "passed": true, - "notes": "bearer_tokens=2, all_match=true" + "notes": "publishable read errored: permission denied for table inventory" }, { - "name": "user A cannot force-read user B note", - "passed": true, - "notes": "status=200" + "name": "implementation uses @supabase/supabase-js", + "passed": false, + "notes": "no @supabase/supabase-js import found — this eval requires the SDK" }, { - "name": "user B cannot force-read user A note", + "name": "report queries via the Data API, not raw SQL", "passed": true, - "notes": "status=200" + "notes": "no psql / raw Postgres driver usage found" } ], "skills": { @@ -10788,41 +11984,12 @@ "loaded": [] }, "docs": { - "calls": [ - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Edge Functions environment variable SUPABASE_ANON_KEY auth user in Deno\", limit: 5) { nodes { title href content ... on Guide { subsections { totalCount } } } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/functions", - "title": "Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", - "title": "Integrating With Supabase Auth" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" - }, - { - "url": "https://supabase.com/docs/guides/functions/secrets", - "title": "Environment Variables" - }, - { - "url": "https://supabase.com/docs/guides/local-development/cli/testing-and-linting", - "title": "Testing and linting" - } - ], - "resultChars": 42100 - } - ] + "calls": [] }, - "prompt": "I built an Edge Function called `private-notes` for showing a user's saved\nprivate notes.\n\nCan you check whether there's any way one user could see another user's notes?\n\nPlease fix and deploy it if needed.", - "promptSourcePath": "evals/build-functions-004-service-role-bypass/PROMPT.md", - "attempts": 1, - "sourcePath": "codex-gpt-5.4-mini-no-skills/build-functions-004-service-role-bypass.json" + "prompt": "Purchasing needs a restock alert. `app/restock.mjs` has the spec in a\ncomment — it runs in our Node backend worker and prints a JSON list of what\nneeds reordering, with who to email about it.\n\nThe data lives in the Supabase project in `supabase/` (already running\nlocally). Finish the script and make sure it prints the right alerts.", + "promptSourcePath": "evals/build-dataapi-002-restock-alert-report/PROMPT.md", + "attempts": 2, + "sourcePath": "codex-gpt-5.4-mini-no-skills/build-dataapi-002-restock-alert-report.json" }, { "experiment": "codex-gpt-5.4-mini-no-skills", @@ -10833,10 +12000,156 @@ "modelId": "gpt-5.4-mini", "reasoningEffort": "medium" }, - "eval": "build-functions-005-dual-auth-user-secret", + "eval": "build-database-001-migrate-postgres-to-supabase", "stage": "build", "product": [ - "edge-functions", + "database" + ], + "topic": [ + "migrations" + ], + "suite": "benchmark", + "interface": "cli", + "passed": true, + "checks": [ + { + "name": "all 3 tables exist (teams, members, tasks)", + "passed": true + }, + { + "name": "row counts match (teams=5, members=10, tasks=13)", + "passed": true + }, + { + "name": "foreign key constraints survived the restore", + "passed": true + }, + { + "name": "tasks_team_status_idx index survived the restore", + "passed": true + }, + { + "name": "sequences synced (next insert won't conflict with existing IDs)", + "passed": true + } + ], + "skills": { + "available": [], + "loaded": [] + }, + "docs": { + "calls": [] + }, + "prompt": "I have an existing Postgres database I want to migrate to Supabase. There's a binary dump at `source.dump` in the current directory.\n\nCan you set up a local Supabase project and restore the dump into it?", + "promptSourcePath": "evals/build-database-001-migrate-postgres-to-supabase/PROMPT.md", + "attempts": 1, + "sourcePath": "codex-gpt-5.4-mini-no-skills/build-database-001-migrate-postgres-to-supabase.json" + }, + { + "experiment": "codex-gpt-5.4-mini-no-skills", + "experimentSuite": "no-skills", + "experimentDisplay": { + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-5.4-mini", + "reasoningEffort": "medium" + }, + "eval": "build-functions-004-service-role-bypass", + "stage": "build", + "product": [ + "edge-functions", + "auth", + "database" + ], + "topic": [ + "rls", + "security", + "sdk" + ], + "suite": "benchmark", + "interface": "mcp", + "passed": true, + "checks": [ + { + "name": "rejects missing auth", + "passed": true, + "notes": "status=401" + }, + { + "name": "user A reads own note", + "passed": true, + "notes": "status=200" + }, + { + "name": "reads only with the caller's JWT", + "passed": true, + "notes": "bearer_tokens=2, all_match=true" + }, + { + "name": "user A cannot force-read user B note", + "passed": true, + "notes": "status=200" + }, + { + "name": "user B cannot force-read user A note", + "passed": true, + "notes": "status=200" + } + ], + "skills": { + "available": [], + "loaded": [] + }, + "docs": { + "calls": [ + { + "source": "search_docs", + "query": "query { searchDocs(query: \"Edge Functions environment variable SUPABASE_ANON_KEY auth user in Deno\", limit: 5) { nodes { title href content ... on Guide { subsections { totalCount } } } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/functions", + "title": "Edge Functions" + }, + { + "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", + "title": "Integrating With Supabase Auth" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", + "title": "Migrating to publishable and secret API keys" + }, + { + "url": "https://supabase.com/docs/guides/functions/secrets", + "title": "Environment Variables" + }, + { + "url": "https://supabase.com/docs/guides/local-development/cli/testing-and-linting", + "title": "Testing and linting" + } + ], + "resultChars": 42100 + } + ] + }, + "prompt": "I built an Edge Function called `private-notes` for showing a user's saved\nprivate notes.\n\nCan you check whether there's any way one user could see another user's notes?\n\nPlease fix and deploy it if needed.", + "promptSourcePath": "evals/build-functions-004-service-role-bypass/PROMPT.md", + "attempts": 1, + "sourcePath": "codex-gpt-5.4-mini-no-skills/build-functions-004-service-role-bypass.json" + }, + { + "experiment": "codex-gpt-5.4-mini-no-skills", + "experimentSuite": "no-skills", + "experimentDisplay": { + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-5.4-mini", + "reasoningEffort": "medium" + }, + "eval": "build-functions-005-dual-auth-user-secret", + "stage": "build", + "product": [ + "edge-functions", "auth", "database" ], @@ -12087,6 +13400,121 @@ "attempts": 1, "sourcePath": "codex-gpt-5.4-mini-no-skills/resolve-security-002-rls-cross-tenant-leak.json" }, + { + "experiment": "codex-gpt-5.6", + "experimentSuite": "benchmark", + "experimentDisplay": { + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-5.6-sol", + "reasoningEffort": "low" + }, + "eval": "build-auth-001-email-password-flow", + "stage": "build", + "product": [ + "auth", + "database" + ], + "topic": [ + "sdk", + "rls" + ], + "suite": "benchmark", + "interface": "cli", + "cliVersion": "2.109.1", + "passed": true, + "checks": [ + { + "name": "auth module loads and the driver completes", + "passed": true, + "notes": "driver produced a result" + }, + { + "name": "signUp creates the account and returns its user id", + "passed": true, + "notes": "db user eebf9d03-9ab8-42c4-a451-e20e7445b43d, signUp returned {\"userId\":\"eebf9d03-9ab8-42c4-a451-e20e7445b43d\"}" + }, + { + "name": "signup metadata reaches the profile (display name)", + "passed": true, + "notes": "profiles.display_name = \"Alex Doe\"" + }, + { + "name": "wrong password is rejected gracefully (no throw, no session)", + "passed": true, + "notes": "{\"error\":\"Invalid login credentials\"}" + }, + { + "name": "signIn with the right password returns the user id", + "passed": true, + "notes": "{\"userId\":\"eebf9d03-9ab8-42c4-a451-e20e7445b43d\"}" + }, + { + "name": "getMyProfile returns the signed-in user's profile", + "passed": true, + "notes": "{\"displayName\":\"Alex Doe\",\"plan\":\"free\"}" + }, + { + "name": "app code does not use the secret / service-role key", + "passed": true, + "notes": "no secret-key references found" + }, + { + "name": "implementation uses @supabase/supabase-js", + "passed": true, + "notes": "imports found in: app/src/auth.mjs" + } + ], + "skills": { + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase" + ] + }, + "docs": { + "calls": [ + { + "source": "search_docs", + "query": "query { searchDocs(query: \"supabase-js signUp email password options data user metadata signInWithPassword getUser profile table RLS auth uid\", limit: 6) { nodes { __typename title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/troubleshooting/customizing-emails-by-language-KZ_38Q", + "title": "Customizing Emails by Language" + }, + { + "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/auth0", + "title": "Migrate from Auth0 to Supabase Auth" + }, + { + "url": "https://supabase.com/docs/guides/auth/managing-user-data", + "title": "User Management" + }, + { + "url": "https://supabase.com/docs/guides/auth/users", + "title": "Users" + }, + { + "url": "https://supabase.com/docs/guides/auth/oauth-server/token-security", + "title": "Token Security and Row Level Security" + }, + { + "url": "https://supabase.com/docs/guides/auth/general-configuration", + "title": "General configuration" + } + ], + "resultChars": 52870 + } + ] + }, + "prompt": "Our app in `app/` needs accounts. Wire up `app/src/auth.mjs` — the stubs in\nthere describe what each function should do. People sign up with an email,\npassword, and display name, sign back in later, and the app greets them with\ntheir profile.\n\nThe Supabase project for this app is in `supabase/` and already running\nlocally. When you're done, the functions should work for real against it.", + "promptSourcePath": "evals/build-auth-001-email-password-flow/PROMPT.md", + "attempts": 1, + "sourcePath": "codex-gpt-5.6/build-auth-001-email-password-flow.json" + }, { "experiment": "codex-gpt-5.6", "experimentSuite": "benchmark", @@ -13017,37 +14445,250 @@ "modelId": "gpt-5.6-sol", "reasoningEffort": "low" }, - "eval": "build-database-001-migrate-postgres-to-supabase", + "eval": "build-dataapi-001-relational-report", "stage": "build", "product": [ + "data-api", "database" ], "topic": [ - "migrations" + "sdk" ], "suite": "benchmark", "interface": "cli", - "passed": true, + "cliVersion": "2.109.1", + "passed": false, "checks": [ { - "name": "all 3 tables exist (teams, members, tasks)", - "passed": true + "name": "report runs and prints JSON", + "passed": true, + "notes": "exit 0" }, { - "name": "row counts match (teams=5, members=10, tasks=13)", - "passed": true + "name": "report numbers match the database (per customer, sorted)", + "passed": true, + "notes": "expected [{\"customer\":\"Ada Lovelace\",\"orderCount\":2,\"totalCents\":41900,\"topProduct\":\"Keyboard\"},{\"customer\":\"Grace Hopper\",\"orderCount\":2,\"totalCents\":15600,\"topProduct\":\"Cable\"},{\"customer\":\"Linus Pauling\",\"orderCount\":1,\"totalCents\":66500,\"topProduct\":\"Monitor\"}], got [{\"customer\":\"Ada Lovelace\",\"orderCount\":2,\"totalCents\":41900,\"topProduct\":\"Keyboard\"},{\"customer\":\"Grace Hopper\",\"orderCount\":2,\"totalCents\":15600,\"topProduct\":\"Cable\"},{\"customer\":\"Linus Pauling\",\"orderCount\":1,\"totalCents\":66500,\"topProduct\":\"Monitor\"}]" }, { - "name": "foreign key constraints survived the restore", - "passed": true + "name": "tables stay locked down (publishable key reads nothing)", + "passed": true, + "notes": "publishable read errored: permission denied for table customers" }, { - "name": "tasks_team_status_idx index survived the restore", - "passed": true + "name": "implementation uses @supabase/supabase-js", + "passed": false, + "notes": "no @supabase/supabase-js import found — this eval requires the SDK" }, { - "name": "sequences synced (next insert won't conflict with existing IDs)", - "passed": true + "name": "report queries via the Data API, not raw SQL", + "passed": true, + "notes": "no psql / raw Postgres driver usage found" + } + ], + "skills": { + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase" + ] + }, + "docs": { + "calls": [ + { + "source": "search_docs", + "query": "query { searchDocs(query: \"supabase-js JavaScript initialize client secret key server-side select nested relationships joins foreign tables handling errors\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/database/connecting-to-postgres/serverless-drivers", + "title": "Serverless Drivers" + }, + { + "url": "https://supabase.com/docs/guides/troubleshooting/performing-administration-tasks-on-the-server-side-with-the-servicerole-secret-BYM4Fa", + "title": "Performing administration tasks on the server side with a secret key" + }, + { + "url": "https://supabase.com/docs/guides/functions/connect-to-postgres", + "title": "Integrating with Supabase Database (Postgres)" + }, + { + "url": "https://supabase.com/docs/guides/api/automatic-retries-in-supabase-js", + "title": "How to do automatic retries with `supabase-js`" + }, + { + "url": "https://supabase.com/docs/guides/database/joins-and-nesting", + "title": "Querying Joins and Nested tables" + } + ], + "resultChars": 43282 + } + ] + }, + "prompt": "We need the nightly sales report working. `app/report.mjs` has the spec in a\ncomment — it runs in our Node backend worker and prints a JSON summary of what\neach customer has ordered.\n\nThe data lives in the Supabase project in `supabase/` (already running\nlocally). Finish the script and make sure it prints the right numbers.", + "promptSourcePath": "evals/build-dataapi-001-relational-report/PROMPT.md", + "attempts": 2, + "sourcePath": "codex-gpt-5.6/build-dataapi-001-relational-report.json" + }, + { + "experiment": "codex-gpt-5.6", + "experimentSuite": "benchmark", + "experimentDisplay": { + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-5.6-sol", + "reasoningEffort": "low" + }, + "eval": "build-dataapi-002-restock-alert-report", + "stage": "build", + "product": [ + "data-api", + "database" + ], + "topic": [ + "sdk" + ], + "suite": "benchmark", + "interface": "cli", + "cliVersion": "2.109.1", + "passed": false, + "checks": [ + { + "name": "report runs and prints JSON", + "passed": true, + "notes": "exit 0" + }, + { + "name": "alerts match the database (below threshold, sorted)", + "passed": true, + "notes": "expected [{\"warehouse\":\"North DC\",\"product\":\"Gizmo\",\"quantity\":3,\"reorderThreshold\":10,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"North DC\",\"product\":\"Widget\",\"quantity\":5,\"reorderThreshold\":20,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"South DC\",\"product\":\"Gadget\",\"quantity\":2,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"},{\"warehouse\":\"West DC\",\"product\":\"Gadget\",\"quantity\":0,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"}], got [{\"warehouse\":\"North DC\",\"product\":\"Gizmo\",\"quantity\":3,\"reorderThreshold\":10,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"North DC\",\"product\":\"Widget\",\"quantity\":5,\"reorderThreshold\":20,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"South DC\",\"product\":\"Gadget\",\"quantity\":2,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"},{\"warehouse\":\"West DC\",\"product\":\"Gadget\",\"quantity\":0,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"}]" + }, + { + "name": "tables stay locked down (publishable key reads nothing)", + "passed": true, + "notes": "publishable read errored: permission denied for table inventory" + }, + { + "name": "implementation uses @supabase/supabase-js", + "passed": false, + "notes": "no @supabase/supabase-js import found — this eval requires the SDK" + }, + { + "name": "report queries via the Data API, not raw SQL", + "passed": true, + "notes": "no psql / raw Postgres driver usage found" + } + ], + "skills": { + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase" + ] + }, + "docs": { + "calls": [ + { + "source": "search_docs", + "query": "query { searchDocs(query: \"supabase-js JavaScript select foreign tables joins filter referenced table column order nested relation\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/database/joins-and-nesting", + "title": "Querying Joins and Nested tables" + }, + { + "url": "https://supabase.com/docs/reference/javascript/using-modifiers-order" + }, + { + "url": "https://supabase.com/docs/guides/api/sql-to-api", + "title": "Converting SQL to JavaScript API" + }, + { + "url": "https://supabase.com/docs/guides/ai/engineering-for-scale", + "title": "Engineering for Scale" + } + ], + "resultChars": 32269 + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"Supabase secret key backend apikey header Authorization sb_secret JavaScript Data API\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", + "title": "New API Keys and Asymmetric Authentication" + }, + { + "url": "https://supabase.com/docs/guides/functions/auth", + "title": "Securing Edge Functions" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-nextjs", + "title": "Build a User Management App with Next.js" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-angular", + "title": "Build a User Management App with Angular" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-ionic-angular", + "title": "Build a User Management App with Ionic Angular" + } + ], + "resultChars": 131802 + } + ] + }, + "prompt": "Purchasing needs a restock alert. `app/restock.mjs` has the spec in a\ncomment — it runs in our Node backend worker and prints a JSON list of what\nneeds reordering, with who to email about it.\n\nThe data lives in the Supabase project in `supabase/` (already running\nlocally). Finish the script and make sure it prints the right alerts.", + "promptSourcePath": "evals/build-dataapi-002-restock-alert-report/PROMPT.md", + "attempts": 2, + "sourcePath": "codex-gpt-5.6/build-dataapi-002-restock-alert-report.json" + }, + { + "experiment": "codex-gpt-5.6", + "experimentSuite": "benchmark", + "experimentDisplay": { + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-5.6-sol", + "reasoningEffort": "low" + }, + "eval": "build-database-001-migrate-postgres-to-supabase", + "stage": "build", + "product": [ + "database" + ], + "topic": [ + "migrations" + ], + "suite": "benchmark", + "interface": "cli", + "passed": true, + "checks": [ + { + "name": "all 3 tables exist (teams, members, tasks)", + "passed": true + }, + { + "name": "row counts match (teams=5, members=10, tasks=13)", + "passed": true + }, + { + "name": "foreign key constraints survived the restore", + "passed": true + }, + { + "name": "tasks_team_status_idx index survived the restore", + "passed": true + }, + { + "name": "sequences synced (next insert won't conflict with existing IDs)", + "passed": true } ], "skills": { @@ -15133,6 +16774,156 @@ "attempts": 1, "sourcePath": "codex-gpt-5.6/resolve-security-002-rls-cross-tenant-leak.json" }, + { + "experiment": "codex-gpt-5.6-no-skills", + "experimentSuite": "no-skills", + "experimentDisplay": { + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-5.6-sol", + "reasoningEffort": "low" + }, + "eval": "build-auth-001-email-password-flow", + "stage": "build", + "product": [ + "auth", + "database" + ], + "topic": [ + "sdk", + "rls" + ], + "suite": "benchmark", + "interface": "cli", + "cliVersion": "2.109.1", + "passed": true, + "checks": [ + { + "name": "auth module loads and the driver completes", + "passed": true, + "notes": "driver produced a result" + }, + { + "name": "signUp creates the account and returns its user id", + "passed": true, + "notes": "db user 483dfe8d-dc9c-4bd6-8d03-7d7456b49080, signUp returned {\"userId\":\"483dfe8d-dc9c-4bd6-8d03-7d7456b49080\"}" + }, + { + "name": "signup metadata reaches the profile (display name)", + "passed": true, + "notes": "profiles.display_name = \"Alex Doe\"" + }, + { + "name": "wrong password is rejected gracefully (no throw, no session)", + "passed": true, + "notes": "{\"error\":\"Invalid login credentials\"}" + }, + { + "name": "signIn with the right password returns the user id", + "passed": true, + "notes": "{\"userId\":\"483dfe8d-dc9c-4bd6-8d03-7d7456b49080\"}" + }, + { + "name": "getMyProfile returns the signed-in user's profile", + "passed": true, + "notes": "{\"displayName\":\"Alex Doe\",\"plan\":\"free\"}" + }, + { + "name": "app code does not use the secret / service-role key", + "passed": true, + "notes": "no secret-key references found" + }, + { + "name": "implementation uses @supabase/supabase-js", + "passed": true, + "notes": "imports found in: app/src/auth.mjs" + } + ], + "skills": { + "available": [], + "loaded": [] + }, + "docs": { + "calls": [ + { + "source": "search_docs", + "query": "query { searchDocs(query: \"supabase-js createClient auth signUp email password options data signInWithPassword getUser select single profile\", limit: 5) { nodes { ... on Guide { title href content } ... on ClientLibraryFunctionReference { title href content language methodName } } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/auth0", + "title": "Migrate from Auth0 to Supabase Auth" + }, + { + "url": "https://supabase.com/docs/guides/auth/passwords", + "title": "Password-based Auth" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-saml-sso", + "title": "Configure SAML SSO" + }, + { + "url": "https://supabase.com/docs/guides/auth", + "title": "Auth" + }, + { + "url": "https://supabase.com/docs/guides/auth/auth-email-passwordless", + "title": "Passwordless email logins" + } + ], + "resultChars": 102589 + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"JavaScript signUp email password options data user metadata signInWithPassword select single\", limit: 10) { nodes { ... on ClientLibraryFunctionReference { title href content language methodName } ... on Guide { title href content } } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/reference/javascript/auth-signinwithpassword" + }, + { + "url": "https://supabase.com/docs/reference/dart/auth-signinwithpassword" + }, + { + "url": "https://supabase.com/docs/guides/auth/passwords", + "title": "Password-based Auth" + }, + { + "url": "https://supabase.com/docs/reference/swift/auth-signinwithpassword", + "title": "signInWithPassword()" + }, + { + "url": "https://supabase.com/docs/guides/auth/auth-email-passwordless", + "title": "Passwordless email logins" + }, + { + "url": "https://supabase.com/docs/reference/csharp/auth-signinwithpassword", + "title": "SignIn(email, password)" + }, + { + "url": "https://supabase.com/docs/reference/javascript/auth-signinwithpasskey" + }, + { + "url": "https://supabase.com/docs/reference/swift/auth-signinwithpasskey", + "title": "signInWithPasskey()" + }, + { + "url": "https://supabase.com/docs/guides/auth/passkeys", + "title": "Passkey authentication" + }, + { + "url": "https://supabase.com/docs/reference/dart/auth-signinwithpasskey" + } + ], + "resultChars": 75745 + } + ] + }, + "prompt": "Our app in `app/` needs accounts. Wire up `app/src/auth.mjs` — the stubs in\nthere describe what each function should do. People sign up with an email,\npassword, and display name, sign back in later, and the app greets them with\ntheir profile.\n\nThe Supabase project for this app is in `supabase/` and already running\nlocally. When you're done, the functions should work for real against it.", + "promptSourcePath": "evals/build-auth-001-email-password-flow/PROMPT.md", + "attempts": 1, + "sourcePath": "codex-gpt-5.6-no-skills/build-auth-001-email-password-flow.json" + }, { "experiment": "codex-gpt-5.6-no-skills", "experimentSuite": "no-skills", @@ -15455,6 +17246,214 @@ "attempts": 1, "sourcePath": "codex-gpt-5.6-no-skills/build-cli-003-pg-cron-queue-workflow.json" }, + { + "experiment": "codex-gpt-5.6-no-skills", + "experimentSuite": "no-skills", + "experimentDisplay": { + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-5.6-sol", + "reasoningEffort": "low" + }, + "eval": "build-dataapi-001-relational-report", + "stage": "build", + "product": [ + "data-api", + "database" + ], + "topic": [ + "sdk" + ], + "suite": "benchmark", + "interface": "cli", + "cliVersion": "2.109.1", + "passed": false, + "checks": [ + { + "name": "report runs and prints JSON", + "passed": true, + "notes": "exit 0" + }, + { + "name": "report numbers match the database (per customer, sorted)", + "passed": true, + "notes": "expected [{\"customer\":\"Ada Lovelace\",\"orderCount\":2,\"totalCents\":41900,\"topProduct\":\"Keyboard\"},{\"customer\":\"Grace Hopper\",\"orderCount\":2,\"totalCents\":15600,\"topProduct\":\"Cable\"},{\"customer\":\"Linus Pauling\",\"orderCount\":1,\"totalCents\":66500,\"topProduct\":\"Monitor\"}], got [{\"customer\":\"Ada Lovelace\",\"orderCount\":2,\"totalCents\":41900,\"topProduct\":\"Keyboard\"},{\"customer\":\"Grace Hopper\",\"orderCount\":2,\"totalCents\":15600,\"topProduct\":\"Cable\"},{\"customer\":\"Linus Pauling\",\"orderCount\":1,\"totalCents\":66500,\"topProduct\":\"Monitor\"}]" + }, + { + "name": "tables stay locked down (publishable key reads nothing)", + "passed": true, + "notes": "publishable read errored: permission denied for table customers" + }, + { + "name": "implementation uses @supabase/supabase-js", + "passed": false, + "notes": "no @supabase/supabase-js import found — this eval requires the SDK" + }, + { + "name": "report queries via the Data API, not raw SQL", + "passed": true, + "notes": "no psql / raw Postgres driver usage found" + } + ], + "skills": { + "available": [], + "loaded": [] + }, + "docs": { + "calls": [ + { + "source": "search_docs", + "query": "query { searchDocs(query: \"Supabase secret key REST API apikey Authorization header sb_secret backend\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", + "title": "Migrating to publishable and secret API keys" + }, + { + "url": "https://supabase.com/docs/guides/functions/auth", + "title": "Securing Edge Functions" + }, + { + "url": "https://supabase.com/docs/guides/local-development/cli/getting-started", + "title": "Supabase CLI" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", + "title": "New API Keys and Asymmetric Authentication" + }, + { + "url": "https://supabase.com/docs/guides/api/creating-routes", + "title": "Creating API Routes" + } + ], + "resultChars": 57476 + } + ] + }, + "prompt": "We need the nightly sales report working. `app/report.mjs` has the spec in a\ncomment — it runs in our Node backend worker and prints a JSON summary of what\neach customer has ordered.\n\nThe data lives in the Supabase project in `supabase/` (already running\nlocally). Finish the script and make sure it prints the right numbers.", + "promptSourcePath": "evals/build-dataapi-001-relational-report/PROMPT.md", + "attempts": 2, + "sourcePath": "codex-gpt-5.6-no-skills/build-dataapi-001-relational-report.json" + }, + { + "experiment": "codex-gpt-5.6-no-skills", + "experimentSuite": "no-skills", + "experimentDisplay": { + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-5.6-sol", + "reasoningEffort": "low" + }, + "eval": "build-dataapi-002-restock-alert-report", + "stage": "build", + "product": [ + "data-api", + "database" + ], + "topic": [ + "sdk" + ], + "suite": "benchmark", + "interface": "cli", + "cliVersion": "2.109.1", + "passed": false, + "checks": [ + { + "name": "report runs and prints JSON", + "passed": true, + "notes": "exit 0" + }, + { + "name": "alerts match the database (below threshold, sorted)", + "passed": true, + "notes": "expected [{\"warehouse\":\"North DC\",\"product\":\"Gizmo\",\"quantity\":3,\"reorderThreshold\":10,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"North DC\",\"product\":\"Widget\",\"quantity\":5,\"reorderThreshold\":20,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"South DC\",\"product\":\"Gadget\",\"quantity\":2,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"},{\"warehouse\":\"West DC\",\"product\":\"Gadget\",\"quantity\":0,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"}], got [{\"warehouse\":\"North DC\",\"product\":\"Gizmo\",\"quantity\":3,\"reorderThreshold\":10,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"North DC\",\"product\":\"Widget\",\"quantity\":5,\"reorderThreshold\":20,\"supplierEmail\":\"acme@example.com\"},{\"warehouse\":\"South DC\",\"product\":\"Gadget\",\"quantity\":2,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"},{\"warehouse\":\"West DC\",\"product\":\"Gadget\",\"quantity\":0,\"reorderThreshold\":15,\"supplierEmail\":\"parts@example.com\"}]" + }, + { + "name": "tables stay locked down (publishable key reads nothing)", + "passed": true, + "notes": "publishable read errored: permission denied for table inventory" + }, + { + "name": "implementation uses @supabase/supabase-js", + "passed": false, + "notes": "no @supabase/supabase-js import found — this eval requires the SDK" + }, + { + "name": "report queries via the Data API, not raw SQL", + "passed": true, + "notes": "no psql / raw Postgres driver usage found" + } + ], + "skills": { + "available": [], + "loaded": [] + }, + "docs": { + "calls": [ + { + "source": "search_docs", + "query": "query { searchDocs(query: \"supabase-js nested select foreign key relationships inner join filtering related table service_role secret key createClient\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/reference/javascript/oauth-admin-createclient" + }, + { + "url": "https://supabase.com/docs/reference/javascript/oauth-admin-getclient" + }, + { + "url": "https://supabase.com/docs/guides/functions/connect-to-postgres", + "title": "Integrating with Supabase Database (Postgres)" + }, + { + "url": "https://supabase.com/docs/reference/javascript/oauth-admin-updateclient" + }, + { + "url": "https://supabase.com/docs/reference/javascript/oauth-admin-deleteclient" + } + ], + "resultChars": 8555 + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"secret key apikey Authorization header Supabase Data API REST server-side\", limit: 6) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/realtime/getting_started", + "title": "Getting Started with Realtime" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", + "title": "New API Keys and Asymmetric Authentication" + }, + { + "url": "https://supabase.com/docs/guides/functions/auth", + "title": "Securing Edge Functions" + }, + { + "url": "https://supabase.com/docs/guides/api/creating-routes", + "title": "Creating API Routes" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/api-keys", + "title": "Understanding API keys" + }, + { + "url": "https://supabase.com/docs/guides/auth/signing-keys", + "title": "JWT Signing Keys" + } + ], + "resultChars": 96290 + } + ] + }, + "prompt": "Purchasing needs a restock alert. `app/restock.mjs` has the spec in a\ncomment — it runs in our Node backend worker and prints a JSON list of what\nneeds reordering, with who to email about it.\n\nThe data lives in the Supabase project in `supabase/` (already running\nlocally). Finish the script and make sure it prints the right alerts.", + "promptSourcePath": "evals/build-dataapi-002-restock-alert-report/PROMPT.md", + "attempts": 2, + "sourcePath": "codex-gpt-5.6-no-skills/build-dataapi-002-restock-alert-report.json" + }, { "experiment": "codex-gpt-5.6-no-skills", "experimentSuite": "no-skills", diff --git a/apps/web/src/data/regression-eval-results.json b/apps/web/src/data/regression-eval-results.json index 882e2557..cd9f18c1 100644 --- a/apps/web/src/data/regression-eval-results.json +++ b/apps/web/src/data/regression-eval-results.json @@ -484,6 +484,68 @@ "attempts": 1, "sourcePath": "claude-code-sonnet-5/resolve-reliability-001-unhealthy-project-recovery.json" }, + { + "experiment": "claude-code-sonnet-5", + "experimentSuite": "regression", + "experimentDisplay": { + "agent": "claude-code", + "modelProvider": "anthropic", + "modelId": "claude-sonnet-5", + "reasoningEffort": "high" + }, + "eval": "resolve-sdk-001-legacy-key-migration", + "stage": "resolve", + "product": [ + "data-api", + "auth" + ], + "topic": [ + "sdk", + "security" + ], + "suite": "regression", + "interface": "cli", + "cliVersion": "2.109.1", + "passed": true, + "checks": [ + { + "name": "posts script still lists published posts", + "passed": true, + "notes": "got [\"Announcing vector buckets\",\"Realtime broadcast tips\",\"Row level security explained\"]" + }, + { + "name": "stats script still counts drafts (secret key bypasses RLS)", + "passed": true, + "notes": "got {\"drafts\":2}, expected 2 drafts" + }, + { + "name": "legacy anon/service_role JWTs removed from the app", + "passed": true, + "notes": "no legacy JWTs found" + }, + { + "name": "public script does not hold the secret key", + "passed": true, + "notes": "posts.mjs holds no secret-key reference" + } + ], + "skills": { + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase" + ] + }, + "docs": { + "calls": [] + }, + "prompt": "Heads-up from the platform team: the legacy JWT-based API keys (`anon` /\n`service_role`) are going away for our projects soon, in favor of the new\npublishable/secret keys. The little blog tooling app in `app/` still uses the\nlegacy keys.\n\nMigrate it over. Both scripts need to keep working — `npm run posts` and\n`npm run stats` (run them from `app/`). The local Supabase project in\n`supabase/` is already running.", + "promptSourcePath": "evals/resolve-sdk-001-legacy-key-migration/PROMPT.md", + "attempts": 1, + "sourcePath": "claude-code-sonnet-5/resolve-sdk-001-legacy-key-migration.json" + }, { "experiment": "claude-code-sonnet-5", "experimentSuite": "regression", @@ -976,6 +1038,63 @@ "attempts": 2, "sourcePath": "claude-code-sonnet-5-no-skills/resolve-reliability-001-unhealthy-project-recovery.json" }, + { + "experiment": "claude-code-sonnet-5-no-skills", + "experimentSuite": "regression", + "experimentDisplay": { + "agent": "claude-code", + "modelProvider": "anthropic", + "modelId": "claude-sonnet-5", + "reasoningEffort": "high" + }, + "eval": "resolve-sdk-001-legacy-key-migration", + "stage": "resolve", + "product": [ + "data-api", + "auth" + ], + "topic": [ + "sdk", + "security" + ], + "suite": "regression", + "interface": "cli", + "cliVersion": "2.109.1", + "passed": true, + "checks": [ + { + "name": "posts script still lists published posts", + "passed": true, + "notes": "got [\"Announcing vector buckets\",\"Realtime broadcast tips\",\"Row level security explained\"]" + }, + { + "name": "stats script still counts drafts (secret key bypasses RLS)", + "passed": true, + "notes": "got {\"drafts\":2}, expected 2 drafts" + }, + { + "name": "legacy anon/service_role JWTs removed from the app", + "passed": true, + "notes": "no legacy JWTs found" + }, + { + "name": "public script does not hold the secret key", + "passed": true, + "notes": "posts.mjs holds no secret-key reference" + } + ], + "skills": { + "available": [], + "loaded": [] + }, + "docs": { + "calls": [] + }, + "prompt": "Heads-up from the platform team: the legacy JWT-based API keys (`anon` /\n`service_role`) are going away for our projects soon, in favor of the new\npublishable/secret keys. The little blog tooling app in `app/` still uses the\nlegacy keys.\n\nMigrate it over. Both scripts need to keep working — `npm run posts` and\n`npm run stats` (run them from `app/`). The local Supabase project in\n`supabase/` is already running.", + "promptSourcePath": "evals/resolve-sdk-001-legacy-key-migration/PROMPT.md", + "attempts": 1, + "sourcePath": "claude-code-sonnet-5-no-skills/resolve-sdk-001-legacy-key-migration.json" + }, { "experiment": "claude-code-sonnet-5-no-skills", "experimentSuite": "regression", diff --git a/evals/build-auth-001-email-password-flow/EVAL.ts b/evals/build-auth-001-email-password-flow/EVAL.ts new file mode 100644 index 00000000..f658d512 --- /dev/null +++ b/evals/build-auth-001-email-password-flow/EVAL.ts @@ -0,0 +1,238 @@ +import { + type CheckResult, + type LocalStackEvalContext, + type LocalStackScorer, +} from '@supabase-evals/core'; + +// Email+password auth benchmark: the prompt asks the agent to finish the +// app's auth layer (app/src/auth.mjs) against the running local stack and +// never names supabase-js — the "uses @supabase/supabase-js" check is +// GATING. On top of SDK discovery it measures driving auth correctly: +// passing the display name as signup user metadata (the seeded profiles +// trigger reads raw_user_meta_data), handling bad credentials gracefully, +// and reading the RLS-scoped profile with the session actually attached. + +const APP_DIR = 'app'; +const DRIVER = 'eval-driver.mjs'; +const DRIVER_MARKER = '___EVAL_DRIVER___'; + +// Runs inside the sandbox, in one process, exactly like the app would use +// the module: sign up, fail a sign-in, sign in, read the profile. +const DRIVER_SOURCE = ` +import { signUp, signIn, getMyProfile } from './src/auth.mjs'; + +const [email, password, displayName, wrongPassword] = process.argv.slice(2); +const out = {}; +const step = async (name, fn) => { + try { + out[name] = await fn(); + } catch (error) { + out[name] = { + threw: String(error instanceof Error ? error.message : error), + }; + } +}; +await step('signUp', () => signUp(email, password, displayName)); +await step('signInWrong', () => signIn(email, wrongPassword)); +await step('signIn', () => signIn(email, password)); +await step('profile', () => getMyProfile()); +console.log('${DRIVER_MARKER}' + JSON.stringify(out)); +process.exit(0); +`; + +interface DriverStep { + userId?: unknown; + displayName?: unknown; + plan?: unknown; + error?: unknown; + threw?: unknown; +} + +const scorer: LocalStackScorer = async (ctx) => { + const checks: CheckResult[] = []; + try { + // Unique suffix keeps signup emails collision-free across attempts. + const suffix = Date.now().toString(36); + const email = `alex-${suffix}@example.com`; + const password = 'correct-horse-battery'; + const wrongPassword = 'wrong-horse-battery'; + const displayName = 'Alex Doe'; + + const status = await readStatus(ctx); + const apiUrl = str(status.API_URL); + const publishableKey = str(status.PUBLISHABLE_KEY); + if (!apiUrl || !publishableKey) { + return fail( + 'read stack config from `supabase status`', + `missing API_URL/PUBLISHABLE_KEY — is the stack running on a new-enough CLI? got keys: ${Object.keys(status).join(', ')}` + ); + } + + const written = await writeDriver(ctx); + if (!written.ok) { + return fail( + 'installed the eval driver', + written.stderr.trim() || written.stdout.trim() + ); + } + const run = await ctx.exec( + `cd ${APP_DIR} && SUPABASE_URL="${apiUrl}" SUPABASE_PUBLISHABLE_KEY="${publishableKey}" ` + + `node ${DRIVER} "${email}" "${password}" "${displayName}" "${wrongPassword}"`, + { timeoutMs: 60_000 } + ); + const out = parseDriverOutput(run.stdout); + checks.push({ + name: 'auth module loads and the driver completes', + passed: out !== undefined, + notes: out + ? 'driver produced a result' + : `no driver output — ${preview(run.stderr || run.stdout)}`, + }); + + const signUp = (out?.signUp ?? {}) as DriverStep; + const signInWrong = (out?.signInWrong ?? {}) as DriverStep; + const signIn = (out?.signIn ?? {}) as DriverStep; + const profile = (out?.profile ?? {}) as DriverStep; + + // Ground truth from the database (superuser query bypasses RLS). + const { rows: userRows } = await ctx.query( + `select u.id::text as id, p.display_name, p.plan + from auth.users u + left join public.profiles p on p.id = u.id + where u.email = '${email}'` + ); + const dbUser = userRows[0]; + + checks.push({ + name: 'signUp creates the account and returns its user id', + passed: !!dbUser && !!signUp.userId && signUp.userId === dbUser.id, + notes: dbUser + ? `db user ${dbUser.id}, signUp returned ${JSON.stringify(signUp)}` + : 'no auth.users row for the signup email', + }); + + // The seeded trigger falls back to the email local part, so the real + // display name only arrives if signUp sent it as user metadata. + checks.push({ + name: 'signup metadata reaches the profile (display name)', + passed: dbUser?.display_name === displayName, + notes: `profiles.display_name = ${JSON.stringify(dbUser?.display_name ?? null)}`, + }); + + checks.push({ + name: 'wrong password is rejected gracefully (no throw, no session)', + passed: !!signInWrong.error && !signInWrong.userId && !signInWrong.threw, + notes: JSON.stringify(signInWrong), + }); + + checks.push({ + name: 'signIn with the right password returns the user id', + passed: !!dbUser && signIn.userId === dbUser.id, + notes: JSON.stringify(signIn), + }); + + checks.push({ + name: "getMyProfile returns the signed-in user's profile", + passed: profile.displayName === displayName && profile.plan === 'free', + notes: JSON.stringify(profile), + }); + + // Client-side code: RLS + the publishable key are enough; the secret / + // service-role key must not appear anywhere in the app. + const secretScan = await ctx.exec( + `grep -rlE --exclude-dir=node_modules 'sb_secret_|SERVICE_ROLE' ${APP_DIR} || true` + ); + checks.push({ + name: 'app code does not use the secret / service-role key', + passed: secretScan.stdout.trim() === '', + notes: secretScan.stdout.trim() || 'no secret-key references found', + }); + + // GATING: the auth layer must be built on supabase-js, even though the + // prompt never names it. + checks.push(await sdkUsageCheck(ctx)); + + return { passed: checks.every((c) => c.passed), checks }; + } catch (error) { + const msg = error instanceof Error ? error.message : String(error); + checks.push({ + name: 'scorer completed without errors', + passed: false, + notes: msg, + }); + return { passed: false, checks }; + } +}; + +export default scorer; + +function writeDriver(ctx: LocalStackEvalContext) { + const encoded = Buffer.from(DRIVER_SOURCE, 'utf-8').toString('base64'); + return ctx.exec(`echo ${encoded} | base64 -d > ${APP_DIR}/${DRIVER}`); +} + +function parseDriverOutput( + stdout: string +): Record | undefined { + const line = stdout + .split('\n') + .find((candidate) => candidate.includes(DRIVER_MARKER)); + if (!line) return undefined; + try { + return JSON.parse( + line.slice(line.indexOf(DRIVER_MARKER) + DRIVER_MARKER.length) + ); + } catch { + return undefined; + } +} + +/** + * GATING: some app code file must genuinely import @supabase/supabase-js — + * we match the quoted module specifier, not a bare mention in a comment. + */ +async function sdkUsageCheck(ctx: LocalStackEvalContext): Promise { + const NAME = 'implementation uses @supabase/supabase-js'; + const scan = await ctx.exec( + `grep -rlE --exclude-dir=node_modules --include='*.mjs' --include='*.js' --include='*.cjs' --include='*.ts' ` + + `"['\\"](npm:)?@supabase/supabase-js" ${APP_DIR} || true` + ); + const files = scan.stdout.trim(); + return { + name: NAME, + passed: files !== '', + notes: files + ? `imports found in: ${files.replace(/\s+/g, ', ')}` + : 'no @supabase/supabase-js import found — this eval requires the SDK', + }; +} + +function str(value: unknown): string | undefined { + return typeof value === 'string' && value.length > 0 ? value : undefined; +} + +function preview(body: string): string { + return body.replace(/\s+/g, ' ').slice(0, 160); +} + +function fail( + name: string, + notes: string +): { passed: false; checks: CheckResult[] } { + return { passed: false, checks: [{ name, passed: false, notes }] }; +} + +/** Parse `supabase status -o json` for the stack's URL and keys. */ +async function readStatus( + ctx: LocalStackEvalContext +): Promise> { + const res = await ctx.exec('supabase status -o json'); + const start = res.stdout.indexOf('{'); + const end = res.stdout.lastIndexOf('}'); + if (start === -1 || end <= start) { + throw new Error( + `could not read \`supabase status\`: ${res.stderr || res.stdout}` + ); + } + return JSON.parse(res.stdout.slice(start, end + 1)); +} diff --git a/evals/build-auth-001-email-password-flow/PROMPT.md b/evals/build-auth-001-email-password-flow/PROMPT.md new file mode 100644 index 00000000..9061dada --- /dev/null +++ b/evals/build-auth-001-email-password-flow/PROMPT.md @@ -0,0 +1,31 @@ +--- +stage: build +suite: benchmark +interface: cli +cliVersion: 2.109.1 +product: + - auth + - database +topic: + - sdk + - rls +services: + - gotrue + - kong + - postgrest +projectRunning: true +motivation: >- + The signup → profile-trigger flow is a recurring pain point + (supabase/supabase#37497, supabase/supabase#35997, and the canonical + pattern in https://supabase.com/docs/guides/auth/managing-user-data); + auth is also the largest supabase-js surface with no dedicated eval + coverage. +--- + +Our app in `app/` needs accounts. Wire up `app/src/auth.mjs` — the stubs in +there describe what each function should do. People sign up with an email, +password, and display name, sign back in later, and the app greets them with +their profile. + +The Supabase project for this app is in `supabase/` and already running +locally. When you're done, the functions should work for real against it. diff --git a/evals/build-auth-001-email-password-flow/README.md b/evals/build-auth-001-email-password-flow/README.md new file mode 100644 index 00000000..5a238bf0 --- /dev/null +++ b/evals/build-auth-001-email-password-flow/README.md @@ -0,0 +1,19 @@ +# build-auth-001-email-password-flow + +Benchmark for supabase-js auth flows. The prompt is a casual "our app needs +accounts" ask pointing at stubbed functions in `local/app/src/auth.mjs` +(signUp / signIn / getMyProfile) — it never names supabase-js; the +"uses @supabase/supabase-js" check is GATING, like its precedent in +build-functions-005. + +The seed teaches through data rather than the prompt: the `profiles` trigger +falls back to the email local part unless the signup sends `display_name` +as user metadata, so the "display name reaches the profile" check only +passes when the agent wires `signUp` with `options.data`. RLS on `profiles` +plus the publishable key make the client-side path the only sanctioned one; +a "no secret key in app code" check guards the boundary. + +Scoring installs a small driver (`app/eval-driver.mjs`) that imports the +agent's module and exercises the contract in one process — sign up, wrong +password (must not throw), correct sign-in, profile read — then verifies +results against the database with the superuser connection. diff --git a/evals/build-auth-001-email-password-flow/local/app/package.json b/evals/build-auth-001-email-password-flow/local/app/package.json new file mode 100644 index 00000000..ccbe7c7e --- /dev/null +++ b/evals/build-auth-001-email-password-flow/local/app/package.json @@ -0,0 +1,5 @@ +{ + "name": "acme-app", + "private": true, + "type": "module" +} diff --git a/evals/build-auth-001-email-password-flow/local/app/src/auth.mjs b/evals/build-auth-001-email-password-flow/local/app/src/auth.mjs new file mode 100644 index 00000000..2e312295 --- /dev/null +++ b/evals/build-auth-001-email-password-flow/local/app/src/auth.mjs @@ -0,0 +1,36 @@ +// Auth layer for the app. The rest of the app calls these three functions; +// wire them up to our Supabase project (it's running locally — see +// ../../supabase). Connection settings come from the environment: +// SUPABASE_URL and SUPABASE_PUBLISHABLE_KEY. +// +// This module is used from client-side code, so it must only ever hold the +// publishable (client) key. + +/** + * Create an account with email + password. `displayName` should end up as + * the user's profile display name. + * + * Resolves to `{ userId }` on success, or `{ error: string }` on failure. + */ +export async function signUp(email, password, displayName) { + throw new Error('TODO: implement signUp'); +} + +/** + * Sign in with email + password. + * + * Resolves to `{ userId }` on success, or `{ error: string }` on failure + * (e.g. wrong password) — it must not throw for bad credentials. + */ +export async function signIn(email, password) { + throw new Error('TODO: implement signIn'); +} + +/** + * The currently signed-in user's profile from the `profiles` table, as + * `{ displayName, plan }`. Resolves to `{ error: string }` when nobody is + * signed in. + */ +export async function getMyProfile() { + throw new Error('TODO: implement getMyProfile'); +} diff --git a/evals/build-auth-001-email-password-flow/local/supabase/config.toml b/evals/build-auth-001-email-password-flow/local/supabase/config.toml new file mode 100644 index 00000000..275fb4b8 --- /dev/null +++ b/evals/build-auth-001-email-password-flow/local/supabase/config.toml @@ -0,0 +1,165 @@ +project_id = "sandbox-auth-flow" + +[api] +enabled = true +port = 54321 +schemas = ["public", "graphql_public"] +extra_search_path = ["public", "extensions"] +max_rows = 1000 + +[api.tls] +enabled = false + +[db] +port = 54322 +shadow_port = 54320 +major_version = 17 + +[db.pooler] +enabled = false +port = 54329 +pool_mode = "transaction" +default_pool_size = 20 +max_client_conn = 100 + +[db.migrations] +enabled = true +schema_paths = [] + +[db.seed] +enabled = false + +[realtime] +enabled = true + +[studio] +enabled = true +port = 54323 +api_url = "http://127.0.0.1" +openai_api_key = "env(OPENAI_API_KEY)" + +[inbucket] +enabled = true +port = 54324 + +[storage] +enabled = true +file_size_limit = "50MiB" + +[storage.s3_protocol] +enabled = true + +[storage.analytics] +enabled = false +max_namespaces = 5 +max_tables = 10 +max_catalogs = 2 + +[storage.vector] +enabled = false +max_buckets = 10 +max_indexes = 5 + +[auth] +enabled = true +site_url = "http://127.0.0.1:3000" +additional_redirect_urls = ["https://127.0.0.1:3000"] +jwt_expiry = 3600 +enable_refresh_token_rotation = true +refresh_token_reuse_interval = 10 +enable_signup = true +enable_anonymous_sign_ins = false +enable_manual_linking = false +minimum_password_length = 6 +password_requirements = "" + +[auth.rate_limit] +email_sent = 2 +sms_sent = 30 +anonymous_users = 30 +token_refresh = 150 +sign_in_sign_ups = 30 +token_verifications = 30 +web3 = 30 + +[auth.email] +enable_signup = true +double_confirm_changes = true +enable_confirmations = false +secure_password_change = false +max_frequency = "1s" +otp_length = 6 +otp_expiry = 3600 + +[auth.sms] +enable_signup = false +enable_confirmations = false +template = "Your code is {{ .Code }}" +max_frequency = "5s" + +[auth.sms.twilio] +enabled = false +account_sid = "" +message_service_sid = "" +auth_token = "env(SUPABASE_AUTH_SMS_TWILIO_AUTH_TOKEN)" + +[auth.mfa] +max_enrolled_factors = 10 + +[auth.mfa.totp] +enroll_enabled = false +verify_enabled = false + +[auth.mfa.phone] +enroll_enabled = false +verify_enabled = false +otp_length = 6 +template = "Your code is {{ .Code }}" +max_frequency = "5s" + +[auth.external.apple] +enabled = false +client_id = "" +secret = "env(SUPABASE_AUTH_EXTERNAL_APPLE_SECRET)" +redirect_uri = "" +url = "" +skip_nonce_check = false +email_optional = false + +[auth.web3.solana] +enabled = false + +[auth.third_party.firebase] +enabled = false + +[auth.third_party.auth0] +enabled = false + +[auth.third_party.aws_cognito] +enabled = false + +[auth.third_party.clerk] +enabled = false + +[auth.oauth_server] +enabled = false +authorization_url_path = "/oauth/consent" +allow_dynamic_registration = false + +[edge_runtime] +enabled = true +policy = "per_worker" +inspector_port = 8083 +deno_version = 2 + +[analytics] +enabled = true +port = 54327 +backend = "postgres" + +[experimental] +orioledb_version = "" +s3_host = "env(S3_HOST)" +s3_region = "env(S3_REGION)" +s3_access_key = "env(S3_ACCESS_KEY)" +s3_secret_key = "env(S3_SECRET_KEY)" diff --git a/evals/build-auth-001-email-password-flow/local/supabase/migrations/0000_profiles_schema.sql b/evals/build-auth-001-email-password-flow/local/supabase/migrations/0000_profiles_schema.sql new file mode 100644 index 00000000..b0eec56e --- /dev/null +++ b/evals/build-auth-001-email-password-flow/local/supabase/migrations/0000_profiles_schema.sql @@ -0,0 +1,53 @@ +-- Public profile for each account, created automatically on signup. Guarded +-- by row-level security so users can only see and edit their own profile. +create table public.profiles ( + id uuid primary key references auth.users (id) on delete cascade, + display_name text not null, + plan text not null default 'free', + created_at timestamptz not null default now() +); + +alter table public.profiles enable row level security; + +create policy "users can read their own profile" + on public.profiles + for select + to authenticated + using (auth.uid() = id); + +create policy "users can update their own profile" + on public.profiles + for update + to authenticated + using (auth.uid() = id) + with check (auth.uid() = id); + +-- Newer Supabase CLIs no longer auto-grant privileges on migration-created +-- tables, so grant them explicitly. RLS scopes the rows. +grant select, update on public.profiles to authenticated; + +-- Create the profile row when a user signs up. The display name comes from +-- the signup user metadata; if the app doesn't send one, we fall back to the +-- email local part. +create function public.handle_new_user() +returns trigger +language plpgsql +security definer +set search_path = '' +as $$ +begin + insert into public.profiles (id, display_name) + values ( + new.id, + coalesce( + new.raw_user_meta_data ->> 'display_name', + split_part(new.email, '@', 1) + ) + ); + return new; +end; +$$; + +create trigger on_auth_user_created + after insert on auth.users + for each row execute function public.handle_new_user(); diff --git a/evals/build-dataapi-001-relational-report/EVAL.ts b/evals/build-dataapi-001-relational-report/EVAL.ts new file mode 100644 index 00000000..0454fb8c --- /dev/null +++ b/evals/build-dataapi-001-relational-report/EVAL.ts @@ -0,0 +1,214 @@ +import { + type CheckResult, + type LocalStackEvalContext, + type LocalStackScorer, +} from '@supabase-evals/core'; + +// Relational-report benchmark for the Data API: the prompt asks the agent to +// finish a backend reporting script over a seeded relational schema +// (customers → orders → order_items → products) and never names supabase-js — +// the "uses @supabase/supabase-js" check is GATING, and shelling out to +// psql / a raw Postgres driver instead fails. Expected numbers are computed +// from the database at scoring time, so the seed stays the single source of +// truth. + +const APP_DIR = 'app'; +const REPORT = 'report.mjs'; + +interface ReportRow { + customer: unknown; + orderCount: unknown; + totalCents: unknown; + topProduct: unknown; +} + +const EXPECTED_SQL = ` +with per_product as ( + select c.name as customer, p.name as product, + sum(oi.quantity)::int as units + from public.customers c + join public.orders o on o.customer_id = c.id + join public.order_items oi on oi.order_id = o.id + join public.products p on p.id = oi.product_id + group by c.name, p.name +), totals as ( + select c.name as customer, + count(distinct o.id)::int as order_count, + sum(oi.quantity * p.price_cents)::int as total_cents + from public.customers c + join public.orders o on o.customer_id = c.id + join public.order_items oi on oi.order_id = o.id + join public.products p on p.id = oi.product_id + group by c.name +) +select t.customer, + t.order_count, + t.total_cents, + (select pp.product + from per_product pp + where pp.customer = t.customer + order by pp.units desc, pp.product asc + limit 1) as top_product + from totals t + order by t.customer asc +`; + +const scorer: LocalStackScorer = async (ctx) => { + const checks: CheckResult[] = []; + try { + const status = await readStatus(ctx); + const apiUrl = str(status.API_URL); + const secretKey = str(status.SECRET_KEY); + if (!apiUrl || !secretKey) { + return fail( + 'read stack config from `supabase status`', + `missing API_URL/SECRET_KEY — is the stack running on a new-enough CLI? got keys: ${Object.keys(status).join(', ')}` + ); + } + + // Be generous about a missing install step; the eval is about the report, + // not npm. A no-op when the agent already installed dependencies. + await ctx.exec( + `cd ${APP_DIR} && [ -d node_modules ] || npm install --no-audit --no-fund --silent || true`, + { timeoutMs: 180_000 } + ); + + const run = await ctx.exec( + `cd ${APP_DIR} && SUPABASE_URL="${apiUrl}" SUPABASE_SECRET_KEY="${secretKey}" node ${REPORT}`, + { timeoutMs: 60_000 } + ); + const actual = parseReport(run.stdout); + checks.push({ + name: 'report runs and prints JSON', + passed: actual !== undefined, + notes: + actual !== undefined + ? `exit ${run.exitCode}` + : `no JSON array in output — ${preview(run.stderr || run.stdout)}`, + }); + + // Ground truth straight from the seeded database. + const { rows } = await ctx.query(EXPECTED_SQL); + const expected = rows.map((row) => ({ + customer: row.customer, + orderCount: row.order_count, + totalCents: row.total_cents, + topProduct: row.top_product, + })); + const normalized = (actual ?? []).map((row) => ({ + customer: row.customer, + orderCount: row.orderCount, + totalCents: row.totalCents, + topProduct: row.topProduct, + })); + checks.push({ + name: 'report numbers match the database (per customer, sorted)', + passed: JSON.stringify(normalized) === JSON.stringify(expected), + notes: `expected ${JSON.stringify(expected)}, got ${JSON.stringify(normalized)}`, + }); + + // The tables are backend-only: RLS with no policies. The right fix is the + // secret key in the worker — not opening the tables up to client keys. + const client = await ctx.getClient(); + const probe = await client.from('customers').select('id'); + checks.push({ + name: 'tables stay locked down (publishable key reads nothing)', + passed: (probe.data ?? []).length === 0, + notes: probe.error + ? `publishable read errored: ${probe.error.message}` + : `publishable read returned ${(probe.data ?? []).length} rows`, + }); + + // GATING: the report must be built on supabase-js, even though the prompt + // never names it… + checks.push(await sdkUsageCheck(ctx)); + + // …and must actually query through the Data API, not shell out to psql or + // a raw Postgres driver. + const rawSqlScan = await ctx.exec( + `grep -rlE --exclude-dir=node_modules --include='*.mjs' --include='*.js' --include='*.cjs' --include='*.ts' ` + + `"psql|['\\"](pg|postgres|pg-promise)['\\"]" ${APP_DIR} || true` + ); + checks.push({ + name: 'report queries via the Data API, not raw SQL', + passed: rawSqlScan.stdout.trim() === '', + notes: + rawSqlScan.stdout.trim().replace(/\s+/g, ', ') || + 'no psql / raw Postgres driver usage found', + }); + + return { passed: checks.every((c) => c.passed), checks }; + } catch (error) { + const msg = error instanceof Error ? error.message : String(error); + checks.push({ + name: 'scorer completed without errors', + passed: false, + notes: msg, + }); + return { passed: false, checks }; + } +}; + +export default scorer; + +function parseReport(stdout: string): ReportRow[] | undefined { + const start = stdout.indexOf('['); + const end = stdout.lastIndexOf(']'); + if (start === -1 || end <= start) return undefined; + try { + const parsed = JSON.parse(stdout.slice(start, end + 1)); + return Array.isArray(parsed) ? (parsed as ReportRow[]) : undefined; + } catch { + return undefined; + } +} + +/** + * GATING: some app code file must genuinely import @supabase/supabase-js — + * we match the quoted module specifier, not a bare mention in a comment. + */ +async function sdkUsageCheck(ctx: LocalStackEvalContext): Promise { + const NAME = 'implementation uses @supabase/supabase-js'; + const scan = await ctx.exec( + `grep -rlE --exclude-dir=node_modules --include='*.mjs' --include='*.js' --include='*.cjs' --include='*.ts' ` + + `"['\\"](npm:)?@supabase/supabase-js" ${APP_DIR} || true` + ); + const files = scan.stdout.trim(); + return { + name: NAME, + passed: files !== '', + notes: files + ? `imports found in: ${files.replace(/\s+/g, ', ')}` + : 'no @supabase/supabase-js import found — this eval requires the SDK', + }; +} + +function str(value: unknown): string | undefined { + return typeof value === 'string' && value.length > 0 ? value : undefined; +} + +function preview(body: string): string { + return body.replace(/\s+/g, ' ').slice(0, 160); +} + +function fail( + name: string, + notes: string +): { passed: false; checks: CheckResult[] } { + return { passed: false, checks: [{ name, passed: false, notes }] }; +} + +/** Parse `supabase status -o json` for the stack's URL and keys. */ +async function readStatus( + ctx: LocalStackEvalContext +): Promise> { + const res = await ctx.exec('supabase status -o json'); + const start = res.stdout.indexOf('{'); + const end = res.stdout.lastIndexOf('}'); + if (start === -1 || end <= start) { + throw new Error( + `could not read \`supabase status\`: ${res.stderr || res.stdout}` + ); + } + return JSON.parse(res.stdout.slice(start, end + 1)); +} diff --git a/evals/build-dataapi-001-relational-report/PROMPT.md b/evals/build-dataapi-001-relational-report/PROMPT.md new file mode 100644 index 00000000..00957811 --- /dev/null +++ b/evals/build-dataapi-001-relational-report/PROMPT.md @@ -0,0 +1,28 @@ +--- +stage: build +suite: benchmark +interface: cli +cliVersion: 2.109.1 +product: + - data-api + - database +topic: + - sdk +services: + - gotrue + - kong + - postgrest +projectRunning: true +motivation: >- + Relationship embedding is the query-builder surface users trip on most + (supabase/postgrest-js#609, supabase/postgrest-js#611, + supabase/supabase-js#1639), and the most-used supabase-js surface had no + dedicated eval coverage. +--- + +We need the nightly sales report working. `app/report.mjs` has the spec in a +comment — it runs in our Node backend worker and prints a JSON summary of what +each customer has ordered. + +The data lives in the Supabase project in `supabase/` (already running +locally). Finish the script and make sure it prints the right numbers. diff --git a/evals/build-dataapi-001-relational-report/local/app/package.json b/evals/build-dataapi-001-relational-report/local/app/package.json new file mode 100644 index 00000000..81652eb0 --- /dev/null +++ b/evals/build-dataapi-001-relational-report/local/app/package.json @@ -0,0 +1,5 @@ +{ + "name": "reporting-worker", + "private": true, + "type": "module" +} diff --git a/evals/build-dataapi-001-relational-report/local/app/report.mjs b/evals/build-dataapi-001-relational-report/local/app/report.mjs new file mode 100644 index 00000000..757366f8 --- /dev/null +++ b/evals/build-dataapi-001-relational-report/local/app/report.mjs @@ -0,0 +1,22 @@ +// Nightly sales report, run inside our Node backend worker: +// +// node report.mjs +// +// Connection settings come from the environment: SUPABASE_URL and +// SUPABASE_SECRET_KEY (this is trusted backend code). +// +// Print to stdout a JSON array with one entry per customer who has placed at +// least one order, sorted by customer name: +// +// { +// "customer": string, // customer name +// "orderCount": number, // how many orders they placed +// "totalCents": number, // total spent across all their orders +// "topProduct": string // product they bought the most units of +// } +// +// If two products tie on units, topProduct is the alphabetically first one. +// +// TODO: implement +console.error('not implemented'); +process.exit(1); diff --git a/evals/build-dataapi-001-relational-report/local/supabase/.branches/_current_branch b/evals/build-dataapi-001-relational-report/local/supabase/.branches/_current_branch new file mode 100644 index 00000000..88d050b1 --- /dev/null +++ b/evals/build-dataapi-001-relational-report/local/supabase/.branches/_current_branch @@ -0,0 +1 @@ +main \ No newline at end of file diff --git a/evals/build-dataapi-001-relational-report/local/supabase/.temp/cli-latest b/evals/build-dataapi-001-relational-report/local/supabase/.temp/cli-latest new file mode 100644 index 00000000..e9acfb34 --- /dev/null +++ b/evals/build-dataapi-001-relational-report/local/supabase/.temp/cli-latest @@ -0,0 +1 @@ +v2.110.0 \ No newline at end of file diff --git a/evals/build-dataapi-001-relational-report/local/supabase/config.toml b/evals/build-dataapi-001-relational-report/local/supabase/config.toml new file mode 100644 index 00000000..8ca57d9f --- /dev/null +++ b/evals/build-dataapi-001-relational-report/local/supabase/config.toml @@ -0,0 +1,165 @@ +project_id = "sandbox-orders-report" + +[api] +enabled = true +port = 54321 +schemas = ["public", "graphql_public"] +extra_search_path = ["public", "extensions"] +max_rows = 1000 + +[api.tls] +enabled = false + +[db] +port = 54322 +shadow_port = 54320 +major_version = 17 + +[db.pooler] +enabled = false +port = 54329 +pool_mode = "transaction" +default_pool_size = 20 +max_client_conn = 100 + +[db.migrations] +enabled = true +schema_paths = [] + +[db.seed] +enabled = false + +[realtime] +enabled = true + +[studio] +enabled = true +port = 54323 +api_url = "http://127.0.0.1" +openai_api_key = "env(OPENAI_API_KEY)" + +[inbucket] +enabled = true +port = 54324 + +[storage] +enabled = true +file_size_limit = "50MiB" + +[storage.s3_protocol] +enabled = true + +[storage.analytics] +enabled = false +max_namespaces = 5 +max_tables = 10 +max_catalogs = 2 + +[storage.vector] +enabled = false +max_buckets = 10 +max_indexes = 5 + +[auth] +enabled = true +site_url = "http://127.0.0.1:3000" +additional_redirect_urls = ["https://127.0.0.1:3000"] +jwt_expiry = 3600 +enable_refresh_token_rotation = true +refresh_token_reuse_interval = 10 +enable_signup = true +enable_anonymous_sign_ins = false +enable_manual_linking = false +minimum_password_length = 6 +password_requirements = "" + +[auth.rate_limit] +email_sent = 2 +sms_sent = 30 +anonymous_users = 30 +token_refresh = 150 +sign_in_sign_ups = 30 +token_verifications = 30 +web3 = 30 + +[auth.email] +enable_signup = true +double_confirm_changes = true +enable_confirmations = false +secure_password_change = false +max_frequency = "1s" +otp_length = 6 +otp_expiry = 3600 + +[auth.sms] +enable_signup = false +enable_confirmations = false +template = "Your code is {{ .Code }}" +max_frequency = "5s" + +[auth.sms.twilio] +enabled = false +account_sid = "" +message_service_sid = "" +auth_token = "env(SUPABASE_AUTH_SMS_TWILIO_AUTH_TOKEN)" + +[auth.mfa] +max_enrolled_factors = 10 + +[auth.mfa.totp] +enroll_enabled = false +verify_enabled = false + +[auth.mfa.phone] +enroll_enabled = false +verify_enabled = false +otp_length = 6 +template = "Your code is {{ .Code }}" +max_frequency = "5s" + +[auth.external.apple] +enabled = false +client_id = "" +secret = "env(SUPABASE_AUTH_EXTERNAL_APPLE_SECRET)" +redirect_uri = "" +url = "" +skip_nonce_check = false +email_optional = false + +[auth.web3.solana] +enabled = false + +[auth.third_party.firebase] +enabled = false + +[auth.third_party.auth0] +enabled = false + +[auth.third_party.aws_cognito] +enabled = false + +[auth.third_party.clerk] +enabled = false + +[auth.oauth_server] +enabled = false +authorization_url_path = "/oauth/consent" +allow_dynamic_registration = false + +[edge_runtime] +enabled = true +policy = "per_worker" +inspector_port = 8083 +deno_version = 2 + +[analytics] +enabled = true +port = 54327 +backend = "postgres" + +[experimental] +orioledb_version = "" +s3_host = "env(S3_HOST)" +s3_region = "env(S3_REGION)" +s3_access_key = "env(S3_ACCESS_KEY)" +s3_secret_key = "env(S3_SECRET_KEY)" diff --git a/evals/build-dataapi-001-relational-report/local/supabase/migrations/0000_orders_schema.sql b/evals/build-dataapi-001-relational-report/local/supabase/migrations/0000_orders_schema.sql new file mode 100644 index 00000000..29f889c1 --- /dev/null +++ b/evals/build-dataapi-001-relational-report/local/supabase/migrations/0000_orders_schema.sql @@ -0,0 +1,69 @@ +-- Order history for the reporting worker. These tables are backend-only: +-- RLS is enabled with no policies, so client-side (publishable) keys read +-- nothing. Trusted backend code authenticates with the secret key, which +-- bypasses RLS. +create table public.customers ( + id bigint generated always as identity primary key, + name text not null, + email text not null unique +); + +create table public.products ( + id bigint generated always as identity primary key, + name text not null, + price_cents int not null +); + +create table public.orders ( + id bigint generated always as identity primary key, + customer_id bigint not null references public.customers (id), + ordered_at timestamptz not null default now() +); + +create table public.order_items ( + id bigint generated always as identity primary key, + order_id bigint not null references public.orders (id), + product_id bigint not null references public.products (id), + quantity int not null check (quantity > 0) +); + +alter table public.customers enable row level security; +alter table public.products enable row level security; +alter table public.orders enable row level security; +alter table public.order_items enable row level security; + +-- Newer Supabase CLIs no longer auto-grant privileges on migration-created +-- tables. The trusted backend (secret key → service_role) needs the SELECT +-- privilege; client-side roles get nothing, so these tables stay backend-only. +grant select on public.customers, public.products, public.orders, + public.order_items to service_role; + +-- Seed data. Alan has no orders and must not appear in the report. +insert into public.customers (name, email) values + ('Ada Lovelace', 'ada@example.com'), + ('Grace Hopper', 'grace@example.com'), + ('Linus Pauling', 'linus@example.com'), + ('Alan Turing', 'alan@example.com'); + +insert into public.products (name, price_cents) values + ('Keyboard', 4500), + ('Mouse', 2500), + ('Monitor', 32000), + ('Cable', 900); + +insert into public.orders (customer_id, ordered_at) values + (1, '2026-06-01T10:00:00Z'), + (1, '2026-06-14T09:30:00Z'), + (2, '2026-06-03T16:20:00Z'), + (2, '2026-06-20T11:05:00Z'), + (3, '2026-06-08T14:45:00Z'); + +insert into public.order_items (order_id, product_id, quantity) values + (1, 1, 2), -- Ada: 2x Keyboard + (1, 4, 1), -- Ada: 1x Cable + (2, 3, 1), -- Ada: 1x Monitor + (3, 2, 3), -- Grace: 3x Mouse + (4, 1, 1), -- Grace: 1x Keyboard + (4, 4, 4), -- Grace: 4x Cable + (5, 3, 2), -- Linus: 2x Monitor + (5, 2, 1); -- Linus: 1x Mouse diff --git a/evals/build-dataapi-002-restock-alert-report/EVAL.ts b/evals/build-dataapi-002-restock-alert-report/EVAL.ts new file mode 100644 index 00000000..948e6271 --- /dev/null +++ b/evals/build-dataapi-002-restock-alert-report/EVAL.ts @@ -0,0 +1,202 @@ +import { + type CheckResult, + type LocalStackEvalContext, + type LocalStackScorer, +} from '@supabase-evals/core'; + +// Companion to build-dataapi-001-relational-report: same "unnamed SDK, empty +// package.json, backend worker script" shape, different schema (warehouses → +// inventory → products → suppliers) and aggregation (below-threshold restock +// alert, not a sales rollup). Checks whether the SDK-adoption split found +// there (claude-code 4/4, codex 0/4) generalizes or was specific to that +// prompt. The "uses @supabase/supabase-js" check is GATING, and shelling out +// to psql / a raw Postgres driver instead fails. Expected alerts are computed +// from the database at scoring time, so the seed stays the single source of +// truth. + +const APP_DIR = 'app'; +const REPORT = 'restock.mjs'; + +interface AlertRow { + warehouse: unknown; + product: unknown; + quantity: unknown; + reorderThreshold: unknown; + supplierEmail: unknown; +} + +const EXPECTED_SQL = ` +select w.name as warehouse, + p.name as product, + i.quantity, + p.reorder_threshold, + s.email as supplier_email + from public.inventory i + join public.warehouses w on w.id = i.warehouse_id + join public.products p on p.id = i.product_id + join public.suppliers s on s.id = p.supplier_id + where i.quantity < p.reorder_threshold + order by w.name asc, p.name asc +`; + +const scorer: LocalStackScorer = async (ctx) => { + const checks: CheckResult[] = []; + try { + const status = await readStatus(ctx); + const apiUrl = str(status.API_URL); + const secretKey = str(status.SECRET_KEY); + if (!apiUrl || !secretKey) { + return fail( + 'read stack config from `supabase status`', + `missing API_URL/SECRET_KEY — is the stack running on a new-enough CLI? got keys: ${Object.keys(status).join(', ')}` + ); + } + + // Be generous about a missing install step; the eval is about the report, + // not npm. A no-op when the agent already installed dependencies. + await ctx.exec( + `cd ${APP_DIR} && [ -d node_modules ] || npm install --no-audit --no-fund --silent || true`, + { timeoutMs: 180_000 } + ); + + const run = await ctx.exec( + `cd ${APP_DIR} && SUPABASE_URL="${apiUrl}" SUPABASE_SECRET_KEY="${secretKey}" node ${REPORT}`, + { timeoutMs: 60_000 } + ); + const actual = parseReport(run.stdout); + checks.push({ + name: 'report runs and prints JSON', + passed: actual !== undefined, + notes: + actual !== undefined + ? `exit ${run.exitCode}` + : `no JSON array in output — ${preview(run.stderr || run.stdout)}`, + }); + + // Ground truth straight from the seeded database. + const { rows } = await ctx.query(EXPECTED_SQL); + const expected = rows.map((row) => ({ + warehouse: row.warehouse, + product: row.product, + quantity: row.quantity, + reorderThreshold: row.reorder_threshold, + supplierEmail: row.supplier_email, + })); + const normalized = (actual ?? []).map((row) => ({ + warehouse: row.warehouse, + product: row.product, + quantity: row.quantity, + reorderThreshold: row.reorderThreshold, + supplierEmail: row.supplierEmail, + })); + checks.push({ + name: 'alerts match the database (below threshold, sorted)', + passed: JSON.stringify(normalized) === JSON.stringify(expected), + notes: `expected ${JSON.stringify(expected)}, got ${JSON.stringify(normalized)}`, + }); + + // The tables are backend-only: RLS with no policies. The right fix is the + // secret key in the worker — not opening the tables up to client keys. + const client = await ctx.getClient(); + const probe = await client.from('inventory').select('id'); + checks.push({ + name: 'tables stay locked down (publishable key reads nothing)', + passed: (probe.data ?? []).length === 0, + notes: probe.error + ? `publishable read errored: ${probe.error.message}` + : `publishable read returned ${(probe.data ?? []).length} rows`, + }); + + // GATING: the report must be built on supabase-js, even though the prompt + // never names it… + checks.push(await sdkUsageCheck(ctx)); + + // …and must actually query through the Data API, not shell out to psql or + // a raw Postgres driver. + const rawSqlScan = await ctx.exec( + `grep -rlE --exclude-dir=node_modules --include='*.mjs' --include='*.js' --include='*.cjs' --include='*.ts' ` + + `"psql|['\\"](pg|postgres|pg-promise)['\\"]" ${APP_DIR} || true` + ); + checks.push({ + name: 'report queries via the Data API, not raw SQL', + passed: rawSqlScan.stdout.trim() === '', + notes: + rawSqlScan.stdout.trim().replace(/\s+/g, ', ') || + 'no psql / raw Postgres driver usage found', + }); + + return { passed: checks.every((c) => c.passed), checks }; + } catch (error) { + const msg = error instanceof Error ? error.message : String(error); + checks.push({ + name: 'scorer completed without errors', + passed: false, + notes: msg, + }); + return { passed: false, checks }; + } +}; + +export default scorer; + +function parseReport(stdout: string): AlertRow[] | undefined { + const start = stdout.indexOf('['); + const end = stdout.lastIndexOf(']'); + if (start === -1 || end <= start) return undefined; + try { + const parsed = JSON.parse(stdout.slice(start, end + 1)); + return Array.isArray(parsed) ? (parsed as AlertRow[]) : undefined; + } catch { + return undefined; + } +} + +/** + * GATING: some app code file must genuinely import @supabase/supabase-js — + * we match the quoted module specifier, not a bare mention in a comment. + */ +async function sdkUsageCheck(ctx: LocalStackEvalContext): Promise { + const NAME = 'implementation uses @supabase/supabase-js'; + const scan = await ctx.exec( + `grep -rlE --exclude-dir=node_modules --include='*.mjs' --include='*.js' --include='*.cjs' --include='*.ts' ` + + `"['\\"](npm:)?@supabase/supabase-js" ${APP_DIR} || true` + ); + const files = scan.stdout.trim(); + return { + name: NAME, + passed: files !== '', + notes: files + ? `imports found in: ${files.replace(/\s+/g, ', ')}` + : 'no @supabase/supabase-js import found — this eval requires the SDK', + }; +} + +function str(value: unknown): string | undefined { + return typeof value === 'string' && value.length > 0 ? value : undefined; +} + +function preview(body: string): string { + return body.replace(/\s+/g, ' ').slice(0, 160); +} + +function fail( + name: string, + notes: string +): { passed: false; checks: CheckResult[] } { + return { passed: false, checks: [{ name, passed: false, notes }] }; +} + +/** Parse `supabase status -o json` for the stack's URL and keys. */ +async function readStatus( + ctx: LocalStackEvalContext +): Promise> { + const res = await ctx.exec('supabase status -o json'); + const start = res.stdout.indexOf('{'); + const end = res.stdout.lastIndexOf('}'); + if (start === -1 || end <= start) { + throw new Error( + `could not read \`supabase status\`: ${res.stderr || res.stdout}` + ); + } + return JSON.parse(res.stdout.slice(start, end + 1)); +} diff --git a/evals/build-dataapi-002-restock-alert-report/PROMPT.md b/evals/build-dataapi-002-restock-alert-report/PROMPT.md new file mode 100644 index 00000000..5b3d1a90 --- /dev/null +++ b/evals/build-dataapi-002-restock-alert-report/PROMPT.md @@ -0,0 +1,32 @@ +--- +stage: build +suite: benchmark +interface: cli +cliVersion: 2.109.1 +product: + - data-api + - database +topic: + - sdk +services: + - gotrue + - kong + - postgrest +projectRunning: true +motivation: >- + build-dataapi-001-relational-report found every codex variant (0/4) skips + @supabase/supabase-js for a bare backend Data API script, hand-rolling raw + HTTP instead, while every claude-code variant (4/4) reached for it + unprompted. A single scenario isn't enough to tell a real model tendency + from a one-off artifact of that prompt's specific shape — this companion + scenario keeps the same "unnamed SDK, empty package.json, backend worker + script" shape but swaps in an unrelated schema and aggregation (inventory + restock alerts vs. sales report) to check whether the pattern generalizes. +--- + +Purchasing needs a restock alert. `app/restock.mjs` has the spec in a +comment — it runs in our Node backend worker and prints a JSON list of what +needs reordering, with who to email about it. + +The data lives in the Supabase project in `supabase/` (already running +locally). Finish the script and make sure it prints the right alerts. diff --git a/evals/build-dataapi-002-restock-alert-report/README.md b/evals/build-dataapi-002-restock-alert-report/README.md new file mode 100644 index 00000000..e9bb4ba6 --- /dev/null +++ b/evals/build-dataapi-002-restock-alert-report/README.md @@ -0,0 +1,6 @@ +Companion to `build-dataapi-001-relational-report`. Same shape (unnamed SDK, +empty `package.json`, bare backend worker script reading `SUPABASE_URL` / +`SUPABASE_SECRET_KEY` from the env) but a different schema and aggregation +(inventory restock alerts vs. a sales report), to check whether that eval's +SDK-adoption split (claude-code 4/4 vs. codex 0/4) is a real model tendency +or an artifact of that one prompt. diff --git a/evals/build-dataapi-002-restock-alert-report/local/app/package.json b/evals/build-dataapi-002-restock-alert-report/local/app/package.json new file mode 100644 index 00000000..4f4a0bce --- /dev/null +++ b/evals/build-dataapi-002-restock-alert-report/local/app/package.json @@ -0,0 +1,5 @@ +{ + "name": "restock-alert-worker", + "private": true, + "type": "module" +} diff --git a/evals/build-dataapi-002-restock-alert-report/local/app/restock.mjs b/evals/build-dataapi-002-restock-alert-report/local/app/restock.mjs new file mode 100644 index 00000000..1d247baf --- /dev/null +++ b/evals/build-dataapi-002-restock-alert-report/local/app/restock.mjs @@ -0,0 +1,24 @@ +// Restock alert worker, run inside our Node backend worker: +// +// node restock.mjs +// +// Connection settings come from the environment: SUPABASE_URL and +// SUPABASE_SECRET_KEY (this is trusted backend code). +// +// Print to stdout a JSON array with one entry per warehouse/product +// combination that's below its reorder threshold, sorted by warehouse name +// then product name: +// +// { +// "warehouse": string, // warehouse name +// "product": string, // product name +// "quantity": number, // current quantity on hand +// "reorderThreshold": number, // reorder threshold for this product +// "supplierEmail": string // email of the product's supplier +// } +// +// Only include rows where quantity is strictly below the reorder threshold. +// +// TODO: implement +console.error('not implemented'); +process.exit(1); diff --git a/evals/build-dataapi-002-restock-alert-report/local/supabase/.branches/_current_branch b/evals/build-dataapi-002-restock-alert-report/local/supabase/.branches/_current_branch new file mode 100644 index 00000000..88d050b1 --- /dev/null +++ b/evals/build-dataapi-002-restock-alert-report/local/supabase/.branches/_current_branch @@ -0,0 +1 @@ +main \ No newline at end of file diff --git a/evals/build-dataapi-002-restock-alert-report/local/supabase/.temp/cli-latest b/evals/build-dataapi-002-restock-alert-report/local/supabase/.temp/cli-latest new file mode 100644 index 00000000..e9acfb34 --- /dev/null +++ b/evals/build-dataapi-002-restock-alert-report/local/supabase/.temp/cli-latest @@ -0,0 +1 @@ +v2.110.0 \ No newline at end of file diff --git a/evals/build-dataapi-002-restock-alert-report/local/supabase/config.toml b/evals/build-dataapi-002-restock-alert-report/local/supabase/config.toml new file mode 100644 index 00000000..06330f25 --- /dev/null +++ b/evals/build-dataapi-002-restock-alert-report/local/supabase/config.toml @@ -0,0 +1,165 @@ +project_id = "sandbox-restock-alert" + +[api] +enabled = true +port = 54321 +schemas = ["public", "graphql_public"] +extra_search_path = ["public", "extensions"] +max_rows = 1000 + +[api.tls] +enabled = false + +[db] +port = 54322 +shadow_port = 54320 +major_version = 17 + +[db.pooler] +enabled = false +port = 54329 +pool_mode = "transaction" +default_pool_size = 20 +max_client_conn = 100 + +[db.migrations] +enabled = true +schema_paths = [] + +[db.seed] +enabled = false + +[realtime] +enabled = true + +[studio] +enabled = true +port = 54323 +api_url = "http://127.0.0.1" +openai_api_key = "env(OPENAI_API_KEY)" + +[inbucket] +enabled = true +port = 54324 + +[storage] +enabled = true +file_size_limit = "50MiB" + +[storage.s3_protocol] +enabled = true + +[storage.analytics] +enabled = false +max_namespaces = 5 +max_tables = 10 +max_catalogs = 2 + +[storage.vector] +enabled = false +max_buckets = 10 +max_indexes = 5 + +[auth] +enabled = true +site_url = "http://127.0.0.1:3000" +additional_redirect_urls = ["https://127.0.0.1:3000"] +jwt_expiry = 3600 +enable_refresh_token_rotation = true +refresh_token_reuse_interval = 10 +enable_signup = true +enable_anonymous_sign_ins = false +enable_manual_linking = false +minimum_password_length = 6 +password_requirements = "" + +[auth.rate_limit] +email_sent = 2 +sms_sent = 30 +anonymous_users = 30 +token_refresh = 150 +sign_in_sign_ups = 30 +token_verifications = 30 +web3 = 30 + +[auth.email] +enable_signup = true +double_confirm_changes = true +enable_confirmations = false +secure_password_change = false +max_frequency = "1s" +otp_length = 6 +otp_expiry = 3600 + +[auth.sms] +enable_signup = false +enable_confirmations = false +template = "Your code is {{ .Code }}" +max_frequency = "5s" + +[auth.sms.twilio] +enabled = false +account_sid = "" +message_service_sid = "" +auth_token = "env(SUPABASE_AUTH_SMS_TWILIO_AUTH_TOKEN)" + +[auth.mfa] +max_enrolled_factors = 10 + +[auth.mfa.totp] +enroll_enabled = false +verify_enabled = false + +[auth.mfa.phone] +enroll_enabled = false +verify_enabled = false +otp_length = 6 +template = "Your code is {{ .Code }}" +max_frequency = "5s" + +[auth.external.apple] +enabled = false +client_id = "" +secret = "env(SUPABASE_AUTH_EXTERNAL_APPLE_SECRET)" +redirect_uri = "" +url = "" +skip_nonce_check = false +email_optional = false + +[auth.web3.solana] +enabled = false + +[auth.third_party.firebase] +enabled = false + +[auth.third_party.auth0] +enabled = false + +[auth.third_party.aws_cognito] +enabled = false + +[auth.third_party.clerk] +enabled = false + +[auth.oauth_server] +enabled = false +authorization_url_path = "/oauth/consent" +allow_dynamic_registration = false + +[edge_runtime] +enabled = true +policy = "per_worker" +inspector_port = 8083 +deno_version = 2 + +[analytics] +enabled = true +port = 54327 +backend = "postgres" + +[experimental] +orioledb_version = "" +s3_host = "env(S3_HOST)" +s3_region = "env(S3_REGION)" +s3_access_key = "env(S3_ACCESS_KEY)" +s3_secret_key = "env(S3_SECRET_KEY)" diff --git a/evals/build-dataapi-002-restock-alert-report/local/supabase/migrations/0000_inventory_schema.sql b/evals/build-dataapi-002-restock-alert-report/local/supabase/migrations/0000_inventory_schema.sql new file mode 100644 index 00000000..0dfdff73 --- /dev/null +++ b/evals/build-dataapi-002-restock-alert-report/local/supabase/migrations/0000_inventory_schema.sql @@ -0,0 +1,68 @@ +-- Inventory for the restock-alert worker. These tables are backend-only: +-- RLS is enabled with no policies, so client-side (publishable) keys read +-- nothing. Trusted backend code authenticates with the secret key, which +-- bypasses RLS. +create table public.warehouses ( + id bigint generated always as identity primary key, + name text not null +); + +create table public.suppliers ( + id bigint generated always as identity primary key, + name text not null, + email text not null unique +); + +create table public.products ( + id bigint generated always as identity primary key, + name text not null, + supplier_id bigint not null references public.suppliers (id), + reorder_threshold int not null +); + +create table public.inventory ( + id bigint generated always as identity primary key, + warehouse_id bigint not null references public.warehouses (id), + product_id bigint not null references public.products (id), + quantity int not null check (quantity >= 0) +); + +alter table public.warehouses enable row level security; +alter table public.suppliers enable row level security; +alter table public.products enable row level security; +alter table public.inventory enable row level security; + +-- Newer Supabase CLIs no longer auto-grant privileges on migration-created +-- tables. The trusted backend (secret key → service_role) needs the SELECT +-- privilege; client-side roles get nothing, so these tables stay backend-only. +grant select on public.warehouses, public.suppliers, public.products, + public.inventory to service_role; + +-- Seed data. +insert into public.warehouses (name) values + ('North DC'), + ('South DC'), + ('West DC'); + +insert into public.suppliers (name, email) values + ('Acme Supplies', 'acme@example.com'), + ('Global Parts', 'parts@example.com'); + +insert into public.products (name, supplier_id, reorder_threshold) values + ('Widget', 1, 20), + ('Gadget', 2, 15), + ('Gizmo', 1, 10); + +-- warehouse_id, product_id, quantity. Below-threshold rows are the alerts; +-- West DC's Widget sits exactly at threshold and must NOT alert (strict +-- less-than), and West DC has no Gizmo row at all (untracked combos are +-- simply absent, not a zero-quantity alert). +insert into public.inventory (warehouse_id, product_id, quantity) values + (1, 1, 5), -- North DC / Widget: 5 < 20 -> alert + (1, 2, 30), -- North DC / Gadget: 30 >= 15 -> ok + (1, 3, 3), -- North DC / Gizmo: 3 < 10 -> alert + (2, 1, 25), -- South DC / Widget: 25 >= 20 -> ok + (2, 2, 2), -- South DC / Gadget: 2 < 15 -> alert + (2, 3, 12), -- South DC / Gizmo: 12 >= 10 -> ok + (3, 1, 20), -- West DC / Widget: 20 >= 20 -> ok (boundary, not strictly below) + (3, 2, 0); -- West DC / Gadget: 0 < 15 -> alert diff --git a/evals/resolve-sdk-001-legacy-key-migration/EVAL.ts b/evals/resolve-sdk-001-legacy-key-migration/EVAL.ts new file mode 100644 index 00000000..9c212f57 --- /dev/null +++ b/evals/resolve-sdk-001-legacy-key-migration/EVAL.ts @@ -0,0 +1,230 @@ +import { + type CheckResult, + type LocalStackEvalContext, + type LocalStackScorer, +} from '@supabase-evals/core'; + +// Legacy → new API key migration (regression): the seeded app authenticates +// with the local stack's legacy demo JWTs (anon + service_role) and works +// out of the box; the task is to move it to the new sb_publishable_… / +// sb_secret_… keys without breaking it. Scored on behavior (both scripts +// still print the right data — the stats script's RLS-bypassing count only +// works with a genuine secret key), on the legacy JWTs being gone, and on +// the key boundary staying intact (the public script must not end up holding +// the secret key). + +const APP_DIR = 'app'; + +// Header+payload prefix shared by both legacy local demo JWTs +// ({"iss":"supabase-demo",…} signed with the default local JWT secret) — +// matching on it catches either key regardless of the signature bytes. +const LEGACY_JWT_MARKER = 'eyJpc3MiOiJzdXBhYmFzZS1kZW1vIi'; + +const scorer: LocalStackScorer = async (ctx) => { + const checks: CheckResult[] = []; + try { + const status = await readStatus(ctx); + const publishableKey = str(status.PUBLISHABLE_KEY); + const secretKey = str(status.SECRET_KEY); + if (!publishableKey || !secretKey) { + return fail( + 'read stack config from `supabase status`', + `missing PUBLISHABLE_KEY/SECRET_KEY — is the stack running on a new-enough CLI? got keys: ${Object.keys(status).join(', ')}` + ); + } + + // Be generous about a missing install step; the eval is about the keys, + // not npm. A no-op when the agent already installed dependencies. + await ctx.exec( + `cd ${APP_DIR} && [ -d node_modules ] || npm install --no-audit --no-fund --silent || true`, + { timeoutMs: 180_000 } + ); + + // Ground truth from the seeded database. + const { rows: publishedRows } = await ctx.query( + `select title from public.posts where published order by title asc` + ); + const expectedTitles = publishedRows.map((row) => row.title); + const { rows: draftRows } = await ctx.query( + `select count(*)::int as n from public.posts where not published` + ); + const expectedDrafts = Number(draftRows[0]?.n ?? -1); + + // 1. The public script still lists published posts (client-key path). + const posts = await ctx.exec(`cd ${APP_DIR} && npm run -s posts`, { + timeoutMs: 60_000, + }); + const postTitles = parseJson(posts.stdout, '[', ']'); + checks.push({ + name: 'posts script still lists published posts', + passed: + JSON.stringify(postTitles ?? null) === JSON.stringify(expectedTitles), + notes: postTitles + ? `got ${JSON.stringify(postTitles)}` + : `no JSON output — ${preview(posts.stderr || posts.stdout)}`, + }); + + // 2. The internal script still counts drafts. Drafts are invisible to the + // publishable key (RLS), so a correct count proves a working secret key. + const stats = await ctx.exec(`cd ${APP_DIR} && npm run -s stats`, { + timeoutMs: 60_000, + }); + const statsOut = parseJson(stats.stdout, '{', '}') as + | { drafts?: unknown } + | undefined; + checks.push({ + name: 'stats script still counts drafts (secret key bypasses RLS)', + passed: statsOut?.drafts === expectedDrafts, + notes: statsOut + ? `got ${JSON.stringify(statsOut)}, expected ${expectedDrafts} drafts` + : `no JSON output — ${preview(stats.stderr || stats.stdout)}`, + }); + + // 3. The legacy JWTs are gone from the app (env files included). + const legacyScan = await ctx.exec( + `grep -rl --exclude-dir=node_modules '${LEGACY_JWT_MARKER}' ${APP_DIR} || true` + ); + checks.push({ + name: 'legacy anon/service_role JWTs removed from the app', + passed: legacyScan.stdout.trim() === '', + notes: + legacyScan.stdout.trim().replace(/\s+/g, ', ') || + 'no legacy JWTs found', + }); + + // 4. Key boundary intact: the public script must not hold the secret key, + // neither as a literal nor via an env var that resolves to it. + checks.push(await publicScriptKeyCheck(ctx)); + + return { passed: checks.every((c) => c.passed), checks }; + } catch (error) { + const msg = error instanceof Error ? error.message : String(error); + checks.push({ + name: 'scorer completed without errors', + passed: false, + notes: msg, + }); + return { passed: false, checks }; + } +}; + +export default scorer; + +/** + * The public (posts) script must not use the secret key: no sb_secret_ + * literal in its source, and no reference to an env var whose value in .env + * is a secret key. + */ +async function publicScriptKeyCheck( + ctx: LocalStackEvalContext +): Promise { + const NAME = 'public script does not hold the secret key'; + const entry = await resolveScriptEntry(ctx, 'posts'); + const source = await ctx + .readFile(`${APP_DIR}/${entry}`) + .catch(() => undefined); + if (source === undefined) { + return { + name: NAME, + passed: false, + notes: `could not read ${APP_DIR}/${entry} to inspect`, + }; + } + if (source.includes('sb_secret_')) { + return { + name: NAME, + passed: false, + notes: `${entry} contains an sb_secret_ literal`, + }; + } + const env = await ctx.readFile(`${APP_DIR}/.env`).catch(() => ''); + const secretVars = parseEnv(env) + .filter(([, value]) => value.startsWith('sb_secret_')) + .map(([key]) => key); + const leaked = secretVars.filter((name) => source.includes(name)); + return { + name: NAME, + passed: leaked.length === 0, + notes: leaked.length + ? `${entry} references secret-key env var(s): ${leaked.join(', ')}` + : `${entry} holds no secret-key reference`, + }; +} + +/** File the given npm script runs, e.g. `posts` → `posts.mjs`. */ +async function resolveScriptEntry( + ctx: LocalStackEvalContext, + script: string +): Promise { + const fallback = `${script}.mjs`; + try { + const pkg = JSON.parse(await ctx.readFile(`${APP_DIR}/package.json`)) as { + scripts?: Record; + }; + const command = pkg.scripts?.[script] ?? ''; + return command.match(/[\w./-]+\.(?:mjs|cjs|js|ts)/)?.[0] ?? fallback; + } catch { + return fallback; + } +} + +function parseEnv(content: string): Array<[string, string]> { + return content + .split('\n') + .map((line) => line.trim()) + .filter((line) => line && !line.startsWith('#')) + .flatMap((line) => { + const eq = line.indexOf('='); + if (eq === -1) return []; + const value = line + .slice(eq + 1) + .trim() + .replace(/^['"]|['"]$/g, ''); + return [[line.slice(0, eq).trim(), value] as [string, string]]; + }); +} + +function parseJson( + stdout: string, + open: string, + close: string +): unknown | undefined { + const start = stdout.indexOf(open); + const end = stdout.lastIndexOf(close); + if (start === -1 || end <= start) return undefined; + try { + return JSON.parse(stdout.slice(start, end + 1)); + } catch { + return undefined; + } +} + +function str(value: unknown): string | undefined { + return typeof value === 'string' && value.length > 0 ? value : undefined; +} + +function preview(body: string): string { + return body.replace(/\s+/g, ' ').slice(0, 160); +} + +function fail( + name: string, + notes: string +): { passed: false; checks: CheckResult[] } { + return { passed: false, checks: [{ name, passed: false, notes }] }; +} + +/** Parse `supabase status -o json` for the stack's URL and keys. */ +async function readStatus( + ctx: LocalStackEvalContext +): Promise> { + const res = await ctx.exec('supabase status -o json'); + const start = res.stdout.indexOf('{'); + const end = res.stdout.lastIndexOf('}'); + if (start === -1 || end <= start) { + throw new Error( + `could not read \`supabase status\`: ${res.stderr || res.stdout}` + ); + } + return JSON.parse(res.stdout.slice(start, end + 1)); +} diff --git a/evals/resolve-sdk-001-legacy-key-migration/PROMPT.md b/evals/resolve-sdk-001-legacy-key-migration/PROMPT.md new file mode 100644 index 00000000..7c59bf8c --- /dev/null +++ b/evals/resolve-sdk-001-legacy-key-migration/PROMPT.md @@ -0,0 +1,27 @@ +--- +stage: resolve +suite: regression +interface: cli +cliVersion: 2.109.1 +product: + - data-api + - auth +topic: + - sdk + - security +services: + - gotrue + - kong + - postgrest +projectRunning: true +motivation: https://github.com/orgs/supabase/discussions/29260 +--- + +Heads-up from the platform team: the legacy JWT-based API keys (`anon` / +`service_role`) are going away for our projects soon, in favor of the new +publishable/secret keys. The little blog tooling app in `app/` still uses the +legacy keys. + +Migrate it over. Both scripts need to keep working — `npm run posts` and +`npm run stats` (run them from `app/`). The local Supabase project in +`supabase/` is already running. diff --git a/evals/resolve-sdk-001-legacy-key-migration/README.md b/evals/resolve-sdk-001-legacy-key-migration/README.md new file mode 100644 index 00000000..c5bc32d5 --- /dev/null +++ b/evals/resolve-sdk-001-legacy-key-migration/README.md @@ -0,0 +1,24 @@ +# resolve-sdk-001-legacy-key-migration + +A working blog-tooling app (`local/app/`) authenticates with the **legacy +local demo JWTs** — the deterministic `anon` / `service_role` keys every +local stack issues when `supabase/config.toml` doesn't override the JWT +secret (payload `{"iss":"supabase-demo",…}` signed with the default +`super-secret-jwt-token-with-at-least-32-characters-long`). That's what lets +the seed hardcode valid keys in `.env` before the stack exists. + +The task is to migrate the app to the new `sb_publishable_…` / `sb_secret_…` +keys (see the motivation link). The scorer checks behavior, not process: + +1. `npm run posts` still prints the published titles (client-key path). +2. `npm run stats` still prints the draft count — drafts are hidden from the + publishable key by RLS, so a correct count proves a real secret key. +3. No legacy demo JWT remains anywhere in `app/` (matched on the shared + header+payload prefix, so it catches both keys). +4. The public script doesn't end up holding the secret key, directly or via + an env var that resolves to one. + +Assumptions to keep in mind: the pinned `cliVersion` must expose +`PUBLISHABLE_KEY` / `SECRET_KEY` in `supabase status -o json`, and the stack +must still accept legacy JWTs by default (true as of 2.109.1) or the seeded +app would be broken before the agent starts. diff --git a/evals/resolve-sdk-001-legacy-key-migration/local/app/.env b/evals/resolve-sdk-001-legacy-key-migration/local/app/.env new file mode 100644 index 00000000..e9c51c59 --- /dev/null +++ b/evals/resolve-sdk-001-legacy-key-migration/local/app/.env @@ -0,0 +1,4 @@ +# Supabase keys for local dev (from `supabase status`) +SUPABASE_URL=http://127.0.0.1:54321 +SUPABASE_ANON_KEY=eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJzdXBhYmFzZS1kZW1vIiwicm9sZSI6ImFub24iLCJleHAiOjE5ODM4MTI5OTZ9.CRXP1A7WOeoJeXxjNni43kdQwgnWNReilDMblYTn_I0 +SUPABASE_SERVICE_ROLE_KEY=eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJzdXBhYmFzZS1kZW1vIiwicm9sZSI6InNlcnZpY2Vfcm9sZSIsImV4cCI6MTk4MzgxMjk5Nn0.EGIM96RAZx35lJzdJsyH-qQwv8Hdp7fsn3W0YpN81IU diff --git a/evals/resolve-sdk-001-legacy-key-migration/local/app/package.json b/evals/resolve-sdk-001-legacy-key-migration/local/app/package.json new file mode 100644 index 00000000..5cab9c72 --- /dev/null +++ b/evals/resolve-sdk-001-legacy-key-migration/local/app/package.json @@ -0,0 +1,12 @@ +{ + "name": "blog-tools", + "private": true, + "type": "module", + "scripts": { + "posts": "node --env-file=.env posts.mjs", + "stats": "node --env-file=.env stats.mjs" + }, + "dependencies": { + "@supabase/supabase-js": "^2.58.0" + } +} diff --git a/evals/resolve-sdk-001-legacy-key-migration/local/app/posts.mjs b/evals/resolve-sdk-001-legacy-key-migration/local/app/posts.mjs new file mode 100644 index 00000000..749820f0 --- /dev/null +++ b/evals/resolve-sdk-001-legacy-key-migration/local/app/posts.mjs @@ -0,0 +1,19 @@ +import { createClient } from '@supabase/supabase-js'; + +// Public site: lists published post titles. Runs with the project's public +// (client-side) API key, so RLS applies. +const supabase = createClient( + process.env.SUPABASE_URL, + process.env.SUPABASE_ANON_KEY +); + +const { data, error } = await supabase + .from('posts') + .select('title') + .order('title'); + +if (error) { + console.error(error.message); + process.exit(1); +} +console.log(JSON.stringify(data.map((post) => post.title))); diff --git a/evals/resolve-sdk-001-legacy-key-migration/local/app/stats.mjs b/evals/resolve-sdk-001-legacy-key-migration/local/app/stats.mjs new file mode 100644 index 00000000..a7155ccd --- /dev/null +++ b/evals/resolve-sdk-001-legacy-key-migration/local/app/stats.mjs @@ -0,0 +1,19 @@ +import { createClient } from '@supabase/supabase-js'; + +// Internal tooling: counts unpublished drafts across all posts. Trusted +// backend only — runs with the project's server-side key, which bypasses RLS. +const supabase = createClient( + process.env.SUPABASE_URL, + process.env.SUPABASE_SERVICE_ROLE_KEY +); + +const { count, error } = await supabase + .from('posts') + .select('*', { count: 'exact', head: true }) + .eq('published', false); + +if (error) { + console.error(error.message); + process.exit(1); +} +console.log(JSON.stringify({ drafts: count })); diff --git a/evals/resolve-sdk-001-legacy-key-migration/local/supabase/config.toml b/evals/resolve-sdk-001-legacy-key-migration/local/supabase/config.toml new file mode 100644 index 00000000..d9722d7e --- /dev/null +++ b/evals/resolve-sdk-001-legacy-key-migration/local/supabase/config.toml @@ -0,0 +1,165 @@ +project_id = "sandbox-key-migration" + +[api] +enabled = true +port = 54321 +schemas = ["public", "graphql_public"] +extra_search_path = ["public", "extensions"] +max_rows = 1000 + +[api.tls] +enabled = false + +[db] +port = 54322 +shadow_port = 54320 +major_version = 17 + +[db.pooler] +enabled = false +port = 54329 +pool_mode = "transaction" +default_pool_size = 20 +max_client_conn = 100 + +[db.migrations] +enabled = true +schema_paths = [] + +[db.seed] +enabled = false + +[realtime] +enabled = true + +[studio] +enabled = true +port = 54323 +api_url = "http://127.0.0.1" +openai_api_key = "env(OPENAI_API_KEY)" + +[inbucket] +enabled = true +port = 54324 + +[storage] +enabled = true +file_size_limit = "50MiB" + +[storage.s3_protocol] +enabled = true + +[storage.analytics] +enabled = false +max_namespaces = 5 +max_tables = 10 +max_catalogs = 2 + +[storage.vector] +enabled = false +max_buckets = 10 +max_indexes = 5 + +[auth] +enabled = true +site_url = "http://127.0.0.1:3000" +additional_redirect_urls = ["https://127.0.0.1:3000"] +jwt_expiry = 3600 +enable_refresh_token_rotation = true +refresh_token_reuse_interval = 10 +enable_signup = true +enable_anonymous_sign_ins = false +enable_manual_linking = false +minimum_password_length = 6 +password_requirements = "" + +[auth.rate_limit] +email_sent = 2 +sms_sent = 30 +anonymous_users = 30 +token_refresh = 150 +sign_in_sign_ups = 30 +token_verifications = 30 +web3 = 30 + +[auth.email] +enable_signup = true +double_confirm_changes = true +enable_confirmations = false +secure_password_change = false +max_frequency = "1s" +otp_length = 6 +otp_expiry = 3600 + +[auth.sms] +enable_signup = false +enable_confirmations = false +template = "Your code is {{ .Code }}" +max_frequency = "5s" + +[auth.sms.twilio] +enabled = false +account_sid = "" +message_service_sid = "" +auth_token = "env(SUPABASE_AUTH_SMS_TWILIO_AUTH_TOKEN)" + +[auth.mfa] +max_enrolled_factors = 10 + +[auth.mfa.totp] +enroll_enabled = false +verify_enabled = false + +[auth.mfa.phone] +enroll_enabled = false +verify_enabled = false +otp_length = 6 +template = "Your code is {{ .Code }}" +max_frequency = "5s" + +[auth.external.apple] +enabled = false +client_id = "" +secret = "env(SUPABASE_AUTH_EXTERNAL_APPLE_SECRET)" +redirect_uri = "" +url = "" +skip_nonce_check = false +email_optional = false + +[auth.web3.solana] +enabled = false + +[auth.third_party.firebase] +enabled = false + +[auth.third_party.auth0] +enabled = false + +[auth.third_party.aws_cognito] +enabled = false + +[auth.third_party.clerk] +enabled = false + +[auth.oauth_server] +enabled = false +authorization_url_path = "/oauth/consent" +allow_dynamic_registration = false + +[edge_runtime] +enabled = true +policy = "per_worker" +inspector_port = 8083 +deno_version = 2 + +[analytics] +enabled = true +port = 54327 +backend = "postgres" + +[experimental] +orioledb_version = "" +s3_host = "env(S3_HOST)" +s3_region = "env(S3_REGION)" +s3_access_key = "env(S3_ACCESS_KEY)" +s3_secret_key = "env(S3_SECRET_KEY)" diff --git a/evals/resolve-sdk-001-legacy-key-migration/local/supabase/migrations/0000_posts_schema.sql b/evals/resolve-sdk-001-legacy-key-migration/local/supabase/migrations/0000_posts_schema.sql new file mode 100644 index 00000000..28e2c5b8 --- /dev/null +++ b/evals/resolve-sdk-001-legacy-key-migration/local/supabase/migrations/0000_posts_schema.sql @@ -0,0 +1,27 @@ +-- Blog posts. Published posts are public (readable with the client key); +-- drafts are only reachable by trusted backend code that bypasses RLS. +create table public.posts ( + id bigint generated always as identity primary key, + title text not null, + published boolean not null default false +); + +alter table public.posts enable row level security; + +create policy "anyone can read published posts" + on public.posts + for select + to anon, authenticated + using (published); + +-- Newer Supabase CLIs no longer auto-grant privileges on migration-created +-- tables, so grant them explicitly. RLS scopes the rows for client roles; +-- service_role (trusted backend) bypasses RLS but still needs the privilege. +grant select on public.posts to anon, authenticated, service_role; + +insert into public.posts (title, published) values + ('Announcing vector buckets', true), + ('Realtime broadcast tips', true), + ('Row level security explained', true), + ('DRAFT: pricing update', false), + ('DRAFT: roadmap 2027', false);