refactor(evals): split deleted-user-access into two scenarios

apps/web/src/data/regression-eval-results.json

@@ -269,6 +269,72 @@
     "attempts": 1,
     "sourcePath": "claude-code-sonnet-5/build-realtime-001-live-chat-updates.json"
   },
+  {
+    "experiment": "claude-code-sonnet-5",
+    "experimentSuite": "regression",
+    "experimentDisplay": {
+      "agent": "claude-code",
+      "modelProvider": "anthropic",
+      "modelId": "claude-sonnet-5",
+      "reasoningEffort": "high"
+    },
+    "eval": "investigate-auth-002-api-key-migration",
+    "stage": "investigate",
+    "product": [
+      "auth"
+    ],
+    "topic": [
+      "security",
+      "sdk"
+    ],
+    "suite": "regression",
+    "passed": false,
+    "checks": [
+      {
+        "name": "mapped legacy anon/service_role onto publishable/secret",
+        "passed": false,
+        "judgeNotes": "It maps anon→publishable and service_role→secret correctly, but it also presents the migration as a mechanical swap/rename-reissue, implying drop-in replacement despite the new keys not being interchangeable JWTs."
+      },
+      {
+        "name": "placed the publishable key in the frontend with RLS still applying",
+        "passed": true,
+        "judgeNotes": "The assistant clearly states the publishable key belongs in frontend code and that requests made with it have RLS fully enforced. It does not claim the publishable key bypasses RLS or authenticates a user, and it keeps the secret key backend-only."
+      },
+      {
+        "name": "kept the secret key server-only and RLS-bypassing",
+        "passed": true,
+        "judgeNotes": "The assistant maps sb_secret to legacy service_role, states it is backend/server-only, says it bypasses RLS, and warns never to expose it in public clients."
+      }
+    ],
+    "skills": {
+      "available": [
+        "supabase",
+        "supabase-postgres-best-practices"
+      ],
+      "loaded": [
+        "supabase"
+      ]
+    },
+    "docs": {
+      "calls": [
+        {
+          "source": "web_fetch",
+          "query": "Summarize the difference between publishable and secret API keys vs legacy anon/service_role keys, which is meant for frontend/browser use, which is meant for backend/server use, and how they interact with Row Level Security (RLS).",
+          "hasContent": true,
+          "pages": [
+            {
+              "url": "https://supabase.com/docs/guides/api/api-keys.md"
+            }
+          ],
+          "resultChars": 1286
+        }
+      ]
+    },
+    "prompt": "We're migrating off the legacy anon/service_role API keys to the new publishable\nand secret keys. The team isn't sure which one belongs in the frontend and what\neach means for RLS. Give us a quick rundown.",
+    "promptSourcePath": "evals/investigate-auth-002-api-key-migration/PROMPT.md",
+    "attempts": 2,
+    "sourcePath": "claude-code-sonnet-5/investigate-auth-002-api-key-migration.json"
+  },
   {
     "experiment": "claude-code-sonnet-5",
     "experimentSuite": "regression",
@@ -736,6 +802,55 @@
     "attempts": 1,
     "sourcePath": "claude-code-sonnet-5-no-skills/build-realtime-001-live-chat-updates.json"
   },
+  {
+    "experiment": "claude-code-sonnet-5-no-skills",
+    "experimentSuite": "regression",
+    "experimentDisplay": {
+      "agent": "claude-code",
+      "modelProvider": "anthropic",
+      "modelId": "claude-sonnet-5",
+      "reasoningEffort": "high"
+    },
+    "eval": "investigate-auth-002-api-key-migration",
+    "stage": "investigate",
+    "product": [
+      "auth"
+    ],
+    "topic": [
+      "security",
+      "sdk"
+    ],
+    "suite": "regression",
+    "passed": false,
+    "checks": [
+      {
+        "name": "mapped legacy anon/service_role onto publishable/secret",
+        "passed": false,
+        "judgeNotes": "The mapping is correct (anon → publishable, service_role → secret), but the answer repeatedly frames them as direct replacements / just swapping keys, which the rubric says should fail because the new keys are not exact drop-in interchangeable replacements."
+      },
+      {
+        "name": "placed the publishable key in the frontend with RLS still applying",
+        "passed": true,
+        "judgeNotes": "The answer correctly says the publishable key belongs in frontend/client code and that RLS is fully enforced for requests made with it. It also correctly keeps the secret key/server role backend-only."
+      },
+      {
+        "name": "kept the secret key server-only and RLS-bypassing",
+        "passed": true,
+        "judgeNotes": "The assistant states the secret key replaces/maps to service_role, must be backend/server-only and never shipped to clients, and bypasses RLS."
+      }
+    ],
+    "skills": {
+      "available": [],
+      "loaded": []
+    },
+    "docs": {
+      "calls": []
+    },
+    "prompt": "We're migrating off the legacy anon/service_role API keys to the new publishable\nand secret keys. The team isn't sure which one belongs in the frontend and what\neach means for RLS. Give us a quick rundown.",
+    "promptSourcePath": "evals/investigate-auth-002-api-key-migration/PROMPT.md",
+    "attempts": 2,
+    "sourcePath": "claude-code-sonnet-5-no-skills/investigate-auth-002-api-key-migration.json"
+  },
   {
     "experiment": "claude-code-sonnet-5-no-skills",
     "experimentSuite": "regression",

evals/investigate-auth-001-deleted-user-access/EVAL.ts

@@ -28,7 +28,7 @@ const scorer: ToolScorer = async (ctx) => {
       await checkRefreshTokenRejected(ctx, users),
       await checkCannotSignBackIn(ctx, users),
       await checkBystanderUnaffected(users),
-      await checkRevocationDiagnosis(ctx),
+      ...(await checkRevocationAnswer(ctx)),
     ];
 
     return {
@@ -199,29 +199,55 @@ async function checkBystanderUnaffected(
   };
 }
 
-async function checkRevocationDiagnosis(
+async function checkRevocationAnswer(
   ctx: ToolEvalContext
-): Promise<CheckResult> {
-  const verdict = await judge({
-    input: serializeTranscript(ctx.transcript, {
-      includeToolCallInputs: true,
+): Promise<CheckResult[]> {
+  const input = serializeTranscript(ctx.transcript, {
+    includeToolCallInputs: true,
+  });
+
+  const [causeFound, flowFixed, windowExplained] = await Promise.all([
+    judge({
+      input,
+      rubric: stripIndent`
+        Pass if the assistant identifies that delete_account only soft-deletes the profile row and never removes the auth user or revokes their sessions, so the account keeps signing in.
+
+        Fail if the assistant blames the symptom on caching or client bugs instead of the flow, or never explains why the deleted account still works.
+      `,
     }),
-    rubric: stripIndent`
-      Pass if the assistant diagnoses why the deleted account still works, fixes the delete-account flow with real revocation, explains any remaining access-token window consistently with the fix it implemented, and correctly clarifies publishable vs secret API keys.
+    judge({
+      input,
+      rubric: stripIndent`
+        Pass if the assistant fixes the flow so the auth user is deleted (or equivalently their identity and sessions are removed), revoking their sessions and refresh tokens.
 
-      A passing answer should:
-      - Identify that delete_account only soft-deletes the profile row and never removes the auth user or revokes their sessions, so the account keeps signing in.
-      - Fix the flow so the auth user is deleted (or equivalently their identity and sessions are removed), revoking their sessions and refresh tokens.
-      - Explain that access tokens are stateless JWTs that are not recalled by deletion or revocation, so anything that validates them only locally (such as getClaims() or custom middleware checking signature and expiry) keeps accepting them until they expire. Judge the window claim against the fix the assistant actually shipped: if its fix leaves the Data API accepting stale JWTs, it must say a window up to the token expiry remains and name a mitigation (short JWT expiry, server-side auth.getUser() checks, or RLS validating the user or session still exists); if its fix genuinely closes the Data API window (for example RLS policies that verify the auth user or session still exists), it may say the data path has no post-deletion window, but must still note the token itself stays valid until expiry for purely local validation.
-      - Explain that the publishable key belongs in frontend code where requests run under the signed-in user's JWT and RLS applies, while the secret key is server-only, bypasses RLS, and must never ship to the client.
+        Fail if the assistant leaves the flow as a soft delete without revoking auth access.
+      `,
+    }),
+    judge({
+      input,
+      rubric: stripIndent`
+        Pass if the assistant explains that access tokens are stateless JWTs that are not recalled by deletion or revocation, so anything that validates them only locally (such as getClaims() or custom middleware checking signature and expiry) keeps accepting them until they expire. Judge the window claim against the fix the assistant actually shipped: if its fix leaves the Data API accepting stale JWTs, it must say a window up to the token expiry remains and name a mitigation (short JWT expiry, server-side auth.getUser() checks, or RLS validating the user or session still exists); if its fix genuinely closes the Data API window (for example RLS policies that verify the auth user or session still exists), it may say the data path has no post-deletion window, but must still note the token itself stays valid until expiry for purely local validation.
 
-      Fail if the assistant leaves the flow as a soft delete without revoking auth access, claims existing access tokens are instantly invalidated everywhere the moment the user is deleted without any caveat, makes a window claim that contradicts its own implemented fix in either direction, recommends putting the secret key or legacy service_role key in frontend code, says RLS applies to the secret key, or blames the symptom on caching or client bugs instead of the flow.
-    `,
-  });
+        Fail if the assistant claims existing access tokens are instantly invalidated everywhere the moment the user is deleted without any caveat, or makes a window claim that contradicts its own implemented fix in either direction.
+      `,
+    }),
+  ]);
 
-  return {
-    name: 'diagnosed and explained session revocation',
-    passed: verdict.passed,
-    judgeNotes: verdict.notes,
-  };
+  return [
+    {
+      name: 'diagnosed the soft-delete as the cause',
+      passed: causeFound.passed,
+      judgeNotes: causeFound.notes,
+    },
+    {
+      name: 'fixed the flow to revoke auth access',
+      passed: flowFixed.passed,
+      judgeNotes: flowFixed.notes,
+    },
+    {
+      name: 'explained the access-token window consistently with its fix',
+      passed: windowExplained.passed,
+      judgeNotes: windowExplained.notes,
+    },
+  ];
 }

evals/investigate-auth-001-deleted-user-access/PROMPT.md

@@ -6,8 +6,7 @@ product:
   - auth
 topic:
   - security
-  - sdk
-motivation: AI-820, AI-422, https://supabase.com/docs/guides/troubleshooting/should-i-set-a-shorter-max-age-parameter-on-the-cookies-8sbF4V
+motivation: AI-820, https://supabase.com/docs/guides/troubleshooting/should-i-set-a-shorter-max-age-parameter-on-the-cookies-8sbF4V
 ---
 
 Last week support removed a user through our app's delete-account flow — the
@@ -18,8 +17,3 @@ data like nothing happened.
 Figure out why the account still works, fix the flow so a deleted account
 loses access, and tell me whether there is any window where they could still
 get in after the fix.
-
-One more thing while you're at it: we're migrating off the legacy
-anon/service_role API keys to the new publishable and secret keys, and the
-team isn't sure which one belongs in the frontend and what each means for
-RLS. Give us a quick rundown.

evals/investigate-auth-002-api-key-migration/EVAL.ts

@@ -0,0 +1,63 @@
+import {
+  judge,
+  serializeTranscript,
+  type CheckResult,
+  type ToolScorer,
+} from '@supabase-evals/core';
+import { stripIndent } from 'common-tags';
+
+const scorer: ToolScorer = async (ctx) => {
+  const input = serializeTranscript(ctx.transcript);
+
+  const [mapping, frontendKey, serverKey] = await Promise.all([
+    judge({
+      input,
+      rubric: stripIndent`
+        Pass if the assistant maps the legacy keys onto the new ones: the anon key corresponds to the publishable key, and the service_role key corresponds to a secret key.
+
+        Fail if the assistant inverts that mapping, never relates the legacy keys to the new ones, or presents the new keys as exact drop-in replacements. They are not interchangeable with the legacy ones: the new keys are not JWTs and behave differently in places such as Edge Functions, so "same role, new key" is right and "just swap the string" is not.
+      `,
+    }),
+    judge({
+      input,
+      rubric: stripIndent`
+        Pass if the assistant puts the publishable key in frontend code and says RLS still applies to requests made with it. Detail beyond that (naming the anon and authenticated roles, or that the key identifies the app rather than a user) is a bonus, not a requirement: the user asked which key belongs in the frontend and what it means for RLS, so answering that is enough.
+
+        Fail if the assistant says the publishable key bypasses RLS, says the key itself authenticates a user or carries their identity, or puts a secret key in the frontend instead.
+      `,
+    }),
+    judge({
+      input,
+      rubric: stripIndent`
+        Pass if the assistant keeps a secret key server-only, maps it to service_role, and says it bypasses RLS and must never ship to the client.
+
+        Fail if the assistant recommends putting a secret key or the legacy service_role key in frontend code, or says RLS applies to the secret key.
+      `,
+    }),
+  ]);
+
+  const checks: CheckResult[] = [
+    {
+      name: 'mapped legacy anon/service_role onto publishable/secret',
+      passed: mapping.passed,
+      judgeNotes: mapping.notes,
+    },
+    {
+      name: 'placed the publishable key in the frontend with RLS still applying',
+      passed: frontendKey.passed,
+      judgeNotes: frontendKey.notes,
+    },
+    {
+      name: 'kept the secret key server-only and RLS-bypassing',
+      passed: serverKey.passed,
+      judgeNotes: serverKey.notes,
+    },
+  ];
+
+  return {
+    passed: checks.every((check) => check.passed),
+    checks,
+  };
+};
+
+export default scorer;

evals/investigate-auth-002-api-key-migration/PROMPT.md

@@ -0,0 +1,14 @@
+---
+stage: investigate
+suite: regression
+product:
+  - auth
+topic:
+  - security
+  - sdk
+motivation: AI-422
+---
+
+We're migrating off the legacy anon/service_role API keys to the new publishable
+and secret keys. The team isn't sure which one belongs in the frontend and what
+each means for RLS. Give us a quick rundown.