From 417e1b67466cb3ba5bb592b46d9a6655750becbb Mon Sep 17 00:00:00 2001 From: mattrossman <22670878+mattrossman@users.noreply.github.com> Date: Sat, 1 Aug 2026 07:17:15 +0000 Subject: [PATCH] chore: refresh eval results --- .../web/src/data/regression-eval-results.json | 426 +++++++----------- 1 file changed, 163 insertions(+), 263 deletions(-) diff --git a/apps/web/src/data/regression-eval-results.json b/apps/web/src/data/regression-eval-results.json index 4f7e59cd..700e0538 100644 --- a/apps/web/src/data/regression-eval-results.json +++ b/apps/web/src/data/regression-eval-results.json @@ -38,17 +38,17 @@ { "name": "user with JWT reads only their own rows", "passed": true, - "notes": "status 200: {\"data\":[{\"user_id\":\"f214cdc1-b432-4379-94d2-f9ca3873ac65\",\"metric\":\"steps_a_ms76nxkn\",\"value\":111}]}" + "notes": "status 200: [{\"user_id\":\"5dfd7dee-c1de-4fed-8355-032c7b4616f9\",\"metric\":\"steps_a_msa1fnuc\",\"value\":111}]" }, { "name": "user cannot read another user's rows by passing user_id", "passed": true, - "notes": "status 200: {\"data\":[{\"user_id\":\"f214cdc1-b432-4379-94d2-f9ca3873ac65\",\"metric\":\"steps_a_ms76nxkn\",\"value\":111}]}" + "notes": "status 200: [{\"user_id\":\"5dfd7dee-c1de-4fed-8355-032c7b4616f9\",\"metric\":\"steps_a_msa1fnuc\",\"value\":111}]" }, { "name": "service key bypasses RLS to read the target user's rows", "passed": true, - "notes": "status 200: {\"data\":[{\"user_id\":\"5a03947a-138d-4984-adeb-096609e24f4c\",\"metric\":\"steps_b_ms76nxkn\",\"value\":222}]}" + "notes": "status 200: [{\"user_id\":\"b8e3ef73-ab44-4ccd-9400-b31dd84a88f2\",\"metric\":\"steps_b_msa1fnuc\",\"value\":222}]" }, { "name": "non-service key is not granted service access", @@ -84,173 +84,113 @@ "calls": [ { "source": "search_docs", - "query": "{ searchDocs(query: \"@supabase/server edge function withSupabase package\", limit: 8) { nodes { title href content } } }", + "query": "{ searchDocs(query: \"@supabase/server withSupabase edge function multi-auth\", limit: 10) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/auth/choosing-a-server-package", - "title": "Which package to use" - }, - { - "url": "https://supabase.com/docs/guides/functions/auth", - "title": "Securing Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/functions/connect-to-postgres", - "title": "Integrating with Supabase Database (Postgres)" - }, - { - "url": "https://supabase.com/docs/guides/database/connecting-to-postgres/serverless-drivers", - "title": "Serverless Drivers" - }, - { - "url": "https://supabase.com/docs/guides/functions", - "title": "Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/auth/server-side", - "title": "Server-Side Rendering" + "url": "https://supabase.com/docs/guides/functions/auth-legacy-jwt", + "title": "Integrating With Supabase Auth" }, { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-functions", - "title": "Self-Hosted Functions" + "url": "https://supabase.com/docs/guides/auth/choosing-a-server-package", + "title": "Which package to use" }, { "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", "title": "Migrating to publishable and secret API keys" - } - ], - "resultChars": 62021 - }, - { - "source": "web_search", - "query": "\"@supabase/server\" npm withSupabase edge function", - "hasContent": false, - "pages": [ - { - "url": "https://supabase.com/blog/introducing-supabase-server", - "title": "Introducing @supabase/server" }, { - "url": "https://supabase.com/docs/guides/functions/quickstart", - "title": "Getting Started with Edge Functions | Supabase Docs" + "url": "https://supabase.com/docs/guides/functions/auth", + "title": "Securing Edge Functions" }, { - "url": "https://supabase.com/blog/edge-functions-node-npm", - "title": "Edge Functions: Node and native npm compatibility" + "url": "https://supabase.com/docs/guides/auth/enterprise-sso", + "title": "Enterprise Single Sign-On" }, { - "url": "https://supabase.com/docs/guides/functions/routing", - "title": "Handling Routing in Functions | Supabase Docs" + "url": "https://supabase.com/docs/guides/functions/examples/resumable-websockets", + "title": "Resumable WebSockets with Edge Functions" }, { - "url": "https://supabase.com/docs/guides/functions/auth", - "title": "Securing Edge Functions | Supabase Docs" + "url": "https://supabase.com/docs/guides/functions", + "title": "Edge Functions" }, { - "url": "https://supabase.com/docs/guides/functions/connect-to-postgres", - "title": "Integrating with Supabase Database (Postgres) | Supabase Docs" + "url": "https://supabase.com/docs/guides/auth", + "title": "Auth" }, { - "url": "https://supabase.com/docs/guides/functions/examples/image-manipulation", - "title": "Image Manipulation | Supabase Docs" + "url": "https://supabase.com/docs/guides/ai-tools/byo-mcp", + "title": "Deploy MCP servers" }, { - "url": "https://supabase.com/docs/guides/ai-tools/ai-prompts/edge-functions", - "title": "AI Prompt: Writing Supabase Edge Functions | Supabase Docs" + "url": "https://supabase.com/docs/guides/database/extensions/pg_net", + "title": "pg_net: Async Networking" } ], - "resultChars": 3177 + "resultChars": 74670 }, { "source": "search_docs", - "query": "{ searchDocs(query: \"new API keys publishable secret key edge functions environment variables SUPABASE_SECRET_KEY\", limit: 5) { nodes { title href content } } }", + "query": "{ searchDocs(query: \"verify_jwt combining auth modes edge function config.toml\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" - }, - { - "url": "https://supabase.com/docs/guides/functions/auth", - "title": "Securing Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-auth-keys", - "title": "New API Keys and Asymmetric Authentication" + "url": "https://supabase.com/docs/guides/functions/function-configuration", + "title": "Function Configuration" }, { - "url": "https://supabase.com/docs/guides/functions/secrets", - "title": "Environment Variables" + "url": "https://supabase.com/docs/guides/functions/deploy", + "title": "Deploy to Production" }, - { - "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-envoy", - "title": "Envoy API Gateway" - } - ], - "resultChars": 65829 - }, - { - "source": "search_docs", - "query": "{ searchDocs(query: \"supabase config.toml edge functions import_map deno.json npm specifiers verify_jwt\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ { "url": "https://supabase.com/docs/guides/functions/development-tips", "title": "Development tips" }, { - "url": "https://supabase.com/docs/guides/functions/function-configuration", - "title": "Function Configuration" - }, - { - "url": "https://supabase.com/docs/guides/functions/dependencies", - "title": "Managing dependencies" + "url": "https://supabase.com/docs/guides/functions/auth-headers", + "title": "Authorization headers" }, { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-generate-speech-stream", - "title": "Streaming Speech with ElevenLabs" - }, - { - "url": "https://supabase.com/docs/guides/security/npm-security", - "title": "Securing npm installs" + "url": "https://supabase.com/docs/guides/functions/examples/mcp-server-mcp-lite", + "title": "Building an MCP Server with mcp-lite" } ], - "resultChars": 42096 + "resultChars": 27882 }, { "source": "search_docs", - "query": "{ searchDocs(query: \"verify_jwt edge functions new secret key apikey header not a JWT authorization header required\", limit: 5) { nodes { title href content } } }", + "query": "{ searchDocs(query: \"failed to determine entrypoint edge runtime serve error\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/getting-started/api-keys", - "title": "Understanding API keys" + "url": "https://supabase.com/docs/guides/functions/error-codes", + "title": "Error codes" }, { - "url": "https://supabase.com/docs/guides/auth/signing-keys", - "title": "JWT Signing Keys" + "url": "https://supabase.com/docs/guides/functions/status-codes", + "title": "Status codes" }, { - "url": "https://supabase.com/docs/guides/auth/auth-hooks/send-email-hook", - "title": "Send Email Hook" + "url": "https://supabase.com/docs/guides/functions/function-configuration", + "title": "Function Configuration" }, { - "url": "https://supabase.com/docs/guides/getting-started/migrating-to-new-api-keys", - "title": "Migrating to publishable and secret API keys" + "url": "https://supabase.com/docs/guides/troubleshooting/edge-function-fails-deploy", + "title": "Edge Function deploy failed due to an internal error" }, { - "url": "https://supabase.com/docs/guides/functions/error-codes", - "title": "Error codes" + "url": "https://supabase.com/docs/guides/troubleshooting/issues-serving-edge-functions-locally", + "title": "Issues serving Edge Functions locally" } ], - "resultChars": 110652 + "resultChars": 20326 }, { "source": "web_search", - "query": "\"failed to determine entrypoint\" supabase functions serve worker boot error", + "query": "edge-runtime \"failed to determine entrypoint\" supabase functions serve", "pages": [], - "resultChars": 2141 + "resultChars": 1938 } ] }, @@ -288,7 +228,7 @@ { "name": "did not recommend read replicas for Realtime", "passed": true, - "judgeNotes": "The answer correctly treats the request as a Supabase Realtime/Postgres Changes setup, adds the table to the supabase_realtime publication, verifies it, and provides a client subscription snippet. It does not recommend or imply read replicas are needed or useful for live updates." + "judgeNotes": "The answer treats the task as Supabase Realtime/Postgres Changes setup, adds the table to the supabase_realtime publication, provides client subscription code, and does not recommend or imply read replicas are needed or useful for live updates." } ], "skills": { @@ -304,62 +244,31 @@ "calls": [ { "source": "search_docs", - "query": "{ searchDocs(query: \"enable realtime postgres changes table subscribe\", limit: 5) { nodes { ... on Guide { title href content } } } }", + "query": "{ searchDocs(query: \"enable realtime postgres changes RLS replica identity\", limit: 5) { nodes { ... on Guide { title href content } } } }", "hasContent": true, "pages": [ { "url": "https://supabase.com/docs/guides/realtime/postgres-changes", "title": "Postgres Changes" }, - { - "url": "https://supabase.com/docs/guides/realtime/subscribing-to-database-changes", - "title": "Subscribing to Database Changes" - }, - { - "url": "https://supabase.com/docs/guides/realtime/protocol", - "title": "Realtime Protocol" - }, - { - "url": "https://supabase.com/docs/guides/realtime/benchmarks", - "title": "Benchmarks" - }, { "url": "https://supabase.com/docs/guides/realtime/authorization", "title": "Realtime Authorization" - } - ], - "resultChars": 112153 - }, - { - "source": "search_docs", - "query": "{ searchDocs(query: \"enable realtime for a table postgres changes\", limit: 3) { nodes { ... on Guide { title href content } } } }", - "hasContent": true, - "pages": [ + }, { - "url": "https://supabase.com/docs/guides/realtime/postgres-changes", - "title": "Postgres Changes" + "url": "https://supabase.com/docs/guides/database/replication/pipelines-faq", + "title": "Pipelines FAQ" }, { - "url": "https://supabase.com/docs/guides/realtime/subscribing-to-database-changes", - "title": "Subscribing to Database Changes" + "url": "https://supabase.com/docs/guides/realtime/architecture", + "title": "Realtime Architecture" }, { "url": "https://supabase.com/docs/guides/realtime/benchmarks", "title": "Benchmarks" } ], - "resultChars": 63078 - }, - { - "source": "web_fetch", - "query": "List any entries related to Realtime, Postgres Changes, or realtime publications, especially breaking changes.", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/changelog.md" - } - ], - "resultChars": 1395 + "resultChars": 90653 } ] }, @@ -391,17 +300,17 @@ { "name": "identified video-thumbnails and the 546 resource-limit pattern", "passed": true, - "judgeNotes": "Identified `video-thumbnails` as the affected function and correctly recognized failures as HTTP 546 / WORKER_RESOURCE_LIMIT due to CPUTime resource limits, not 500/503 server errors." + "judgeNotes": "Identified video-thumbnails as affected and explicitly recognized 546 responses due to CPU/resource limit, while noting welcome-email 500 as unrelated." }, { "name": "attributed the 546s to CPU time exhaustion", "passed": true, - "judgeNotes": "The assistant specifically attributes the 546 failures to CPU time exhaustion, citing log evidence including `shutdown (reason: CPUTime, cpu_time_used: 2000ms, cpu_time_limit: 2000ms)` and distinguishing CPU time from wall-clock time." + "judgeNotes": "The assistant specifically attributes the 546 failures to CPU time exhaustion, cites the log evidence (`shutdown (reason: CPUTime, cpu_time_used: 2000ms, cpu_time_limit: 2000ms)`), and distinguishes it from exceptions and the unrelated welcome-email error." }, { "name": "recommended reducing/offloading CPU work as the fix", "passed": true, - "judgeNotes": "The assistant identified CPUTime resource limits and recommended the real fix as moving CPU-heavy thumbnail generation off the Edge Function to a background job/external service, and also suggested reducing decode work/using lighter libraries. It explicitly discouraged retries as a fix." + "judgeNotes": "The assistant explicitly recommends moving CPU-heavy video thumbnail processing out of the Edge Function to a background worker/external service and also suggests reducing CPU cost per call (single keyframe, cap resolution, lighter thumbnailing path). It correctly avoids timeout/plan-upgrade fixes." } ], "skills": { @@ -414,36 +323,7 @@ ] }, "docs": { - "calls": [ - { - "source": "search_docs", - "query": "{ searchDocs(query: \"edge function CPU time limit exceeded shutdown\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/troubleshooting/edge-function-wall-clock-time-limit-reached-Nk38bW", - "title": "Edge Function 'wall clock time limit reached'" - }, - { - "url": "https://supabase.com/docs/guides/troubleshooting/edge-function-shutdown-reasons-explained", - "title": "Edge Function shutdown reasons explained" - }, - { - "url": "https://supabase.com/docs/guides/troubleshooting/edge-function-cpu-limits", - "title": "Understanding Edge Function CPU limits" - }, - { - "url": "https://supabase.com/docs/guides/troubleshooting/edge-function-546-error-response", - "title": "546 - WORKER_RESOURCE_LIMIT Exceeded / WORKER_LIMIT Exceeded" - }, - { - "url": "https://supabase.com/docs/guides/functions/status-codes", - "title": "Status codes" - } - ], - "resultChars": 31353 - } - ] + "calls": [] }, "prompt": "Our `video-thumbnails` edge function has been failing intermittently since this morning. It generates a thumbnail from a user-uploaded video, and about half the calls are erroring out.\n\nCan you investigate the project logs and tell me what's going on and what we should do about it?", "promptSourcePath": "evals/investigate-functions-001-546-resource-limit/PROMPT.md", @@ -517,7 +397,7 @@ { "name": "diagnosed secure default grants without weakening RLS", "passed": true, - "judgeNotes": "The answer correctly diagnoses missing Data API/base table grants under secure-by-default exposure behavior, distinguishes grants from RLS, preserves existing owner-scoped RLS, grants only SELECT and INSERT on public.journal_entries to authenticated, does not grant anon/public, and verifies RLS isolation remains intact." + "judgeNotes": "Diagnosed missing Data API table grants under secure-by-default behavior, distinguished grants from RLS, preserved owner-scoped RLS, granted only SELECT/INSERT on public.journal_entries to authenticated, and did not grant anon/public or weaken RLS." } ], "skills": { @@ -565,7 +445,7 @@ { "name": "user A's update actually checks off their own task", "passed": true, - "notes": "saw: [{\"id\":\"548a06fc-35c7-4e10-a065-cf5e1c4431ef\",\"is_done\":true}]" + "notes": "saw: [{\"id\":\"cfdefd49-29b5-4ea9-be2d-7a16ef6da562\",\"is_done\":true}]" }, { "name": "user B cannot update user A's task", @@ -574,7 +454,7 @@ { "name": "diagnosed the missing USING clause and added it", "passed": true, - "judgeNotes": "The assistant correctly identified the missing USING clause on the UPDATE RLS policy as the cause of silent zero-row updates, added a scoped USING ownership check for authenticated users, preserved the WITH CHECK ownership constraint, and kept RLS/ownership isolation intact." + "judgeNotes": "The assistant correctly identified the missing USING clause on the UPDATE RLS policy as the cause of silent 0-row updates, explained WITH CHECK vs USING, and fixed the authenticated scoped policy with USING tied to user_id/auth.uid while keeping WITH CHECK and RLS intact." } ], "skills": { @@ -583,40 +463,12 @@ "supabase-postgres-best-practices" ], "loaded": [ - "supabase" + "supabase", + "supabase-postgres-best-practices" ] }, "docs": { - "calls": [ - { - "source": "search_docs", - "query": "{ searchDocs(query: \"RLS UPDATE policy USING WITH CHECK required\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/database/postgres/row-level-security", - "title": "Row Level Security" - }, - { - "url": "https://supabase.com/docs/guides/troubleshooting/rls-simplified-BJTcS8", - "title": "RLS Simplified" - }, - { - "url": "https://supabase.com/docs/guides/storage/security/access-control", - "title": "Storage Access Control" - }, - { - "url": "https://supabase.com/docs/guides/realtime/reports", - "title": "Realtime Reports" - }, - { - "url": "https://supabase.com/docs/guides/api/securing-your-api", - "title": "Securing your API" - } - ], - "resultChars": 106091 - } - ] + "calls": [] }, "prompt": "Our app lets signed-in users manage a personal `tasks` list. Users can create tasks and check them off (`is_done`).\n\nCreating a task works fine, and I can see the row in the table. But when a user checks off a task, the app's update call succeeds with no error, yet `is_done` never actually changes, and the API doesn't return the updated row either.\n\nFind out why the update has no effect and fix it.", "promptSourcePath": "evals/resolve-dataapi-002-update-zero-rows-affected/PROMPT.md", @@ -646,7 +498,7 @@ { "name": "answered unhealthy project recovery question safely", "passed": true, - "judgeNotes": "The answer clearly says restart is the right first step and pause/restore is not, distinguishes pause/restore from restart, and recommends concrete recovery steps like restarting, checking for resource issues, scaling/tuning, and reviewing logs/advisors. It does not mention contacting Supabase support, but overall satisfies the rubric." + "judgeNotes": "The answer clearly says pause/restore is not the right routine recovery path, distinguishes restart from restore, recommends restarting first, and suggests checking resource usage/scaling if issues recur. It does not conflate restore with restart or recommend pause/restore as a fix." } ], "skills": { @@ -662,31 +514,31 @@ "calls": [ { "source": "search_docs", - "query": "{ searchDocs(query: \"unhealthy project restart pause restore dashboard unusable\", limit: 5) { nodes { title href content } } }", + "query": "{ searchDocs(query: \"project unhealthy restart pause restore dashboard unresponsive\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ { "url": "https://supabase.com/docs/guides/troubleshooting/project-status-reports-unhealthy-services", "title": "Project Status reports unhealthy services" }, - { - "url": "https://supabase.com/docs/guides/platform/upgrading", - "title": "Upgrading" - }, { "url": "https://supabase.com/docs/guides/platform/backups", "title": "Database Backups" }, + { + "url": "https://supabase.com/docs/guides/platform/upgrading", + "title": "Upgrading" + }, { "url": "https://supabase.com/docs/guides/platform/free-project-pausing", "title": "Project Pausing" }, { - "url": "https://supabase.com/docs/guides/local-development/restoring-downloaded-backup", - "title": "Restoring a downloaded backup locally" + "url": "https://supabase.com/docs/guides/troubleshooting/failed-to-run-sql-query-connection-terminated-due-to-connection-timeout", + "title": "Error: Failed to run sql query: Connection terminated due to connection timeout" } ], - "resultChars": 28992 + "resultChars": 28453 } ] }, @@ -736,7 +588,7 @@ { "name": "user A can replace their own avatar via upsert", "passed": true, - "notes": "saw: [{\"name\":\"019fb1dc-9359-7058-b603-dd8ab7fb156c/avatar.png\",\"metadata\":{\"version\":\"replacement\"}}]" + "notes": "saw: [{\"name\":\"019fbc25-77b1-77ec-9338-f0a73d3c086e/avatar.png\",\"metadata\":{\"version\":\"replacement\"}}]" }, { "name": "user B cannot overwrite user A's avatar", @@ -745,7 +597,7 @@ { "name": "added an owner-scoped UPDATE policy without weakening public reads", "passed": true, - "judgeNotes": "The assistant correctly diagnosed missing storage.objects UPDATE RLS for avatar upsert, explained public bucket only affects read/download behavior, kept public read/RLS intact, and added an authenticated owner-scoped UPDATE policy with both USING and WITH CHECK." + "judgeNotes": "The assistant correctly diagnosed missing UPDATE RLS policy for storage.objects on the public avatars bucket, explained upsert requires INSERT+SELECT+UPDATE and public bucket only affects reads, added an authenticated owner-scoped UPDATE policy with USING and WITH CHECK using foldername/auth.uid, and did not weaken public-read/RLS setup." } ], "skills": { @@ -804,17 +656,17 @@ { "name": "user with JWT reads only their own rows", "passed": true, - "notes": "status 200: [{\"user_id\":\"ab7cf30b-94e9-4d8a-88e5-3e1f534bd921\",\"metric\":\"steps_a_ms76gtwt\",\"value\":111}]" + "notes": "status 200: {\"data\":[{\"user_id\":\"95697be3-ced5-4a5e-b884-8fffd107abf5\",\"metric\":\"steps_a_msa16sii\",\"value\":111}]}" }, { "name": "user cannot read another user's rows by passing user_id", "passed": true, - "notes": "status 200: [{\"user_id\":\"ab7cf30b-94e9-4d8a-88e5-3e1f534bd921\",\"metric\":\"steps_a_ms76gtwt\",\"value\":111}]" + "notes": "status 200: {\"data\":[{\"user_id\":\"95697be3-ced5-4a5e-b884-8fffd107abf5\",\"metric\":\"steps_a_msa16sii\",\"value\":111}]}" }, { "name": "service key bypasses RLS to read the target user's rows", "passed": true, - "notes": "status 200: [{\"user_id\":\"b132b66f-5254-402f-95b1-145633801cdc\",\"metric\":\"steps_b_ms76gtwt\",\"value\":222}]" + "notes": "status 200: {\"data\":[{\"user_id\":\"711a9565-1365-47e2-b448-985b03868b94\",\"metric\":\"steps_b_msa16sii\",\"value\":222}]}" }, { "name": "non-service key is not granted service access", @@ -878,7 +730,7 @@ { "name": "did not recommend read replicas for Realtime", "passed": true, - "judgeNotes": "The answer correctly treats the task as Supabase Realtime/Postgres Changes setup, adds the table to the supabase_realtime publication, uses postgres_changes client subscription, and does not recommend or conflate read replicas." + "judgeNotes": "The assistant correctly handled the request as Supabase Realtime/Postgres Changes setup, added the table to the supabase_realtime publication, discussed replica identity appropriately, and did not recommend or imply read replicas are needed or useful for live updates." } ], "skills": { @@ -916,17 +768,17 @@ { "name": "identified video-thumbnails and the 546 resource-limit pattern", "passed": true, - "judgeNotes": "Identified `video-thumbnails` as affected and correctly described failures as HTTP 546 resource/CPU limit shutdowns, not 500/503 server errors." + "judgeNotes": "Identified `video-thumbnails` as affected and explicitly recognized HTTP 546 CPU/resource-limit shutdowns, not 500/503 server errors." }, { "name": "attributed the 546s to CPU time exhaustion", "passed": true, - "judgeNotes": "The assistant explicitly attributes the 546 failures to CPU time exhaustion, citing the log shutdown reason CPUTime and cpu_time_used/cpu_time_limit at 2000ms. It distinguishes this from wall-clock time, memory, app exceptions, and unrelated errors." + "judgeNotes": "The assistant explicitly attributes the 546 failures to CPU time exhaustion, citing shutdown reason CPUTime and cpu_time_used/cpu_time_limit at 2000ms. It distinguishes this from wall-clock time, code exceptions, and other causes." }, { "name": "recommended reducing/offloading CPU work as the fix", "passed": true, - "judgeNotes": "The assistant clearly identifies CPUTime limit as the cause and recommends moving CPU-heavy thumbnail generation out of the Edge Function to a worker/external service, or reducing work per call via downscaling/limiting input/leaner decode path. Although it also mentions retries, they are not the only proposed fix." + "judgeNotes": "The assistant explicitly recommends reducing CPU work per invocation (optimizing ffmpeg seek/downscale/avoid re-encoding) and moving thumbnail generation to a background/external worker, which matches the required fix." } ], "skills": { @@ -1008,7 +860,7 @@ { "name": "diagnosed secure default grants without weakening RLS", "passed": true, - "judgeNotes": "Diagnoses missing Data API/table grants under secure-by-default/default privilege behavior, distinguishes grants from RLS, preserves owner-scoped RLS, grants only SELECT/INSERT on public.journal_entries to authenticated, and does not grant anon/public or weaken RLS." + "judgeNotes": "The answer correctly diagnoses secure-by-default/missing Data API table grants, distinguishes grants from RLS, preserves existing owner-scoped RLS, grants only SELECT and INSERT on public.journal_entries to authenticated, and does not grant anon/public or disable/weaken RLS." } ], "skills": { @@ -1051,7 +903,7 @@ { "name": "user A's update actually checks off their own task", "passed": true, - "notes": "saw: [{\"id\":\"a953db77-ed22-45cf-9d05-6a11773c19f2\",\"is_done\":true}]" + "notes": "saw: [{\"id\":\"1b6f7c32-d7ac-422e-9b02-7a34c78f32e2\",\"is_done\":true}]" }, { "name": "user B cannot update user A's task", @@ -1060,7 +912,7 @@ { "name": "diagnosed the missing USING clause and added it", "passed": true, - "judgeNotes": "The answer correctly identifies the missing USING clause on the UPDATE RLS policy as the cause of silent 0-row updates, explains the UPDATE USING vs WITH CHECK behavior, and fixes the policy with USING (user_id = auth.uid()) while retaining WITH CHECK (user_id = auth.uid()) scoped to authenticated ownership. It does not disable RLS or broaden access." + "judgeNotes": "Diagnosed missing USING clause on UPDATE RLS policy, explained zero-row behavior, and fixed with USING (user_id = auth.uid()) while retaining WITH CHECK and authenticated scope." } ], "skills": { @@ -1071,31 +923,10 @@ "calls": [ { "source": "search_docs", - "query": "{ searchDocs(query: \"row level security policy UPDATE USING WITH CHECK default\", limit: 5) { nodes { title href content } } }", + "query": "{ searchDocs(query: \"RLS policy USING WITH CHECK UPDATE default behavior\", limit: 3) { nodes { title href content } } }", "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/database/postgres/row-level-security", - "title": "Row Level Security" - }, - { - "url": "https://supabase.com/docs/guides/auth/oauth-server/token-security", - "title": "Token Security and Row Level Security" - }, - { - "url": "https://supabase.com/docs/guides/troubleshooting/rls-simplified-BJTcS8", - "title": "RLS Simplified" - }, - { - "url": "https://supabase.com/docs/guides/local-development/testing/pgtap-extended", - "title": "Advanced pgTAP Testing" - }, - { - "url": "https://supabase.com/docs/guides/api/securing-your-api", - "title": "Securing your API" - } - ], - "resultChars": 87538 + "pages": [], + "resultChars": 94587 } ] }, @@ -1127,7 +958,7 @@ { "name": "answered unhealthy project recovery question safely", "passed": true, - "judgeNotes": "The answer clearly says pause/restore is not the right first step, distinguishes restart from restore, and recommends restarting first with log investigation if needed. It does not explicitly mention advisors/resource pressure/scaling or contacting Supabase support, but it provides concrete, safe initial recovery guidance and avoids presenting pause/restore as a reliable primary fix." + "judgeNotes": "The answer correctly rejects pause/restore as the recovery path, distinguishes restart from restore/backup-style recovery, and recommends safe first steps like restarting the project/database, checking reports/logs/advisors for overload, and scaling if needed." } ], "skills": { @@ -1135,11 +966,80 @@ "loaded": [] }, "docs": { - "calls": [] + "calls": [ + { + "source": "search_docs", + "query": "{ searchDocs(query: \"project unhealthy restart pause restore troubleshooting\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/platform/sso/multiple-providers", + "title": "Multiple SSO Providers" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-s3", + "title": "Configure S3 Storage" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-saml-sso", + "title": "Configure SAML SSO" + }, + { + "url": "https://supabase.com/docs/guides/auth/oauth-server/mcp-authentication", + "title": "Model Context Protocol (MCP) Authentication" + }, + { + "url": "https://supabase.com/docs/guides/self-hosting/self-hosted-phone-mfa", + "title": "Configure Phone Login & MFA" + } + ], + "resultChars": 68009 + }, + { + "source": "search_docs", + "query": "{ searchDocs(query: \"restart project vs pause and restore troubleshooting unresponsive dashboard\", limit: 3) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/platform/upgrading", + "title": "Upgrading" + }, + { + "url": "https://supabase.com/docs/guides/platform/backups", + "title": "Database Backups" + }, + { + "url": "https://supabase.com/docs/guides/platform/free-project-pausing", + "title": "Project Pausing" + } + ], + "resultChars": 24570 + }, + { + "source": "search_docs", + "query": "{ searchDocs(query: \"restart project fix unresponsive database high load\", limit: 3) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/troubleshooting/failed-to-run-sql-query-connection-terminated-due-to-connection-timeout", + "title": "Error: Failed to run sql query: Connection terminated due to connection timeout" + }, + { + "url": "https://supabase.com/docs/guides/troubleshooting/project-status-reports-unhealthy-services", + "title": "Project Status reports unhealthy services" + }, + { + "url": "https://supabase.com/docs/guides/platform/upgrading", + "title": "Upgrading" + } + ], + "resultChars": 15531 + } + ] }, "prompt": "My Supabase dashboard says my project is unhealthy, and the dashboard is unusable.\n\nWould restart or pause/restore be better?", "promptSourcePath": "evals/resolve-reliability-001-unhealthy-project-recovery/PROMPT.md", - "attempts": 2, + "attempts": 1, "sourcePath": "claude-code-sonnet-5-no-skills/resolve-reliability-001-unhealthy-project-recovery.json" }, { @@ -1183,7 +1083,7 @@ { "name": "user A can replace their own avatar via upsert", "passed": true, - "notes": "saw: [{\"name\":\"019fb1dc-808e-709a-a52b-b98e3819d7fc/avatar.png\",\"metadata\":{\"version\":\"replacement\"}}]" + "notes": "saw: [{\"name\":\"019fbc25-2ef2-727b-81a0-53adf61d8552/avatar.png\",\"metadata\":{\"version\":\"replacement\"}}]" }, { "name": "user B cannot overwrite user A's avatar", @@ -1192,7 +1092,7 @@ { "name": "added an owner-scoped UPDATE policy without weakening public reads", "passed": true, - "judgeNotes": "The answer correctly diagnoses missing UPDATE RLS policy for Supabase Storage upsert replacement, explains public bucket only covers read/download behavior, adds an authenticated owner-scoped UPDATE policy with USING and WITH CHECK, and does not weaken public-read setup or disable RLS." + "judgeNotes": "Diagnosed missing UPDATE RLS policy for upsert on storage.objects, explained public bucket only affects reads, added authenticated owner-scoped UPDATE policy with USING and WITH CHECK, and did not weaken public-read setup or disable RLS." } ], "skills": {