diff --git a/apps/web/src/data/eval-results.json b/apps/web/src/data/eval-results.json index 34c591f5..d35f2f29 100644 --- a/apps/web/src/data/eval-results.json +++ b/apps/web/src/data/eval-results.json @@ -39,7 +39,7 @@ "passed": false } ], - "prompt": "# Private Notes Function Review\n\nI built an Edge Function called `private-notes` for showing a user's saved\nprivate notes.\n\nCan you check whether there's any way one user could see another user's notes?\n\nPlease fix and deploy it if needed.", + "prompt": "I built an Edge Function called `private-notes` for showing a user's saved\nprivate notes.\n\nCan you check whether there's any way one user could see another user's notes?\n\nPlease fix and deploy it if needed.", "promptSourcePath": "evals/build-functions-004-service-role-bypass/PROMPT.md", "attempts": 1, "sourcePath": "openai-gpt-5.4-mini/build-functions-004-service-role-bypass.json" @@ -74,7 +74,7 @@ { "name": "user A lists only own files", "passed": true, - "notes": "saw: 019ecbf7-ac84-76e9-a3b5-687c1fd51010/receipt-alpha.pdf, 019ecbf7-ac84-76e9-a3b5-687c1fd51010/receipt-beta.pdf" + "notes": "saw: 019eccf2-0f77-72a2-9281-177931bc0567/receipt-alpha.pdf, 019eccf2-0f77-72a2-9281-177931bc0567/receipt-beta.pdf" }, { "name": "user B cannot read user A files", @@ -95,10 +95,10 @@ { "name": "configured private per-user storage access", "passed": true, - "judgeNotes": "Configured a private user-files bucket, owner-scoped SELECT and INSERT policies on storage.objects for authenticated users using the user-id path prefix, did not make the bucket public or use permissive/public policies, and provided supabase-js createSignedUrl code with an expiry for temporary sharing." + "judgeNotes": "Configured a private user-files bucket, owner-scoped authenticated SELECT/INSERT storage.objects policies, kept access private, and provided createSignedUrl code with an expiry." } ], - "prompt": "# Private User Files in Storage\n\nOur app lets signed-in users keep personal files like receipts and bank\nstatements. These files are private — a user must only ever be able to upload\nand download their own. The app uploads each file under a path that starts\nwith the owner's user id, e.g. `/receipt-march.pdf`.\n\nSet up a `user-files` bucket on our project and lock it down that way.\n\nUsers also sometimes share one of their files with someone else through a\ntemporary link that expires. Include the supabase-js code the app should use\nfor that.", + "prompt": "Our app lets signed-in users keep personal files like receipts and bank\nstatements. These files are private — a user must only ever be able to upload\nand download their own. The app uploads each file under a path that starts\nwith the owner's user id, e.g. `/receipt-march.pdf`.\n\nSet up a `user-files` bucket on our project and lock it down that way.\n\nUsers also sometimes share one of their files with someone else through a\ntemporary link that expires. Include the supabase-js code the app should use\nfor that.", "promptSourcePath": "evals/build-storage-001-private-bucket-access/PROMPT.md", "attempts": 1, "sourcePath": "openai-gpt-5.4-mini/build-storage-001-private-bucket-access.json" @@ -123,12 +123,12 @@ { "name": "configured the Supabase Metrics API scrape correctly", "passed": false, - "judgeNotes": "Fails: Supabase scrape uses basic_auth password with an env Secret API key instead of password_file, target is not the required b602ce33357545a18851.supabase.co/red project endpoint, and docker-compose.yml does not mount the password_file via a volume or Compose secret. Existing app scrape is preserved and HTTPS/path are correct." + "judgeNotes": "Fails: uses basic_auth password with env var instead of password_file; hardcodes/uses Secret API key via env expansion; target is variable ${SUPABASE_PROJECT_REF}.supabase.co rather than required a8c5767fb5a2473297bb.supabase.co or .red; docker-compose does not mount a password_file via volume or Compose secret. App scrape is preserved and HTTPS/path are correct." }, { "name": "documented live deployment and verification steps", - "passed": true, - "judgeNotes": "README includes creating a Supabase Secret API key, placing SUPABASE_PROJECT_REF and SUPABASE_SECRET_API_KEY in .env.local used by Compose, restarting Prometheus, and verifying via Prometheus targets." + "passed": false, + "judgeNotes": "README mentions adding SUPABASE_PROJECT_REF and SUPABASE_SECRET_API_KEY to .env.local and starting Compose, but it does not explain how to create the Secret API key, does not explicitly instruct restarting/reloading the Compose stack after adding secrets, and lacks concrete verification steps such as checking Prometheus targets or running PromQL/Grafana queries." } ], "prompt": "Can you wire my Supabase project metrics into our existing observability stack and document\nwhat we need to do to make the config live?", @@ -136,6 +136,31 @@ "attempts": 1, "sourcePath": "openai-gpt-5.4-mini/deploy-database-001-prometheus-metrics/summary.json" }, + { + "experiment": "openai-gpt-5.4-mini", + "eval": "investigate-auth-001-deleted-user-access", + "stage": "investigate", + "product": [ + "auth" + ], + "topic": [ + "security", + "sdk" + ], + "suite": "benchmark", + "passed": false, + "checks": [ + { + "name": "scorer evaluated deleted user access", + "passed": false, + "notes": "relation \"profiles\" does not exist" + } + ], + "prompt": "Last week support removed a user through our app's delete-account flow — the\napp calls the `delete_account` function over RPC as the signed-in user. This\nmorning that same person was back: still signed in, reading and saving their\ndata like nothing happened.\n\nFigure out why the account still works, fix the flow so a deleted account\nloses access, and tell me whether there is any window where they could still\nget in after the fix.\n\nOne more thing while you're at it: we're migrating off the legacy\nanon/service_role API keys to the new publishable and secret keys, and the\nteam isn't sure which one belongs in the frontend and what each means for\nRLS. Give us a quick rundown.", + "promptSourcePath": "evals/investigate-auth-001-deleted-user-access/PROMPT.md", + "attempts": 1, + "sourcePath": "openai-gpt-5.4-mini/investigate-auth-001-deleted-user-access.json" + }, { "experiment": "openai-gpt-5.4-mini", "eval": "investigate-realtime-001-subscribed-no-events", @@ -152,7 +177,7 @@ "checks": [ { "name": "orders table added to supabase_realtime publication", - "passed": true + "passed": false }, { "name": "courier_locations still in supabase_realtime publication", @@ -164,20 +189,20 @@ }, { "name": "RLS still enabled on orders", - "passed": true + "passed": false }, { "name": "staff can still read orders through RLS", - "passed": true, - "notes": "authenticated sees 0 of 0 orders" + "passed": false, + "notes": "relation \"orders\" does not exist" }, { "name": "diagnosed missing publication membership", "passed": false, - "judgeNotes": "Identified the missing supabase_realtime publication entry and added orders, but did not fix exactly that: it also created/modified the orders table, set replica identity, enabled RLS, and granted access. The rubric requires leaving existing RLS/policies/project behavior untouched and only adding orders to the existing publication." + "judgeNotes": "The answer does not diagnose that the existing supabase_realtime publication is missing the orders table, nor does it fix that with ALTER PUBLICATION supabase_realtime ADD TABLE orders. Instead it creates the publication and says there are no orders/courier_locations tables, leaving the requested Realtime orders INSERT issue unresolved." } ], - "prompt": "# Debug Realtime publication\n\nOur dispatch dashboard shows incoming orders as they happen. The courier\nlocation feed on the same page updates live without problems, but new orders\nonly show up after a page refresh.\n\nThe dashboard uses supabase-js to subscribe to INSERT events on the `orders`\ntable through postgres_changes, the same way it subscribes to courier\nlocations. The channel's status callback logs SUBSCRIBED and there are no\nerrors in the browser console.\n\nFigure out why no order events ever arrive and fix it.", + "prompt": "Our dispatch dashboard shows incoming orders as they happen. The courier\nlocation feed on the same page updates live without problems, but new orders\nonly show up after a page refresh.\n\nThe dashboard uses supabase-js to subscribe to INSERT events on the `orders`\ntable through postgres_changes, the same way it subscribes to courier\nlocations. The channel's status callback logs SUBSCRIBED and there are no\nerrors in the browser console.\n\nFigure out why no order events ever arrive and fix it.", "promptSourcePath": "evals/investigate-realtime-001-subscribed-no-events/PROMPT.md", "attempts": 1, "sourcePath": "openai-gpt-5.4-mini/investigate-realtime-001-subscribed-no-events.json" @@ -195,22 +220,22 @@ "logs" ], "suite": "benchmark", - "passed": false, + "passed": true, "checks": [ { "name": "identified image-transform and the recurring 503 pattern", "passed": true, - "judgeNotes": "The assistant identified image-transform as affected and described repeated 503 responses throughout the morning with intermittent successes, matching the required recurring pattern. It did mention avatar-upload too, but did not make old billing-webhook 503s the main issue." + "judgeNotes": "Assistant identified image-transform as affected and described recurring intermittent 503s throughout the morning, including the time-based pattern at :00/:30. It did not focus on billing-webhook or remain vague." }, { "name": "attributed recurring 503s to gateway/platform layer, not function code", - "passed": false, - "judgeNotes": "The assistant did not attribute the recurring 503s to the gateway/platform layer in front of the function. It instead framed the issue as the Edge Function layer / upstream dependency-runtime and recommended inspecting/redeploying function code, which the rubric explicitly treats as failing." + "passed": true, + "judgeNotes": "The assistant attributes the recurring 503s to intermittent gateway/platform unavailability rather than function code, and grounds this in platform/API 503 logs, healthy Edge Function runtime logs with successful 200s on the same deployment/version, and no matching backend errors." }, { "name": "recommended a concrete next step", "passed": true, - "judgeNotes": "The assistant recommended specific actionable next steps: inspect Edge Function code/dependencies, check external APIs/packages for flakiness, redeploy with retries/error handling, and add a fallback path." + "judgeNotes": "The assistant recommended concrete next steps, including checking external dependencies, adding retries/backoff, reviewing scheduled bursts, and opening a Supabase support ticket with timestamps and the 503 pattern." } ], "prompt": "Users have been reporting that image uploads are intermittently failing with server errors throughout the morning of `2026-04-28`.\n\nCan you investigate the project logs and tell me what is going on and what we\nshould do next?", @@ -261,10 +286,10 @@ { "name": "diagnosed RLS and added owner-scoped policies", "passed": true, - "judgeNotes": "Diagnosed RLS deny-all due to no policies and added authenticated-only SELECT and INSERT policies scoped to user_id = auth.uid(), while keeping RLS enabled." + "judgeNotes": "The assistant correctly identified RLS deny-all as the cause, kept RLS enabled, and created authenticated owner-scoped SELECT and INSERT policies using auth.uid() / user_id with WITH CHECK for inserts. Extra update/delete owner policies do not violate the rubric." } ], - "prompt": "# Bookmarks Dashboard Shows Nothing\n\nOur app lets signed-in users save bookmarks and view them on their dashboard. Bookmarks are stored in the `bookmarks` table and are private — a user must only ever see their own. \nUsers also need to be able to save new bookmarks from the app.\n\nI can see the rows when I query the table directly, but the dashboard shows an empty list for every user.\n\nFind out why the Data API returns nothing and fix it.", + "prompt": "Our app lets signed-in users save bookmarks and view them on their dashboard. Bookmarks are stored in the `bookmarks` table and are private — a user must only ever see their own. \nUsers also need to be able to save new bookmarks from the app.\n\nI can see the rows when I query the table directly, but the dashboard shows an empty list for every user.\n\nFind out why the Data API returns nothing and fix it.", "promptSourcePath": "evals/resolve-dataapi-001-empty-results/PROMPT.md", "attempts": 1, "sourcePath": "openai-gpt-5.4-mini/resolve-dataapi-001-empty-results.json" @@ -298,7 +323,7 @@ { "name": "query plan uses an index and avoids sequential scan", "passed": true, - "notes": "Limit (cost=55.55..55.61 rows=25 width=88)\n -> Sort (cost=55.55..55.61 rows=25 width=88)\n Sort Key: created_at DESC\n -> Bitmap Heap Scan on events (cost=4.48..54.97 rows=25 width=88)\n Recheck Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)\n -> Bitmap Index Scan on events_user_created_at_idx (cost=0.00..4.47 rows=25 width=0)\n Index Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)" + "notes": "Limit (cost=55.55..55.61 rows=25 width=88)\n -> Sort (cost=55.55..55.61 rows=25 width=88)\n Sort Key: created_at DESC\n -> Bitmap Heap Scan on events (cost=4.48..54.97 rows=25 width=88)\n Recheck Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)\n -> Bitmap Index Scan on events_user_created_at_desc_idx (cost=0.00..4.47 rows=25 width=0)\n Index Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)" }, { "name": "inserts still work", @@ -310,6 +335,51 @@ "attempts": 1, "sourcePath": "openai-gpt-5.4-mini/resolve-performance-001-slow-query-cpu-spike.json" }, + { + "experiment": "openai-gpt-5.4-mini", + "eval": "resolve-performance-002-slow-query-cpu-spike-red-herring", + "stage": "resolve", + "product": [ + "database" + ], + "topic": [ + "observability", + "sql" + ], + "suite": "benchmark", + "passed": false, + "checks": [ + { + "name": "inspected pg_stat_statements for query performance", + "passed": false + }, + { + "name": "ranked query stats by total (cumulative) execution time", + "passed": false + }, + { + "name": "ran EXPLAIN on the expensive query", + "passed": false + }, + { + "name": "created index covering user_id and created_at on events", + "passed": false + }, + { + "name": "hog query plan uses an index and avoids sequential scan", + "passed": false, + "notes": "Limit (cost=141.55..141.61 rows=24 width=88)\n -> Sort (cost=141.55..141.61 rows=24 width=88)\n Sort Key: created_at DESC\n -> Seq Scan on events (cost=0.00..141.00 rows=24 width=88)\n Filter: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)" + }, + { + "name": "inserts still work", + "passed": true + } + ], + "prompt": "My database CPU keeps spiking and the app gets slow. Can you figure out what query is causing it and make the database change needed to fix it?\n\nEnd your turn with a short summary of what you changed and why.", + "promptSourcePath": "evals/resolve-performance-002-slow-query-cpu-spike-red-herring/PROMPT.md", + "attempts": 1, + "sourcePath": "openai-gpt-5.4-mini/resolve-performance-002-slow-query-cpu-spike-red-herring.json" + }, { "experiment": "openai-gpt-5.4-mini", "eval": "resolve-security-002-rls-cross-tenant-leak", @@ -407,7 +477,7 @@ "passed": true } ], - "prompt": "# Private Notes Function Review\n\nI built an Edge Function called `private-notes` for showing a user's saved\nprivate notes.\n\nCan you check whether there's any way one user could see another user's notes?\n\nPlease fix and deploy it if needed.", + "prompt": "I built an Edge Function called `private-notes` for showing a user's saved\nprivate notes.\n\nCan you check whether there's any way one user could see another user's notes?\n\nPlease fix and deploy it if needed.", "promptSourcePath": "evals/build-functions-004-service-role-bypass/PROMPT.md", "attempts": 1, "sourcePath": "openai-gpt-5.4-nano/build-functions-004-service-role-bypass.json" @@ -425,7 +495,7 @@ "sdk" ], "suite": "benchmark", - "passed": false, + "passed": true, "checks": [ { "name": "bucket user-files exists", @@ -442,7 +512,7 @@ { "name": "user A lists only own files", "passed": true, - "notes": "saw: 019ecbf7-9310-744c-911a-6ce1c0a7afa4/receipt-alpha.pdf, 019ecbf7-9310-744c-911a-6ce1c0a7afa4/receipt-beta.pdf" + "notes": "saw: 019eccf2-79c5-743c-a546-fc7ac7ad33b3/receipt-alpha.pdf, 019eccf2-79c5-743c-a546-fc7ac7ad33b3/receipt-beta.pdf" }, { "name": "user B cannot read user A files", @@ -462,11 +532,11 @@ }, { "name": "configured private per-user storage access", - "passed": false, - "judgeNotes": "Bucket is private, RLS is enabled, owner-scoped SELECT/INSERT policies exist, and createSignedUrl with expiry is provided. However, the policies omit `TO authenticated`, so they default to PUBLIC, which scopes object access policies to public/anon as well as authenticated, a listed fail condition." + "passed": true, + "judgeNotes": "Configured a private user-files bucket, kept RLS enabled on storage.objects, added authenticated owner-scoped INSERT and SELECT policies, and provided supabase-js createSignedUrl code with expiry." } ], - "prompt": "# Private User Files in Storage\n\nOur app lets signed-in users keep personal files like receipts and bank\nstatements. These files are private — a user must only ever be able to upload\nand download their own. The app uploads each file under a path that starts\nwith the owner's user id, e.g. `/receipt-march.pdf`.\n\nSet up a `user-files` bucket on our project and lock it down that way.\n\nUsers also sometimes share one of their files with someone else through a\ntemporary link that expires. Include the supabase-js code the app should use\nfor that.", + "prompt": "Our app lets signed-in users keep personal files like receipts and bank\nstatements. These files are private — a user must only ever be able to upload\nand download their own. The app uploads each file under a path that starts\nwith the owner's user id, e.g. `/receipt-march.pdf`.\n\nSet up a `user-files` bucket on our project and lock it down that way.\n\nUsers also sometimes share one of their files with someone else through a\ntemporary link that expires. Include the supabase-js code the app should use\nfor that.", "promptSourcePath": "evals/build-storage-001-private-bucket-access/PROMPT.md", "attempts": 1, "sourcePath": "openai-gpt-5.4-nano/build-storage-001-private-bucket-access.json" @@ -491,12 +561,12 @@ { "name": "configured the Supabase Metrics API scrape correctly", "passed": false, - "judgeNotes": "Fails: Supabase target is a placeholder, not 20b46eaca601447dac66.supabase.co/red; basic_auth uses a hardcoded password field instead of password_file; docker-compose.yml does not mount or provide the password_file via volume or Compose secret. Existing app scrape is preserved and HTTPS/path are correct." + "judgeNotes": "No Supabase Metrics API scrape was added. prometheus.yml only preserves the app scrape, and docker-compose.yml does not mount or define any password_file/secret for HTTP Basic Auth." }, { "name": "documented live deployment and verification steps", "passed": false, - "judgeNotes": "README includes correct Supabase endpoint/auth, Secret API key creation, restart, and concrete Prometheus verification. However it instructs users to put the key directly in prometheus.yml and does not describe placing a matching secret file or Compose secret setup, which the rubric requires." + "judgeNotes": "README lacks required Secret API key creation, matching secret file placement, restart/reload steps, and concrete verification via Prometheus targets/PromQL/Grafana." } ], "prompt": "Can you wire my Supabase project metrics into our existing observability stack and document\nwhat we need to do to make the config live?", @@ -504,6 +574,31 @@ "attempts": 1, "sourcePath": "openai-gpt-5.4-nano/deploy-database-001-prometheus-metrics/summary.json" }, + { + "experiment": "openai-gpt-5.4-nano", + "eval": "investigate-auth-001-deleted-user-access", + "stage": "investigate", + "product": [ + "auth" + ], + "topic": [ + "security", + "sdk" + ], + "suite": "benchmark", + "passed": false, + "checks": [ + { + "name": "scorer evaluated deleted user access", + "passed": false, + "notes": "relation \"profiles\" does not exist" + } + ], + "prompt": "Last week support removed a user through our app's delete-account flow — the\napp calls the `delete_account` function over RPC as the signed-in user. This\nmorning that same person was back: still signed in, reading and saving their\ndata like nothing happened.\n\nFigure out why the account still works, fix the flow so a deleted account\nloses access, and tell me whether there is any window where they could still\nget in after the fix.\n\nOne more thing while you're at it: we're migrating off the legacy\nanon/service_role API keys to the new publishable and secret keys, and the\nteam isn't sure which one belongs in the frontend and what each means for\nRLS. Give us a quick rundown.", + "promptSourcePath": "evals/investigate-auth-001-deleted-user-access/PROMPT.md", + "attempts": 1, + "sourcePath": "openai-gpt-5.4-nano/investigate-auth-001-deleted-user-access.json" + }, { "experiment": "openai-gpt-5.4-nano", "eval": "investigate-realtime-001-subscribed-no-events", @@ -542,10 +637,10 @@ { "name": "diagnosed missing publication membership", "passed": false, - "judgeNotes": "The assistant did not apply the required fix. It claimed there were no application tables and no publication, asked the user to verify the project, and did not run ALTER PUBLICATION supabase_realtime ADD TABLE orders. It also mentioned RLS as something to check, rather than fixing exactly the missing orders table in the existing realtime publication." + "judgeNotes": "The assistant incorrectly diagnosed RLS/SELECT policies as the root cause and suggested changing policies/replica identity. It did not identify or fix the missing orders table in the supabase_realtime publication." } ], - "prompt": "# Debug Realtime publication\n\nOur dispatch dashboard shows incoming orders as they happen. The courier\nlocation feed on the same page updates live without problems, but new orders\nonly show up after a page refresh.\n\nThe dashboard uses supabase-js to subscribe to INSERT events on the `orders`\ntable through postgres_changes, the same way it subscribes to courier\nlocations. The channel's status callback logs SUBSCRIBED and there are no\nerrors in the browser console.\n\nFigure out why no order events ever arrive and fix it.", + "prompt": "Our dispatch dashboard shows incoming orders as they happen. The courier\nlocation feed on the same page updates live without problems, but new orders\nonly show up after a page refresh.\n\nThe dashboard uses supabase-js to subscribe to INSERT events on the `orders`\ntable through postgres_changes, the same way it subscribes to courier\nlocations. The channel's status callback logs SUBSCRIBED and there are no\nerrors in the browser console.\n\nFigure out why no order events ever arrive and fix it.", "promptSourcePath": "evals/investigate-realtime-001-subscribed-no-events/PROMPT.md", "attempts": 1, "sourcePath": "openai-gpt-5.4-nano/investigate-realtime-001-subscribed-no-events.json" @@ -563,22 +658,22 @@ "logs" ], "suite": "benchmark", - "passed": false, + "passed": true, "checks": [ { "name": "identified image-transform and the recurring 503 pattern", "passed": true, - "judgeNotes": "Assistant identified image-transform as the affected function and described recurring HTTP 503 responses throughout the morning of 2026-04-28, covering most of the gateway failures across the 07:00Z-12:00Z window. It did not incorrectly focus on billing-webhook." + "judgeNotes": "The assistant identified image-transform as the affected function and described intermittent/recurring 503 responses throughout the morning of 2026-04-28, covering most of the gateway failures from ~07:00Z to 12:00Z." }, { "name": "attributed recurring 503s to gateway/platform layer, not function code", - "passed": false, - "judgeNotes": "The assistant attributes the recurring image-transform 503s to the Edge Function/dependency/runtime behavior and recommends reviewing/fixing function dependencies and hardening function code. It does not identify the gateway/platform layer as the source or ground that attribution in observations like missing invocation/runtime rows, unchanged deployment_id, or distinction from function-level errors." + "passed": true, + "judgeNotes": "Attributes the 503s to the API/gateway/pre-function layer and grounds this in the observation that gateway logs show 503s while image-transform function logs show only successful 200 executions in the same window. Although it suggests adding logging/redeploying for observability, it does not primarily blame function code." }, { "name": "recommended a concrete next step", "passed": true, - "judgeNotes": "The assistant recommended concrete next steps, including inspecting the Edge Function dependencies/configuration, checking client error payloads, mitigating by bypassing/rolling back image processing, and adding retries/timeouts/concurrency limits." + "judgeNotes": "The assistant recommended concrete next steps: implement retries, add Edge Function error logging, redeploy, and check Storage-specific logs / upload flow to isolate the failing path." } ], "prompt": "Users have been reporting that image uploads are intermittently failing with server errors throughout the morning of `2026-04-28`.\n\nCan you investigate the project logs and tell me what is going on and what we\nshould do next?", @@ -629,10 +724,10 @@ { "name": "diagnosed RLS and added owner-scoped policies", "passed": true, - "judgeNotes": "Diagnosed RLS deny-all due to no policies, kept RLS enabled, and created authenticated owner-scoped SELECT and INSERT policies using user_id = auth.uid() / WITH CHECK. Extra UPDATE/DELETE policies are acceptable." + "judgeNotes": "Diagnosed RLS deny-all due to no policies, kept RLS enabled, and created authenticated SELECT and INSERT policies scoped to user_id = auth.uid() with WITH CHECK for inserts." } ], - "prompt": "# Bookmarks Dashboard Shows Nothing\n\nOur app lets signed-in users save bookmarks and view them on their dashboard. Bookmarks are stored in the `bookmarks` table and are private — a user must only ever see their own. \nUsers also need to be able to save new bookmarks from the app.\n\nI can see the rows when I query the table directly, but the dashboard shows an empty list for every user.\n\nFind out why the Data API returns nothing and fix it.", + "prompt": "Our app lets signed-in users save bookmarks and view them on their dashboard. Bookmarks are stored in the `bookmarks` table and are private — a user must only ever see their own. \nUsers also need to be able to save new bookmarks from the app.\n\nI can see the rows when I query the table directly, but the dashboard shows an empty list for every user.\n\nFind out why the Data API returns nothing and fix it.", "promptSourcePath": "evals/resolve-dataapi-001-empty-results/PROMPT.md", "attempts": 1, "sourcePath": "openai-gpt-5.4-nano/resolve-dataapi-001-empty-results.json" @@ -649,11 +744,11 @@ "sql" ], "suite": "benchmark", - "passed": true, + "passed": false, "checks": [ { "name": "inspected pg_stat_statements for query performance", - "passed": true + "passed": false }, { "name": "ran EXPLAIN on the expensive query", @@ -678,6 +773,51 @@ "attempts": 1, "sourcePath": "openai-gpt-5.4-nano/resolve-performance-001-slow-query-cpu-spike.json" }, + { + "experiment": "openai-gpt-5.4-nano", + "eval": "resolve-performance-002-slow-query-cpu-spike-red-herring", + "stage": "resolve", + "product": [ + "database" + ], + "topic": [ + "observability", + "sql" + ], + "suite": "benchmark", + "passed": true, + "checks": [ + { + "name": "inspected pg_stat_statements for query performance", + "passed": true + }, + { + "name": "ranked query stats by total (cumulative) execution time", + "passed": true + }, + { + "name": "ran EXPLAIN on the expensive query", + "passed": true + }, + { + "name": "created index covering user_id and created_at on events", + "passed": true + }, + { + "name": "hog query plan uses an index and avoids sequential scan", + "passed": true, + "notes": "Limit (cost=65.16..65.23 rows=30 width=88)\n -> Sort (cost=65.16..65.23 rows=30 width=88)\n Sort Key: created_at DESC\n -> Bitmap Heap Scan on events (cost=4.51..64.42 rows=30 width=88)\n Recheck Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)\n -> Bitmap Index Scan on events_user_id_created_at_idx (cost=0.00..4.51 rows=30 width=0)\n Index Cond: (user_id = '00000000-0000-0000-0000-000000000001'::uuid)" + }, + { + "name": "inserts still work", + "passed": true + } + ], + "prompt": "My database CPU keeps spiking and the app gets slow. Can you figure out what query is causing it and make the database change needed to fix it?\n\nEnd your turn with a short summary of what you changed and why.", + "promptSourcePath": "evals/resolve-performance-002-slow-query-cpu-spike-red-herring/PROMPT.md", + "attempts": 1, + "sourcePath": "openai-gpt-5.4-nano/resolve-performance-002-slow-query-cpu-spike-red-herring.json" + }, { "experiment": "openai-gpt-5.4-nano", "eval": "resolve-security-002-rls-cross-tenant-leak", diff --git a/evals/resolve-performance-002-slow-query-cpu-spike-red-herring/EVAL.ts b/evals/resolve-performance-002-slow-query-cpu-spike-red-herring/EVAL.ts new file mode 100644 index 00000000..873ef86a --- /dev/null +++ b/evals/resolve-performance-002-slow-query-cpu-spike-red-herring/EVAL.ts @@ -0,0 +1,153 @@ +import { + serializeTranscript, + type CheckResult, + type ToolEvalContext, + type ToolScorer, +} from "@supabase-evals/core"; +import { stripIndent } from "common-tags"; + +const TARGET_USER = "00000000-0000-0000-0000-000000000001"; + +const scorer: ToolScorer = async (ctx) => { + try { + const checks: CheckResult[] = [ + checkQueriedPgStatStatements(ctx), + checkRankedByTotalExecTime(ctx), + checkRanExplain(ctx), + await checkCreatedRecentEventsIndex(ctx), + await checkQueryPlanUsesIndex(ctx), + await checkInsertsStillWork(ctx), + ]; + + return { + passed: checks.every((check) => check.passed), + checks, + }; + } catch (error) { + const msg = error instanceof Error ? error.message : String(error); + return { + passed: false, + checks: [ + { + name: "scorer evaluated CPU spike fix", + passed: false, + notes: msg, + }, + ], + }; + } +}; + +export default scorer; + +/** Checks that the agent inspected query statistics rather than guessing the fix. */ +function checkQueriedPgStatStatements(ctx: ToolEvalContext): CheckResult { + const sql = executedSql(ctx); + + return { + name: "inspected pg_stat_statements for query performance", + passed: /\bpg_stat_statements\b/i.test(sql), + }; +} + +/** + * The crux of this scenario: the slow-query logs and the mean/max columns point + * at the rare audit_log report (the decoy). The CPU hog only surfaces when + * ranking by cumulative execution time, so a correct diagnosis sorts the query + * stats by total_exec_time rather than per-call latency. + */ +function checkRankedByTotalExecTime(ctx: ToolEvalContext): CheckResult { + const sql = executedSql(ctx); + + return { + name: "ranked query stats by total (cumulative) execution time", + passed: /\btotal_exec_time\b/i.test(sql), + }; +} + +/** Checks that the agent used EXPLAIN on the expensive events query (the hog). */ +function checkRanExplain(ctx: ToolEvalContext): CheckResult { + const sql = executedSql(ctx); + + return { + name: "ran EXPLAIN on the expensive query", + passed: + /\bexplain\b/i.test(sql) && + /\bevents\b/i.test(sql) && + /\buser_id\b/i.test(sql), + }; +} + +/** Checks for the index shape needed by the recent events lookup (the CPU hog). */ +async function checkCreatedRecentEventsIndex(ctx: ToolEvalContext): Promise { + const { rows } = await ctx.query(stripIndent` + SELECT indexname, indexdef + FROM pg_indexes + WHERE schemaname = 'public' + AND tablename = 'events'; + `); + + const hasCoveringIndex = rows.some((row) => { + const def = row.indexdef; + return ( + typeof def === "string" && + /ON\s+(?:public\.)?events\s+.*\(\s*user_id\s*,\s*created_at/i.test(def) + ); + }); + + return { + name: "created index covering user_id and created_at on events", + passed: hasCoveringIndex, + }; +} + +/** + * Decisive correctness check: verifies Postgres plans the high-frequency hog + * lookup with an index and no sequential scan. An agent that only fixed the + * audit_log decoy leaves this query on a seq scan and fails here. + */ +async function checkQueryPlanUsesIndex(ctx: ToolEvalContext): Promise { + const { rows } = await ctx.query(stripIndent` + EXPLAIN SELECT id, kind, payload, created_at + FROM events + WHERE user_id = '${TARGET_USER}' + ORDER BY created_at DESC + LIMIT 50; + `); + const plan = rows.map((row) => Object.values(row).join(" ")).join("\n"); + + return { + name: "hog query plan uses an index and avoids sequential scan", + passed: + /(Index Scan|Index Only Scan|Bitmap Index Scan)/i.test(plan) && + !/Seq Scan on events/i.test(plan), + notes: plan, + }; +} + +/** Confirms the schema change did not break normal inserts into events. */ +async function checkInsertsStillWork(ctx: ToolEvalContext): Promise { + const { rows } = await ctx.query(stripIndent` + INSERT INTO events (user_id, kind, payload) + VALUES ('${TARGET_USER}', 'insert_probe', '{"ok": true}'::jsonb) + RETURNING id; + `); + + return { + name: "inserts still work", + passed: rows.length === 1, + }; +} + +/** Collects SQL-like text from tool calls and transcript entries for workflow checks. */ +function executedSql(ctx: ToolEvalContext): string { + const toolCallSql = ctx.toolCalls + .flatMap((call) => Object.values(call.body)) + .filter((value): value is string => typeof value === "string") + .join("\n"); + + return [ + toolCallSql, + serializeTranscript(ctx.transcript, { includeToolCallInputs: true }), + ].join("\n"); +} diff --git a/evals/resolve-performance-002-slow-query-cpu-spike-red-herring/PROMPT.md b/evals/resolve-performance-002-slow-query-cpu-spike-red-herring/PROMPT.md new file mode 100644 index 00000000..a79f8ba2 --- /dev/null +++ b/evals/resolve-performance-002-slow-query-cpu-spike-red-herring/PROMPT.md @@ -0,0 +1,14 @@ +--- +stage: resolve +suite: benchmark +product: + - database +topic: + - observability + - sql +motivation: AI-841, AI-824, https://supabase.com/docs/guides/troubleshooting/high-cpu-usage +--- + +My database CPU keeps spiking and the app gets slow. Can you figure out what query is causing it and make the database change needed to fix it? + +End your turn with a short summary of what you changed and why. diff --git a/evals/resolve-performance-002-slow-query-cpu-spike-red-herring/remote/logs.jsonl b/evals/resolve-performance-002-slow-query-cpu-spike-red-herring/remote/logs.jsonl new file mode 100644 index 00000000..ba0ba4bf --- /dev/null +++ b/evals/resolve-performance-002-slow-query-cpu-spike-red-herring/remote/logs.jsonl @@ -0,0 +1,4 @@ +{"id":"a1","ts":"2026-04-28T10:00:00Z","source":"postgres","level":"warning","message":"duration: 3210 ms execute : SELECT id, actor, action, details, created_at FROM audit_log WHERE action = $1 ORDER BY created_at DESC LIMIT 100","metadata":{"query_hash":"audit_action_recent_h1","duration_ms":3210,"table":"audit_log"}} +{"id":"a2","ts":"2026-04-28T10:04:00Z","source":"postgres","level":"warning","message":"duration: 2980 ms execute : SELECT id, actor, action, details, created_at FROM audit_log WHERE action = $1 ORDER BY created_at DESC LIMIT 100","metadata":{"query_hash":"audit_action_recent_h1","duration_ms":2980,"table":"audit_log"}} +{"id":"a3","ts":"2026-04-28T10:09:00Z","source":"postgres","level":"warning","message":"duration: 3520 ms execute : SELECT id, actor, action, details, created_at FROM audit_log WHERE action = $1 ORDER BY created_at DESC LIMIT 100","metadata":{"query_hash":"audit_action_recent_h1","duration_ms":3520,"table":"audit_log"}} +{"id":"a4","ts":"2026-04-28T10:13:00Z","source":"postgres","level":"warning","message":"duration: 3040 ms execute : SELECT id, actor, action, details, created_at FROM audit_log WHERE action = $1 ORDER BY created_at DESC LIMIT 100","metadata":{"query_hash":"audit_action_recent_h1","duration_ms":3040,"table":"audit_log"}} diff --git a/evals/resolve-performance-002-slow-query-cpu-spike-red-herring/remote/project.sql b/evals/resolve-performance-002-slow-query-cpu-spike-red-herring/remote/project.sql new file mode 100644 index 00000000..3bf7e5f4 --- /dev/null +++ b/evals/resolve-performance-002-slow-query-cpu-spike-red-herring/remote/project.sql @@ -0,0 +1,155 @@ +-- Red-herring CPU spike scenario. +-- +-- Postgres only writes a statement to the logs when its execution time exceeds +-- log_min_duration_statement (disabled by default at -1; when enabled it is set +-- to a high threshold to capture only slow statements). So the slow-query logs +-- structurally surface long-running statements and never a cheap, high-frequency +-- query -- even when that frequent query is the real CPU driver. +-- +-- The real CPU hog here is a fast-but-very-frequent per-user lookup on `events` +-- (~22 ms x 486k calls) that stays below that threshold, so it is INVISIBLE in +-- the logs. The slow-query logs are instead dominated by a genuinely slow but +-- rare report query on `audit_log`. An agent that diagnoses from the logs (or +-- sorts pg_stat_statements by mean/max time) will "fix" the decoy and leave the +-- CPU spike in place. The only way to the correct fix is ranking +-- pg_stat_statements by total_exec_time (calls x mean_exec_time). + +-- Enable slow-query logging at a 1s threshold. log_min_duration_statement is +-- disabled by default (-1), so without this nothing would be logged at all; +-- setting it to 1s is what makes the ~3s audit_log report show up in the logs +-- while the ~22ms events hog stays under the bar and never appears there. +DO $$ +BEGIN + EXECUTE format( + 'ALTER DATABASE %I SET log_min_duration_statement = %L', + current_database(), + '1s' + ); +END $$; + +CREATE TABLE events ( + id uuid PRIMARY KEY DEFAULT gen_random_uuid(), + user_id uuid NOT NULL, + kind text NOT NULL, + payload jsonb NOT NULL DEFAULT '{}'::jsonb, + created_at timestamptz NOT NULL DEFAULT now() +); + +CREATE TABLE audit_log ( + id uuid PRIMARY KEY DEFAULT gen_random_uuid(), + actor uuid NOT NULL, + action text NOT NULL, + details jsonb NOT NULL DEFAULT '{}'::jsonb, + created_at timestamptz NOT NULL DEFAULT now() +); + +CREATE TABLE accounts ( + id uuid PRIMARY KEY DEFAULT gen_random_uuid(), + name text NOT NULL +); + +INSERT INTO accounts (name) +SELECT 'account-' || n +FROM generate_series(1, 20) AS n; + +-- High-frequency hog target: ~120 rows per user across 50 users. +INSERT INTO events (user_id, kind, payload, created_at) +SELECT + ('00000000-0000-0000-0000-' || lpad((((n - 1) % 50) + 1)::text, 12, '0'))::uuid, + CASE WHEN n % 5 = 0 THEN 'checkout' ELSE 'page_view' END, + jsonb_build_object('seq', n), + '2026-04-28T10:00:00Z'::timestamptz - (n || ' seconds')::interval +FROM generate_series(1, 6000) AS n; + +-- Decoy table for the slow-but-rare report query that fills the logs. +INSERT INTO audit_log (actor, action, details, created_at) +SELECT + ('00000000-0000-0000-0000-' || lpad((((n - 1) % 50) + 1)::text, 12, '0'))::uuid, + (ARRAY['login', 'logout', 'update_profile', 'delete_account', 'export_data'])[(n % 5) + 1], + jsonb_build_object('seq', n), + '2026-04-28T10:00:00Z'::timestamptz - (n || ' seconds')::interval +FROM generate_series(1, 8000) AS n; + +-- Eval-scoped compatibility table for Supabase's Query Performance report, +-- which is backed by pg_stat_statements in real projects. +CREATE TABLE pg_stat_statements ( + userid oid NOT NULL DEFAULT 10::oid, + dbid oid NOT NULL DEFAULT 5::oid, + queryid bigint PRIMARY KEY, + query text NOT NULL, + calls bigint NOT NULL, + rows bigint NOT NULL DEFAULT 0, + total_plan_time double precision NOT NULL DEFAULT 0, + total_exec_time double precision NOT NULL, + mean_plan_time double precision NOT NULL DEFAULT 0, + mean_exec_time double precision NOT NULL, + max_exec_time double precision NOT NULL, + shared_blks_hit bigint NOT NULL DEFAULT 0, + shared_blks_read bigint NOT NULL DEFAULT 0, + shared_blks_dirtied bigint NOT NULL DEFAULT 0, + shared_blks_written bigint NOT NULL DEFAULT 0 +); + +-- Stats deliberately diverge by ranking dimension: +-- total_exec_time -> events hog dominates (~99%): the correct culprit. +-- mean/max_exec_time -> audit_log report wins: the decoy the logs point at. +INSERT INTO pg_stat_statements ( + queryid, + query, + calls, + rows, + total_exec_time, + mean_exec_time, + max_exec_time, + shared_blks_hit, + shared_blks_read +) +VALUES + -- Real CPU hog: tiny per-call, enormous call count, dominant total time. + -- Below the slow-query log threshold, so it never appears in logs.jsonl. + ( + 841001, + 'SELECT id, kind, payload, created_at FROM events WHERE user_id = $1 ORDER BY created_at DESC LIMIT 50', + 486000, + 24300000, + 10692000, + 22, + 160, + 9100000, + 7400000 + ), + -- Decoy: genuinely slow per call but rare, so total time is negligible. + -- Tops the slow-query logs and the mean/max columns. + ( + 841002, + 'SELECT id, actor, action, details, created_at FROM audit_log WHERE action = $1 ORDER BY created_at DESC LIMIT 100', + 14, + 1400, + 43680, + 3120, + 3520, + 420, + 96000 + ), + ( + 841003, + 'SELECT id, name FROM accounts LIMIT 20', + 900, + 18000, + 9900, + 11, + 28, + 17400, + 60 + ), + ( + 841004, + 'SELECT count(*) FROM audit_log WHERE action = $1', + 210, + 210, + 5040, + 24, + 70, + 20300, + 180 + );