From 0f01288d50b0303b3f7e5e640e3f2cb7e88e980c Mon Sep 17 00:00:00 2001
From: Miranda Limonczenko <miranda.limonczenko@supabase.io>
Date: Wed, 22 Jul 2026 16:12:44 -0700
Subject: [PATCH 1/7] feat: add docs homepage quickstart eval

Adds a happy-path smoke test for the "Help me get set up with
Supabase" prompt shown on the docs homepage, seeded with a bare
Next.js app so the agent has a real project to detect and initialize.
---
 evals/build-cli-004-quickstart-nextjs/EVAL.ts | 101 ++++++++++++++++++
 .../build-cli-004-quickstart-nextjs/PROMPT.md |  15 +++
 .../build-cli-004-quickstart-nextjs/README.md |  11 ++
 .../local/app/layout.tsx                      |  18 ++++
 .../local/app/page.tsx                        |   7 ++
 .../local/next-env.d.ts                       |   5 +
 .../local/next.config.mjs                     |   4 +
 .../local/package.json                        |  21 ++++
 .../local/tsconfig.json                       |  21 ++++
 9 files changed, 203 insertions(+)
 create mode 100644 evals/build-cli-004-quickstart-nextjs/EVAL.ts
 create mode 100644 evals/build-cli-004-quickstart-nextjs/PROMPT.md
 create mode 100644 evals/build-cli-004-quickstart-nextjs/README.md
 create mode 100644 evals/build-cli-004-quickstart-nextjs/local/app/layout.tsx
 create mode 100644 evals/build-cli-004-quickstart-nextjs/local/app/page.tsx
 create mode 100644 evals/build-cli-004-quickstart-nextjs/local/next-env.d.ts
 create mode 100644 evals/build-cli-004-quickstart-nextjs/local/next.config.mjs
 create mode 100644 evals/build-cli-004-quickstart-nextjs/local/package.json
 create mode 100644 evals/build-cli-004-quickstart-nextjs/local/tsconfig.json

diff --git a/evals/build-cli-004-quickstart-nextjs/EVAL.ts b/evals/build-cli-004-quickstart-nextjs/EVAL.ts
new file mode 100644
index 00000000..01b42fe6
--- /dev/null
+++ b/evals/build-cli-004-quickstart-nextjs/EVAL.ts
@@ -0,0 +1,101 @@
+import {
+  judge,
+  serializeTranscript,
+  type CheckResult,
+  type LocalStackEvalContext,
+  type LocalStackScorer,
+} from "@supabase-evals/core";
+import { stripIndent } from "common-tags";
+
+const PLUGIN_INSTALL_PATTERN =
+  /npx\s+plugins\s+add\s+supabase-community\/supabase-plugin/i;
+
+const scorer: LocalStackScorer = async (ctx) => {
+  try {
+    const requiredChecks: CheckResult[] = [
+      await checkSupabaseInitialized(ctx),
+      await checkCliFunctional(ctx),
+      await checkNextStepsSuggested(ctx),
+    ];
+
+    return {
+      passed: requiredChecks.every((check) => check.passed),
+      checks: [...requiredChecks, checkPluginInstallAttempted(ctx)],
+    };
+  } catch (error) {
+    const msg = error instanceof Error ? error.message : String(error);
+    return {
+      passed: false,
+      checks: [
+        { name: "scorer evaluated quickstart setup", passed: false, notes: msg },
+      ],
+    };
+  }
+};
+
+export default scorer;
+
+async function checkSupabaseInitialized(
+  ctx: LocalStackEvalContext,
+): Promise<CheckResult> {
+  const exists = await ctx.fileExists("supabase/config.toml");
+  return {
+    name: "project initialized (supabase/config.toml exists)",
+    passed: exists,
+  };
+}
+
+async function checkCliFunctional(
+  ctx: LocalStackEvalContext,
+): Promise<CheckResult> {
+  const result = await ctx.exec("supabase --version");
+  return {
+    name: "supabase CLI is installed and runnable",
+    passed: result.ok,
+    notes: result.ok ? undefined : result.stderr || result.stdout,
+  };
+}
+
+async function checkNextStepsSuggested(
+  ctx: LocalStackEvalContext,
+): Promise<CheckResult> {
+  const verdict = await judge({
+    input: serializeTranscript(ctx.transcript, { includeToolCallInputs: true }),
+    rubric: stripIndent`
+      Pass if the assistant's final response suggests next steps for using
+      Supabase in this Next.js project that are relevant and specific to what
+      was just set up.
+
+      A passing answer should reference at least one concrete, relevant next
+      step, such as:
+      - starting the local stack (supabase start) or creating a first migration
+      - installing/using @supabase/supabase-js (or an SSR helper) to connect
+        the Next.js app
+      - setting environment variables for the local API URL and anon key
+      - exploring Studio or writing a first table/schema
+
+      Fail if the assistant stops after installation/init with no concrete
+      next steps, or the suggestions are generic enough to apply to any
+      project regardless of Supabase or Next.js.
+    `,
+  });
+
+  return {
+    name: "suggested relevant next steps for a Supabase + Next.js project",
+    passed: verdict.passed,
+    judgeNotes: verdict.notes,
+  };
+}
+
+// Informational only — not required for `passed`. Skills are configured at
+// the experiment level, not per-eval, so whether this step is even necessary
+// depends on which experiment ran it.
+function checkPluginInstallAttempted(ctx: LocalStackEvalContext): CheckResult {
+  const attempted = ctx.toolCalls.some(
+    (call) => call.command && PLUGIN_INSTALL_PATTERN.test(call.command),
+  );
+  return {
+    name: "(informational) attempted `npx plugins add supabase-community/supabase-plugin`",
+    passed: attempted,
+  };
+}
diff --git a/evals/build-cli-004-quickstart-nextjs/PROMPT.md b/evals/build-cli-004-quickstart-nextjs/PROMPT.md
new file mode 100644
index 00000000..8eb9f126
--- /dev/null
+++ b/evals/build-cli-004-quickstart-nextjs/PROMPT.md
@@ -0,0 +1,15 @@
+---
+stage: build
+suite: regression
+interface: cli
+product:
+  - database
+  - data-api
+topic:
+  - sdk
+services: []
+projectRunning: false
+motivation: the "Help me get set up with Supabase" prompt shown on the supabase.com/docs homepage
+---
+
+Help me get set up with Supabase. Do the following: 1. Install the Supabase CLI globally with `npm install -g supabase`. 2. Install the Supabase Plugin with `npx plugins add supabase-community/supabase-plugin`. 3. Review my project and determine whether Supabase is already initialized. If it is not initialized, run `supabase init`. 4. Suggest the most relevant next steps.
diff --git a/evals/build-cli-004-quickstart-nextjs/README.md b/evals/build-cli-004-quickstart-nextjs/README.md
new file mode 100644
index 00000000..f0b3f856
--- /dev/null
+++ b/evals/build-cli-004-quickstart-nextjs/README.md
@@ -0,0 +1,11 @@
+## Adding another docs quickstart test
+
+This guide explains how to add an eval for another AI setup prompt from the docs, such as a different framework's quickstart page.
+
+1. Create a sibling folder, for example `build-cli-005-quickstart-<framework>`.
+2. Copy the target prompt verbatim into `PROMPT.md`. Don't paraphrase it or add detail that the prompt doesn't already give.
+3. Seed `local/` with a minimal, unmodified starter for that framework. Don't include a `supabase/` directory, so the "is Supabase already initialized" step has something real to check.
+4. Reuse this eval's `EVAL.ts` checks. Change only the judge rubric's framework-specific wording.
+5. Validate the eval before you open a PR. Run `pnpm eval:dry -- --eval <new-eval-id> --experiment claude-code-sonnet-5` to check the setup, then run `pnpm eval -- --eval <new-eval-id> --experiment claude-code-sonnet-5` for a full pass.
+
+For the full eval-authoring workflow, including frontmatter fields, suite selection, and submitting for review, see `CONTRIBUTING.md`.
diff --git a/evals/build-cli-004-quickstart-nextjs/local/app/layout.tsx b/evals/build-cli-004-quickstart-nextjs/local/app/layout.tsx
new file mode 100644
index 00000000..ff4380d8
--- /dev/null
+++ b/evals/build-cli-004-quickstart-nextjs/local/app/layout.tsx
@@ -0,0 +1,18 @@
+import type { Metadata } from 'next';
+
+export const metadata: Metadata = {
+  title: 'My App',
+  description: 'Generated by create-next-app',
+};
+
+export default function RootLayout({
+  children,
+}: Readonly<{
+  children: React.ReactNode;
+}>) {
+  return (
+    <html lang="en">
+      <body>{children}</body>
+    </html>
+  );
+}
diff --git a/evals/build-cli-004-quickstart-nextjs/local/app/page.tsx b/evals/build-cli-004-quickstart-nextjs/local/app/page.tsx
new file mode 100644
index 00000000..e0d6da53
--- /dev/null
+++ b/evals/build-cli-004-quickstart-nextjs/local/app/page.tsx
@@ -0,0 +1,7 @@
+export default function Home() {
+  return (
+    <main>
+      <h1>Welcome to My App</h1>
+    </main>
+  );
+}
diff --git a/evals/build-cli-004-quickstart-nextjs/local/next-env.d.ts b/evals/build-cli-004-quickstart-nextjs/local/next-env.d.ts
new file mode 100644
index 00000000..1b3be084
--- /dev/null
+++ b/evals/build-cli-004-quickstart-nextjs/local/next-env.d.ts
@@ -0,0 +1,5 @@
+/// <reference types="next" />
+/// <reference types="next/image-types/global" />
+
+// NOTE: This file should not be edited
+// see https://nextjs.org/docs/app/api-reference/config/typescript for more information.
diff --git a/evals/build-cli-004-quickstart-nextjs/local/next.config.mjs b/evals/build-cli-004-quickstart-nextjs/local/next.config.mjs
new file mode 100644
index 00000000..4678774e
--- /dev/null
+++ b/evals/build-cli-004-quickstart-nextjs/local/next.config.mjs
@@ -0,0 +1,4 @@
+/** @type {import('next').NextConfig} */
+const nextConfig = {};
+
+export default nextConfig;
diff --git a/evals/build-cli-004-quickstart-nextjs/local/package.json b/evals/build-cli-004-quickstart-nextjs/local/package.json
new file mode 100644
index 00000000..82e7d907
--- /dev/null
+++ b/evals/build-cli-004-quickstart-nextjs/local/package.json
@@ -0,0 +1,21 @@
+{
+  "name": "my-app",
+  "version": "0.1.0",
+  "private": true,
+  "scripts": {
+    "dev": "next dev",
+    "build": "next build",
+    "start": "next start"
+  },
+  "dependencies": {
+    "next": "^15.5.0",
+    "react": "^19.2.0",
+    "react-dom": "^19.2.0"
+  },
+  "devDependencies": {
+    "@types/node": "^22.0.0",
+    "@types/react": "^19.2.0",
+    "@types/react-dom": "^19.2.0",
+    "typescript": "^5.6.0"
+  }
+}
diff --git a/evals/build-cli-004-quickstart-nextjs/local/tsconfig.json b/evals/build-cli-004-quickstart-nextjs/local/tsconfig.json
new file mode 100644
index 00000000..6420eed1
--- /dev/null
+++ b/evals/build-cli-004-quickstart-nextjs/local/tsconfig.json
@@ -0,0 +1,21 @@
+{
+  "compilerOptions": {
+    "target": "ES2017",
+    "lib": ["dom", "dom.iterable", "esnext"],
+    "allowJs": true,
+    "skipLibCheck": true,
+    "strict": true,
+    "noEmit": true,
+    "esModuleInterop": true,
+    "module": "esnext",
+    "moduleResolution": "bundler",
+    "resolveJsonModule": true,
+    "isolatedModules": true,
+    "jsx": "preserve",
+    "incremental": true,
+    "plugins": [{ "name": "next" }],
+    "paths": { "@/*": ["./*"] }
+  },
+  "include": ["next-env.d.ts", "**/*.ts", "**/*.tsx", ".next/types/**/*.ts"],
+  "exclude": ["node_modules"]
+}

From 623858aef387ace2496f868e0058b0439646b71d Mon Sep 17 00:00:00 2001
From: Miranda Limonczenko <miranda.limonczenko@supabase.io>
Date: Thu, 23 Jul 2026 12:18:38 -0700
Subject: [PATCH 2/7] feat: support per-eval skills/CLI overrides for
 self-install scenarios
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Adds `skills: []` and `skipCliInstall: true` frontmatter overrides plus
an experiment-level `skipEval` predicate, so a scenario can test an
agent installing its own Supabase CLI/skills regardless of which
experiment runs it. Applies both to the docs homepage quickstart eval,
which now exercises the CLI/plugin-install steps for real instead of
having them pre-satisfied by the sandbox.

Also teaches the eval-refresh workflow's discovery step (and the
underlying `pnpm eval -- list --eval`) to drop experiment/eval pairs
that skipEval would skip, so a no-skills experiment paired with a
skills:[] eval doesn't get scheduled — the pairing that broke
publish-results by expecting an artifact that skipEval prevents from
ever being produced.

Also fixes `npm install -g supabase` failing with EACCES in the
sandbox image: the node:22-slim image installs global packages as
root, so point NPM_CONFIG_PREFIX at a node-owned directory and add it
to PATH so unelevated global installs work.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
---
 .github/workflows/eval-refresh.yml            |  2 +-
 apps/framework/harness/run-eval.ts            | 29 +++++++--
 evals/build-cli-004-quickstart-nextjs/EVAL.ts | 44 +++++++------
 .../build-cli-004-quickstart-nextjs/PROMPT.md |  3 +-
 .../build-cli-004-quickstart-nextjs/README.md | 14 ++--
 experiments/claude-code-sonnet-5-no-skills.ts |  1 +
 packages/core/src/eval-metadata.ts            | 41 ++++++++++--
 packages/core/src/index.ts                    | 15 +++++
 packages/sandbox/Dockerfile                   |  8 +++
 packages/sandbox/src/agent-environment.ts     |  3 +
 packages/sandbox/src/docker-sandbox.ts        |  1 +
 packages/sandbox/src/local-stack-runtime.ts   | 18 ++---
 packages/sandbox/src/supabase.ts              | 30 ++++++---
 packages/sandbox/test/docker.test.ts          | 52 +++++++++++++++
 packages/sandbox/test/unit.test.ts            | 65 +++++++++++++++++++
 15 files changed, 268 insertions(+), 58 deletions(-)

diff --git a/.github/workflows/eval-refresh.yml b/.github/workflows/eval-refresh.yml
index 98b7e215..9b8162d0 100644
--- a/.github/workflows/eval-refresh.yml
+++ b/.github/workflows/eval-refresh.yml
@@ -226,7 +226,7 @@ jobs:
               if [ -n "$experiments_override" ]; then
                 experiments_json="$(jq -Rc 'split(",") | map(gsub("^\\s+|\\s+$"; "")) | map(select(length > 0))' <<< "$experiments_override")"
               else
-                experiments_json="$(pnpm --silent eval -- list --experiment-suite "$experiment_suite")"
+                experiments_json="$(pnpm --silent eval -- list --experiment-suite "$experiment_suite" --eval "$id")"
               fi
 
               while IFS= read -r experiment; do
diff --git a/apps/framework/harness/run-eval.ts b/apps/framework/harness/run-eval.ts
index 0f758e53..e0e13374 100644
--- a/apps/framework/harness/run-eval.ts
+++ b/apps/framework/harness/run-eval.ts
@@ -371,11 +371,14 @@ async function runOne(
   // tools mode its skills are advertised in the prompt and loaded via the
   // load_skill tool. Skill sources (name+dir) are shared by both paths.
   const agentRunsInSandbox = exp.agent.runsInSandbox ?? false;
-  const skillSources = resolveSkillSources(exp.skills);
+  // A per-eval `skills` override replaces the experiment's own list entirely,
+  // so a scenario testing self-installed skills gets an empty list regardless
+  // of which experiment runs it.
+  const skillSources = resolveSkillSources(ev.metadata.skills ?? exp.skills);
   const availableSkills = skillSources.map((skill) => skill.name);
   const toolsSkills =
     ev.mode === 'tools' && !agentRunsInSandbox
-      ? loadToolsSkills(exp.skills)
+      ? loadToolsSkills(ev.metadata.skills ?? exp.skills)
       : [];
   const scorer = (await import(pathToFileURL(ev.evalPath).href)).default as
     | ToolScorer
@@ -435,10 +438,8 @@ async function runOne(
                 invokeFunction: hostedBackend.invokeFunction,
               }
             : undefined,
-          // Skills are installed into the sandbox and discovered by the agent
-          // (the session folds the discovery listing into its promptAddendum),
-          // so no skill text is injected into the prompt here.
           skills: skillSources,
+          skipCliInstall: ev.metadata.skipCliInstall,
         })
       );
 
@@ -634,7 +635,7 @@ async function runConcurrent<T>(
 async function main() {
   if (rawArgs.filter((a) => a !== '--')[0] === 'list') {
     const experiments = await loadExperiments();
-    const filtered =
+    let filtered =
       EXPERIMENT_SUITE_FILTERS.length > 0
         ? experiments.filter(
             (e) =>
@@ -644,6 +645,18 @@ async function main() {
               )
           )
         : experiments;
+    if (EVAL_FILTERS.length > 0) {
+      // Drop experiments that would skipEval every requested eval, so callers
+      // building an experiment x eval matrix (e.g. the eval-refresh workflow)
+      // don't plan a pair that will produce no results — and no artifact —
+      // to upload.
+      const evals = discoverEvals().filter((ev) =>
+        EVAL_FILTERS.includes(ev.id)
+      );
+      filtered = filtered.filter(({ config }) =>
+        evals.some((ev) => !config.skipEval?.(ev))
+      );
+    }
     console.log(JSON.stringify(filtered.map((e) => e.name)));
     return;
   }
@@ -748,6 +761,10 @@ async function main() {
         );
         continue;
       }
+      if (config.skipEval?.(ev)) {
+        console.log(`SKIP ${name} x ${ev.id} (skipEval)`);
+        continue;
+      }
       if (DRY) {
         console.log(formatPlanLine(name, config, ev));
         continue;
diff --git a/evals/build-cli-004-quickstart-nextjs/EVAL.ts b/evals/build-cli-004-quickstart-nextjs/EVAL.ts
index 01b42fe6..3948c7b7 100644
--- a/evals/build-cli-004-quickstart-nextjs/EVAL.ts
+++ b/evals/build-cli-004-quickstart-nextjs/EVAL.ts
@@ -4,30 +4,35 @@ import {
   type CheckResult,
   type LocalStackEvalContext,
   type LocalStackScorer,
-} from "@supabase-evals/core";
-import { stripIndent } from "common-tags";
+} from '@supabase-evals/core';
+import { stripIndent } from 'common-tags';
 
 const PLUGIN_INSTALL_PATTERN =
   /npx\s+plugins\s+add\s+supabase-community\/supabase-plugin/i;
 
 const scorer: LocalStackScorer = async (ctx) => {
   try {
-    const requiredChecks: CheckResult[] = [
+    const checks: CheckResult[] = [
       await checkSupabaseInitialized(ctx),
       await checkCliFunctional(ctx),
       await checkNextStepsSuggested(ctx),
+      checkPluginInstallAttempted(ctx),
     ];
 
     return {
-      passed: requiredChecks.every((check) => check.passed),
-      checks: [...requiredChecks, checkPluginInstallAttempted(ctx)],
+      passed: checks.every((check) => check.passed),
+      checks,
     };
   } catch (error) {
     const msg = error instanceof Error ? error.message : String(error);
     return {
       passed: false,
       checks: [
-        { name: "scorer evaluated quickstart setup", passed: false, notes: msg },
+        {
+          name: 'scorer evaluated quickstart setup',
+          passed: false,
+          notes: msg,
+        },
       ],
     };
   }
@@ -36,28 +41,28 @@ const scorer: LocalStackScorer = async (ctx) => {
 export default scorer;
 
 async function checkSupabaseInitialized(
-  ctx: LocalStackEvalContext,
+  ctx: LocalStackEvalContext
 ): Promise<CheckResult> {
-  const exists = await ctx.fileExists("supabase/config.toml");
+  const exists = await ctx.fileExists('supabase/config.toml');
   return {
-    name: "project initialized (supabase/config.toml exists)",
+    name: 'project initialized (supabase/config.toml exists)',
     passed: exists,
   };
 }
 
 async function checkCliFunctional(
-  ctx: LocalStackEvalContext,
+  ctx: LocalStackEvalContext
 ): Promise<CheckResult> {
-  const result = await ctx.exec("supabase --version");
+  const result = await ctx.exec('supabase --version');
   return {
-    name: "supabase CLI is installed and runnable",
+    name: 'supabase CLI is installed and runnable',
     passed: result.ok,
     notes: result.ok ? undefined : result.stderr || result.stdout,
   };
 }
 
 async function checkNextStepsSuggested(
-  ctx: LocalStackEvalContext,
+  ctx: LocalStackEvalContext
 ): Promise<CheckResult> {
   const verdict = await judge({
     input: serializeTranscript(ctx.transcript, { includeToolCallInputs: true }),
@@ -81,21 +86,22 @@ async function checkNextStepsSuggested(
   });
 
   return {
-    name: "suggested relevant next steps for a Supabase + Next.js project",
+    name: 'suggested relevant next steps for a Supabase + Next.js project',
     passed: verdict.passed,
     judgeNotes: verdict.notes,
   };
 }
 
-// Informational only — not required for `passed`. Skills are configured at
-// the experiment level, not per-eval, so whether this step is even necessary
-// depends on which experiment ran it.
+// Checks the command ran, not that a plugin/skill actually landed — the
+// installer auto-detects agent binaries on PATH (e.g. `claude`) to pick an
+// install target, which this sandbox may not expose, making that outcome
+// environment-dependent rather than something the agent controls.
 function checkPluginInstallAttempted(ctx: LocalStackEvalContext): CheckResult {
   const attempted = ctx.toolCalls.some(
-    (call) => call.command && PLUGIN_INSTALL_PATTERN.test(call.command),
+    (call) => call.command && PLUGIN_INSTALL_PATTERN.test(call.command)
   );
   return {
-    name: "(informational) attempted `npx plugins add supabase-community/supabase-plugin`",
+    name: 'attempted `npx plugins add supabase-community/supabase-plugin`',
     passed: attempted,
   };
 }
diff --git a/evals/build-cli-004-quickstart-nextjs/PROMPT.md b/evals/build-cli-004-quickstart-nextjs/PROMPT.md
index 8eb9f126..a60f7324 100644
--- a/evals/build-cli-004-quickstart-nextjs/PROMPT.md
+++ b/evals/build-cli-004-quickstart-nextjs/PROMPT.md
@@ -7,8 +7,9 @@ product:
   - data-api
 topic:
   - sdk
-services: []
 projectRunning: false
+skills: []
+skipCliInstall: true
 motivation: the "Help me get set up with Supabase" prompt shown on the supabase.com/docs homepage
 ---
 
diff --git a/evals/build-cli-004-quickstart-nextjs/README.md b/evals/build-cli-004-quickstart-nextjs/README.md
index f0b3f856..e4ea702d 100644
--- a/evals/build-cli-004-quickstart-nextjs/README.md
+++ b/evals/build-cli-004-quickstart-nextjs/README.md
@@ -1,11 +1,11 @@
-## Adding another docs quickstart test
+# Adding an eval for another docs guide
 
-This guide explains how to add an eval for another AI setup prompt from the docs, such as a different framework's quickstart page.
+This guide explains how to add an eval for another AI setup prompt from the docs. The prompt can come from a quickstart page for a different framework, or from any other docs guide that gives an agent a setup prompt to follow.
 
-1. Create a sibling folder, for example `build-cli-005-quickstart-<framework>`.
-2. Copy the target prompt verbatim into `PROMPT.md`. Don't paraphrase it or add detail that the prompt doesn't already give.
-3. Seed `local/` with a minimal, unmodified starter for that framework. Don't include a `supabase/` directory, so the "is Supabase already initialized" step has something real to check.
-4. Reuse this eval's `EVAL.ts` checks. Change only the judge rubric's framework-specific wording.
-5. Validate the eval before you open a PR. Run `pnpm eval:dry -- --eval <new-eval-id> --experiment claude-code-sonnet-5` to check the setup, then run `pnpm eval -- --eval <new-eval-id> --experiment claude-code-sonnet-5` for a full pass.
+1. Create a sibling folder. Name it after the guide you're adding, for example `build-cli-005-quickstart-<framework>`.
+2. Copy the target prompt verbatim into `PROMPT.md`. Don't paraphrase it or add detail that the prompt doesn't already give. Keep `skills: []` and `skipCliInstall: true` in the frontmatter if the prompt has the agent install its own tooling.
+3. Seed `local/` with a minimal, unmodified starter that matches the guide's context. Don't include a `supabase/` directory, so the "is Supabase already initialized" step has something real to check.
+4. Reuse this eval's `EVAL.ts` checks as a starting point. Adjust the checks to match what the new guide's prompt requires, including the judge rubric's wording.
+5. Validate the eval before you open a PR. Run `pnpm eval:dry -- --eval <new-eval-id> --experiment claude-code-sonnet-5` to check the setup, and then run `pnpm eval -- --eval <new-eval-id> --experiment claude-code-sonnet-5` for a full pass.
 
 For the full eval-authoring workflow, including frontmatter fields, suite selection, and submitting for review, see `CONTRIBUTING.md`.
diff --git a/experiments/claude-code-sonnet-5-no-skills.ts b/experiments/claude-code-sonnet-5-no-skills.ts
index 9c310c11..e209193e 100644
--- a/experiments/claude-code-sonnet-5-no-skills.ts
+++ b/experiments/claude-code-sonnet-5-no-skills.ts
@@ -18,4 +18,5 @@ export default defineExperiment({
   }),
   localStack: localStackRuntime(),
   skills: [],
+  skipEval: (ev) => ev.metadata.skills?.length === 0,
 });
diff --git a/packages/core/src/eval-metadata.ts b/packages/core/src/eval-metadata.ts
index 59a645cf..af0ec266 100644
--- a/packages/core/src/eval-metadata.ts
+++ b/packages/core/src/eval-metadata.ts
@@ -129,6 +129,22 @@ export type EvalMetadata = {
    * evals only). Defaults to false.
    */
   hostedProject?: boolean;
+  /**
+   * Overrides the experiment's `skills` list for this eval only. An empty
+   * list (`skills: []`) tests an agent with no pre-installed Supabase
+   * skills, regardless of which experiment runs it — for a scenario where
+   * the prompt asks the agent to install skills itself. Omit the key
+   * entirely to use the experiment's own skill list.
+   */
+  skills?: string[];
+  /**
+   * Skips installing the real Supabase CLI into the sandbox before the agent
+   * starts (sandbox evals only). Defaults to false. Set true only for
+   * scenarios whose prompt has the agent install the CLI itself — the
+   * harness itself never invokes `supabase` when this is set, so pair it
+   * with `projectRunning: false`.
+   */
+  skipCliInstall?: boolean;
 };
 
 export type ParsedEvalMarkdown = {
@@ -149,6 +165,8 @@ export const evalMetadataSchema = z.object({
   // non-empty string as true.
   projectRunning: z.union([z.boolean(), z.stringbool()]).optional(),
   hostedProject: z.union([z.boolean(), z.stringbool()]).optional(),
+  skills: z.array(z.string().min(1)).optional(),
+  skipCliInstall: z.union([z.boolean(), z.stringbool()]).optional(),
 });
 
 // Collapse a YAML scalar into a comparable token: trim, lowercase, and fold
@@ -180,12 +198,14 @@ const toTokenList = (value: unknown): unknown => {
     .filter((item) => typeof item === 'string' && item.length > 0);
 };
 
-// `services` are real Supabase CLI service identifiers (e.g. `postgres-meta`,
-// `storage-api`, `edge-runtime`), matched verbatim against ALL_SUPABASE_SERVICES
-// in @supabase-evals/sandbox. They must NOT go through normalizeToken: folding
-// hyphens to underscores would turn `postgres-meta` into `postgres_meta` and
-// fail that match. Only trim + lowercase; preserve hyphens. Blanks are dropped.
-const toServiceList = (value: unknown): unknown =>
+// `services` and `skills` are literal identifiers (Supabase CLI service names
+// like `postgres-meta`, or skill directory names like
+// `supabase-postgres-best-practices`), not enum tokens. They must NOT go
+// through normalizeToken: folding hyphens to underscores would turn
+// `postgres-meta` into `postgres_meta` and fail the verbatim match against
+// ALL_SUPABASE_SERVICES (services) or the skills/ directory (skills). Only
+// trim + lowercase; preserve hyphens. Blanks are dropped.
+const toIdentifierList = (value: unknown): unknown =>
   (Array.isArray(value) ? value : [value])
     .map((item) =>
       typeof item === 'string' ||
@@ -215,10 +235,17 @@ export const evalFrontmatterSchema = z.preprocess((raw) => {
     // `services: []` means database only; an omitted key means the full stack.
     // Only an explicit list is honored — any other value is treated as absent.
     services: Array.isArray(data.services)
-      ? toServiceList(data.services)
+      ? toIdentifierList(data.services)
       : undefined,
     projectRunning: data.projectRunning,
     hostedProject: data.hostedProject,
+    // `skills: []` means no pre-installed skills for this eval; an omitted
+    // key means "use the experiment's own skill list" (see resolveSkillSources
+    // in apps/framework/harness/run-eval.ts).
+    skills: Array.isArray(data.skills)
+      ? toIdentifierList(data.skills)
+      : undefined,
+    skipCliInstall: data.skipCliInstall,
   };
 }, evalMetadataSchema);
 
diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts
index caa480d2..cf25d701 100644
--- a/packages/core/src/index.ts
+++ b/packages/core/src/index.ts
@@ -36,6 +36,7 @@ import {
 import type {
   AgentHarnessId,
   CheckResult,
+  EvalMetadata,
   EvalSuite,
   ExperimentDisplayMetadata,
   ExperimentSuite,
@@ -438,6 +439,13 @@ export type LocalStackSessionArgs = {
    * instead, so they ignore this.
    */
   skills?: readonly SkillSource[];
+  /**
+   * Skip installing the real Supabase CLI into the sandbox (from
+   * `skipCliInstall:` frontmatter), for scenarios whose prompt has the agent
+   * install the CLI itself. Also skips the CLI shim that enforces
+   * `includeServices`, since there's no real binary yet to resolve and wrap.
+   */
+  skipCliInstall?: boolean;
 };
 
 /** A mocked hosted project (platform-lite) the sandbox CLI is linked to. */
@@ -514,6 +522,13 @@ export type ExperimentConfig = {
   /** Local-stack environment (e.g. localStackRuntime() from @supabase-evals/sandbox). */
   localStack?: LocalStackRuntime;
   skills: string[];
+  /**
+   * Skip running specific evals against this experiment, e.g. a `*-no-skills`
+   * experiment skipping evals whose per-eval `skills` override is already
+   * `[]` — running those would just duplicate what this experiment's own
+   * empty skill list already covers.
+   */
+  skipEval?: (ev: { id: string; metadata: EvalMetadata }) => boolean;
 };
 
 export function getExperimentDisplayMetadata(
diff --git a/packages/sandbox/Dockerfile b/packages/sandbox/Dockerfile
index 3d9b771b..4885ae3f 100644
--- a/packages/sandbox/Dockerfile
+++ b/packages/sandbox/Dockerfile
@@ -8,6 +8,14 @@ RUN apt-get update -qq && apt-get install -y -qq --no-install-recommends \
     ca-certificates curl git docker.io postgresql-client \
   && rm -rf /var/lib/apt/lists/*
 
+# Global npm installs (both the baked-in skills CLI below and the agent's own
+# `npm install -g supabase` at setup time) land in a node-owned prefix instead
+# of /usr/local, so they work without root/sudo — matching a typical user's
+# machine, where global npm installs "just work" unelevated.
+USER node
+RUN mkdir -p /home/node/.npm-global
+ENV NPM_CONFIG_PREFIX=/home/node/.npm-global
+
 # Vercel's agent-skills CLI, used to install agent skills from local sources
 # into each session's workspace (both modes). Baked in (pinned) so installs are
 # offline and instant rather than an npx download per session.
diff --git a/packages/sandbox/src/agent-environment.ts b/packages/sandbox/src/agent-environment.ts
index 4d046395..d17bc08d 100644
--- a/packages/sandbox/src/agent-environment.ts
+++ b/packages/sandbox/src/agent-environment.ts
@@ -29,6 +29,8 @@ export interface LocalStackSetup {
   projectRunning?: boolean;
   /** Link the CLI to a mocked hosted project (platform-lite) at this host port. */
   hosted?: { port: number; pgPort?: number; ref: string; accessToken: string };
+  /** Skip installing the real CLI, for scenarios where the agent installs it itself. */
+  skipCliInstall?: boolean;
 }
 
 export interface AgentEnvironmentOptions {
@@ -78,6 +80,7 @@ export async function createAgentEnvironment(
         localDir: options.localDir,
         projectRunning: options.localStack.projectRunning,
         hosted: options.localStack.hosted,
+        skipCliInstall: options.localStack.skipCliInstall,
       });
     } else if (options.localDir) {
       await sandbox.copyToContainer(options.localDir, sandbox.workdir);
diff --git a/packages/sandbox/src/docker-sandbox.ts b/packages/sandbox/src/docker-sandbox.ts
index 42f98052..dbfd0983 100644
--- a/packages/sandbox/src/docker-sandbox.ts
+++ b/packages/sandbox/src/docker-sandbox.ts
@@ -58,6 +58,7 @@ export const SANDBOX_CONTAINER_LABEL = 'supabase-evals-sandbox';
 const CLIENT_TIMEOUT_HEADROOM_MS = 20_000;
 
 const SANDBOX_PATH = [
+  '/home/node/.npm-global/bin',
   '/usr/local/sbin',
   '/usr/local/bin',
   '/usr/sbin',
diff --git a/packages/sandbox/src/local-stack-runtime.ts b/packages/sandbox/src/local-stack-runtime.ts
index 5fe61875..f9564bdf 100644
--- a/packages/sandbox/src/local-stack-runtime.ts
+++ b/packages/sandbox/src/local-stack-runtime.ts
@@ -79,10 +79,8 @@ export function localStackRuntime(
       projectRunning,
       hosted,
       skills,
+      skipCliInstall,
     }) {
-      // Local-stack mode = the shared agent environment with the Supabase local
-      // stack started. Everything else (image, tooling, skills) is identical to
-      // tools mode; only the `localStack` component differs.
       const env = await createAgentEnvironment({
         cliVersion: cliVersion ?? options.cliVersion,
         localDir,
@@ -98,17 +96,21 @@ export function localStackRuntime(
                 accessToken: hosted.accessToken,
               }
             : undefined,
+          skipCliInstall,
         },
       });
       const sandbox = env.sandbox;
 
       const mcpServers = await resolveMcpServers(options, hosted);
 
-      const baseAddendum =
-        'The Supabase CLI (`supabase`), docker, psql, git, and curl are installed in the workspace. ' +
-        'Use the bash tool to run commands (the working directory is always the workspace root) ' +
-        'and the files tools to inspect and modify files. ' +
-        'Services started with `supabase start` are reachable on their default 127.0.0.1 ports.';
+      const baseAddendum = skipCliInstall
+        ? 'docker, psql, git, and curl are installed in the workspace, but the Supabase CLI is not — install it yourself. ' +
+          'Use the bash tool to run commands (the working directory is always the workspace root) ' +
+          'and the files tools to inspect and modify files.'
+        : 'The Supabase CLI (`supabase`), docker, psql, git, and curl are installed in the workspace. ' +
+          'Use the bash tool to run commands (the working directory is always the workspace root) ' +
+          'and the files tools to inspect and modify files. ' +
+          'Services started with `supabase start` are reachable on their default 127.0.0.1 ports.';
 
       return {
         tools: buildLocalStackTools(sandbox),
diff --git a/packages/sandbox/src/supabase.ts b/packages/sandbox/src/supabase.ts
index 53c89c68..0f2c3857 100644
--- a/packages/sandbox/src/supabase.ts
+++ b/packages/sandbox/src/supabase.ts
@@ -155,6 +155,13 @@ export async function installSupabaseCli(
 export interface SetupSupabaseSandboxOptions {
   /** Supabase CLI version to install into the sandbox (pinned default). */
   cliVersion?: string;
+  /**
+   * Skip installing the real Supabase CLI, for scenarios whose prompt has the
+   * agent install it itself. Also skips the CLI shim (there's no real binary
+   * to resolve and wrap yet), so service exclusion isn't enforced — pair with
+   * an eval that omits `services` (full stack) or accepts that limitation.
+   */
+  skipCliInstall?: boolean;
   /**
    * Local-stack services this session needs; every other service is excluded
    * from `supabase start` to keep boots fast. Omitted means the full stack.
@@ -221,8 +228,10 @@ export async function setupSupabaseSandbox(
 ): Promise<void> {
   // The Supabase CLI is a local-stack component: install it here (not in the
   // base image) so tools-mode sandboxes don't have it. Everything below needs
-  // the CLI, so it goes first.
-  await installSupabaseCli(sandbox, options.cliVersion);
+  // the CLI, so it goes first — unless the scenario has the agent install it.
+  if (!options.skipCliInstall) {
+    await installSupabaseCli(sandbox, options.cliVersion);
+  }
 
   // Let the non-root sandbox user talk to the mounted Docker socket by
   // joining the socket's group. chmod would also work but mutates the host
@@ -276,13 +285,16 @@ export async function setupSupabaseSandbox(
   // linked DB commands at the hosted wire endpoint, and appends `-x` to any
   // `supabase start` the agent runs itself (so service exclusions hold even when
   // the stack is already up and the agent restarts it). No-op when there's
-  // nothing to exclude and no hosted wire endpoint.
-  await installSupabaseCliWrapper(
-    sandbox,
-    options.includeServices,
-    poolerUrlPath,
-    options.hosted !== undefined
-  );
+  // nothing to exclude and no hosted wire endpoint. Skipped when there's no CLI
+  // installed yet to resolve and wrap (skipCliInstall).
+  if (!options.skipCliInstall) {
+    await installSupabaseCliWrapper(
+      sandbox,
+      options.includeServices,
+      poolerUrlPath,
+      options.hosted !== undefined
+    );
+  }
 }
 
 /**
diff --git a/packages/sandbox/test/docker.test.ts b/packages/sandbox/test/docker.test.ts
index 7d295c3d..5107b2a3 100644
--- a/packages/sandbox/test/docker.test.ts
+++ b/packages/sandbox/test/docker.test.ts
@@ -112,6 +112,58 @@ describe.runIf(process.env.SANDBOX_DOCKER_TESTS)(
       }
     );
 
+    it(
+      "skipCliInstall leaves the CLI absent, and the wrapper doesn't crash setup",
+      { timeout: TEST_TIMEOUT_MS },
+      async () => {
+        const image = await ensureSupabaseSandboxImage();
+        const sandbox = await DockerSandbox.create({ image, network: 'host' });
+        try {
+          await setupSupabaseSandbox(sandbox, {
+            projectRunning: false,
+            skipCliInstall: true,
+          });
+
+          const version = await sandbox.runShell('supabase --version');
+          expect(version.ok).toBe(false);
+
+          // The agent installs it itself; the harness never touched the CLI.
+          const install = await sandbox.runShellAsRoot(
+            `ARCH="$(dpkg --print-architecture)" && ` +
+              `curl -fsSL "https://github.com/supabase/cli/releases/download/v${SUPABASE_CLI_VERSION}/supabase_${SUPABASE_CLI_VERSION}_linux_$ARCH.deb" -o /tmp/supabase.deb && ` +
+              `dpkg -i /tmp/supabase.deb && rm /tmp/supabase.deb`
+          );
+          expect(install.ok).toBe(true);
+
+          const versionAfter = await sandbox.runShell('supabase --version');
+          expect(versionAfter.ok).toBe(true);
+          expect(versionAfter.stdout.trim()).toBe(SUPABASE_CLI_VERSION);
+        } finally {
+          await sandbox.stop();
+        }
+      }
+    );
+
+    it(
+      'npm install -g lands on PATH without sudo/workarounds',
+      { timeout: TEST_TIMEOUT_MS },
+      async () => {
+        const image = await ensureSupabaseSandboxImage();
+        const sandbox = await DockerSandbox.create({ image, network: 'host' });
+        try {
+          const install = await sandbox.runShell(
+            'npm install -g supabase 2>&1'
+          );
+          expect(install.ok).toBe(true);
+
+          const version = await sandbox.runShell('supabase --version');
+          expect(version.ok).toBe(true);
+        } finally {
+          await sandbox.stop();
+        }
+      }
+    );
+
     it(
       'installs a local skill into the workspace with the skills CLI',
       { timeout: TEST_TIMEOUT_MS },
diff --git a/packages/sandbox/test/unit.test.ts b/packages/sandbox/test/unit.test.ts
index 2713975c..bfbef903 100644
--- a/packages/sandbox/test/unit.test.ts
+++ b/packages/sandbox/test/unit.test.ts
@@ -402,6 +402,71 @@ describe('cliVersion frontmatter', () => {
   });
 });
 
+describe('skills frontmatter', () => {
+  const buildMarkdown = (extra: string) =>
+    [
+      '---',
+      'stage: build',
+      'suite: regression',
+      'interface: cli',
+      'product: [database]',
+      'topic: [sdk]',
+      extra,
+      '---',
+      'body',
+    ].join('\n');
+
+  it('preserves hyphenated skill directory names', () => {
+    const { metadata } = parseEvalMarkdown(
+      buildMarkdown('skills: [supabase, supabase-postgres-best-practices]')
+    );
+    expect(metadata.skills).toEqual([
+      'supabase',
+      'supabase-postgres-best-practices',
+    ]);
+  });
+
+  it('parses an empty override distinctly from an omitted key', () => {
+    const overridden = parseEvalMarkdown(buildMarkdown('skills: []'));
+    expect(overridden.metadata.skills).toEqual([]);
+
+    const omitted = parseEvalMarkdown(buildMarkdown(''));
+    expect(omitted.metadata.skills).toBeUndefined();
+  });
+});
+
+describe('skipCliInstall frontmatter', () => {
+  const buildMarkdown = (extra: string) =>
+    [
+      '---',
+      'stage: build',
+      'suite: regression',
+      'interface: cli',
+      'product: [database]',
+      'topic: [sdk]',
+      extra,
+      '---',
+      'body',
+    ].join('\n');
+
+  it('accepts a real boolean and a quoted string form', () => {
+    expect(
+      parseEvalMarkdown(buildMarkdown('skipCliInstall: true')).metadata
+        .skipCliInstall
+    ).toBe(true);
+    expect(
+      parseEvalMarkdown(buildMarkdown('skipCliInstall: "true"')).metadata
+        .skipCliInstall
+    ).toBe(true);
+  });
+
+  it('defaults to undefined when omitted', () => {
+    expect(
+      parseEvalMarkdown(buildMarkdown('')).metadata.skipCliInstall
+    ).toBeUndefined();
+  });
+});
+
 describe('resolveSandboxPath', () => {
   it('accepts and normalizes relative paths', () => {
     expect(resolveSandboxPath('a/b.txt')).toBe('a/b.txt');

From af773c7f7d779e0c1f38d1f59fd61067cff9de01 Mon Sep 17 00:00:00 2001
From: "github-actions[bot]"
 <41898282+github-actions[bot]@users.noreply.github.com>
Date: Wed, 22 Jul 2026 23:22:28 +0000
Subject: [PATCH 3/7] chore: refresh eval results

Also drops a stale pre-skipEval result row for
claude-code-sonnet-5-no-skills/build-cli-004-quickstart-nextjs: merge
mode in export-results.ts only overwrites keys present in new results
and never prunes keys that stop being produced, so the old
"passed: true" row would otherwise persist through every merge-mode
refresh now that skipEval excludes that experiment from this eval.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
---
 .../web/src/data/regression-eval-results.json | 53 +++++++++++++++++++
 1 file changed, 53 insertions(+)

diff --git a/apps/web/src/data/regression-eval-results.json b/apps/web/src/data/regression-eval-results.json
index 5ac81bd2..e7e858a4 100644
--- a/apps/web/src/data/regression-eval-results.json
+++ b/apps/web/src/data/regression-eval-results.json
@@ -1,4 +1,57 @@
 [
+  {
+    "experiment": "claude-code-sonnet-5",
+    "experimentSuite": "regression",
+    "experimentDisplay": {
+      "agent": "claude-code",
+      "modelProvider": "anthropic",
+      "modelId": "claude-sonnet-5",
+      "reasoningEffort": "high"
+    },
+    "eval": "build-cli-004-quickstart-nextjs",
+    "stage": "build",
+    "product": [
+      "database",
+      "data-api"
+    ],
+    "topic": [
+      "sdk"
+    ],
+    "suite": "regression",
+    "interface": "cli",
+    "passed": false,
+    "checks": [
+      {
+        "name": "project initialized (supabase/config.toml exists)",
+        "passed": true
+      },
+      {
+        "name": "supabase CLI is installed and runnable",
+        "passed": false,
+        "notes": "bash: line 1: supabase: command not found\n"
+      },
+      {
+        "name": "suggested relevant next steps for a Supabase + Next.js project",
+        "passed": true,
+        "judgeNotes": "The response gives several concrete, relevant Supabase/Next.js next steps: running `supabase start`, linking a hosted project, installing `@supabase/supabase-js`/`@supabase/ssr`, creating a Next.js client with URL and anon key, and creating migrations."
+      },
+      {
+        "name": "attempted `npx plugins add supabase-community/supabase-plugin`",
+        "passed": true
+      }
+    ],
+    "skills": {
+      "available": [],
+      "loaded": []
+    },
+    "docs": {
+      "calls": []
+    },
+    "prompt": "Help me get set up with Supabase. Do the following: 1. Install the Supabase CLI globally with `npm install -g supabase`. 2. Install the Supabase Plugin with `npx plugins add supabase-community/supabase-plugin`. 3. Review my project and determine whether Supabase is already initialized. If it is not initialized, run `supabase init`. 4. Suggest the most relevant next steps.",
+    "promptSourcePath": "evals/build-cli-004-quickstart-nextjs/PROMPT.md",
+    "attempts": 2,
+    "sourcePath": "claude-code-sonnet-5/build-cli-004-quickstart-nextjs.json"
+  },
   {
     "experiment": "claude-code-sonnet-5",
     "experimentSuite": "regression",

From 870c4529adf0afd2a15b1b1a31f66fc393ce3d58 Mon Sep 17 00:00:00 2001
From: "github-actions[bot]"
 <41898282+github-actions[bot]@users.noreply.github.com>
Date: Fri, 24 Jul 2026 19:32:49 +0000
Subject: [PATCH 4/7] chore: refresh eval results

---
 apps/web/src/data/regression-eval-results.json | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/apps/web/src/data/regression-eval-results.json b/apps/web/src/data/regression-eval-results.json
index e7e858a4..f8536671 100644
--- a/apps/web/src/data/regression-eval-results.json
+++ b/apps/web/src/data/regression-eval-results.json
@@ -19,7 +19,7 @@
     ],
     "suite": "regression",
     "interface": "cli",
-    "passed": false,
+    "passed": true,
     "checks": [
       {
         "name": "project initialized (supabase/config.toml exists)",
@@ -27,13 +27,12 @@
       },
       {
         "name": "supabase CLI is installed and runnable",
-        "passed": false,
-        "notes": "bash: line 1: supabase: command not found\n"
+        "passed": true
       },
       {
         "name": "suggested relevant next steps for a Supabase + Next.js project",
         "passed": true,
-        "judgeNotes": "The response gives several concrete, relevant Supabase/Next.js next steps: running `supabase start`, linking a hosted project, installing `@supabase/supabase-js`/`@supabase/ssr`, creating a Next.js client with URL and anon key, and creating migrations."
+        "judgeNotes": "The final response provides several concrete Supabase + Next.js next steps, including `supabase start`, linking a hosted project, installing `@supabase/supabase-js`/`@supabase/ssr`, adding Supabase env vars, and creating a first migration."
       },
       {
         "name": "attempted `npx plugins add supabase-community/supabase-plugin`",
@@ -49,7 +48,7 @@
     },
     "prompt": "Help me get set up with Supabase. Do the following: 1. Install the Supabase CLI globally with `npm install -g supabase`. 2. Install the Supabase Plugin with `npx plugins add supabase-community/supabase-plugin`. 3. Review my project and determine whether Supabase is already initialized. If it is not initialized, run `supabase init`. 4. Suggest the most relevant next steps.",
     "promptSourcePath": "evals/build-cli-004-quickstart-nextjs/PROMPT.md",
-    "attempts": 2,
+    "attempts": 1,
     "sourcePath": "claude-code-sonnet-5/build-cli-004-quickstart-nextjs.json"
   },
   {

From 65a349ac933b531a2e5853892c0d74d91b8c12bb Mon Sep 17 00:00:00 2001
From: "github-actions[bot]"
 <41898282+github-actions[bot]@users.noreply.github.com>
Date: Fri, 24 Jul 2026 19:42:34 +0000
Subject: [PATCH 5/7] chore: refresh eval results

---
 apps/web/src/data/regression-eval-results.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/apps/web/src/data/regression-eval-results.json b/apps/web/src/data/regression-eval-results.json
index f8536671..ccd35919 100644
--- a/apps/web/src/data/regression-eval-results.json
+++ b/apps/web/src/data/regression-eval-results.json
@@ -32,7 +32,7 @@
       {
         "name": "suggested relevant next steps for a Supabase + Next.js project",
         "passed": true,
-        "judgeNotes": "The final response provides several concrete Supabase + Next.js next steps, including `supabase start`, linking a hosted project, installing `@supabase/supabase-js`/`@supabase/ssr`, adding Supabase env vars, and creating a first migration."
+        "judgeNotes": "The response includes multiple concrete Supabase/Next.js next steps: `supabase start`, installing `@supabase/supabase-js`/`@supabase/ssr`, setting `.env.local` with Supabase URL and anon key, linking a hosted project, and creating a first migration."
       },
       {
         "name": "attempted `npx plugins add supabase-community/supabase-plugin`",

From f8ee821d86e15f50a4e5184179403b07eb327091 Mon Sep 17 00:00:00 2001
From: Miranda Limonczenko <miranda.limonczenko@supabase.io>
Date: Tue, 28 Jul 2026 12:29:37 -0700
Subject: [PATCH 6/7] fix: tolerate npx flags in plugin-install pattern

mattrossman reported a local run failing this check because the agent
ran `npx --yes plugins add supabase-community/supabase-plugin` and the
regex didn't allow flags between `npx` and `plugins`.
---
 evals/build-cli-004-quickstart-nextjs/EVAL.ts | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/evals/build-cli-004-quickstart-nextjs/EVAL.ts b/evals/build-cli-004-quickstart-nextjs/EVAL.ts
index 3948c7b7..829a1b0f 100644
--- a/evals/build-cli-004-quickstart-nextjs/EVAL.ts
+++ b/evals/build-cli-004-quickstart-nextjs/EVAL.ts
@@ -7,8 +7,10 @@ import {
 } from '@supabase-evals/core';
 import { stripIndent } from 'common-tags';
 
+// Tolerates npx flags between `npx` and `plugins` (e.g. `npx --yes plugins add ...`),
+// since the agent is free to pass its own npx flags and still be attempting the install.
 const PLUGIN_INSTALL_PATTERN =
-  /npx\s+plugins\s+add\s+supabase-community\/supabase-plugin/i;
+  /npx\s+(?:--?\S+\s+)*plugins\s+add\s+supabase-community\/supabase-plugin/i;
 
 const scorer: LocalStackScorer = async (ctx) => {
   try {

From dd7695da554c97f11bae37376ca7e33bfaee233b Mon Sep 17 00:00:00 2001
From: "github-actions[bot]"
 <41898282+github-actions[bot]@users.noreply.github.com>
Date: Tue, 28 Jul 2026 19:35:52 +0000
Subject: [PATCH 7/7] chore: refresh eval results

---
 apps/web/src/data/regression-eval-results.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/apps/web/src/data/regression-eval-results.json b/apps/web/src/data/regression-eval-results.json
index ccd35919..592d9648 100644
--- a/apps/web/src/data/regression-eval-results.json
+++ b/apps/web/src/data/regression-eval-results.json
@@ -32,7 +32,7 @@
       {
         "name": "suggested relevant next steps for a Supabase + Next.js project",
         "passed": true,
-        "judgeNotes": "The response includes multiple concrete Supabase/Next.js next steps: `supabase start`, installing `@supabase/supabase-js`/`@supabase/ssr`, setting `.env.local` with Supabase URL and anon key, linking a hosted project, and creating a first migration."
+        "judgeNotes": "The response includes several concrete Supabase/Next.js next steps: running `supabase start`, installing `@supabase/supabase-js`/`@supabase/ssr`, setting `.env.local` variables for URL and anon key, and creating a first migration."
       },
       {
         "name": "attempted `npx plugins add supabase-community/supabase-plugin`",