feat: add docs homepage quickstart eval

.github/workflows/eval-refresh.yml

@@ -226,7 +226,7 @@ jobs:
               if [ -n "$experiments_override" ]; then
                 experiments_json="$(jq -Rc 'split(",") | map(gsub("^\\s+|\\s+$"; "")) | map(select(length > 0))' <<< "$experiments_override")"
               else
-                experiments_json="$(pnpm --silent eval -- list --experiment-suite "$experiment_suite")"
+                experiments_json="$(pnpm --silent eval -- list --experiment-suite "$experiment_suite" --eval "$id")"
               fi
 
               while IFS= read -r experiment; do

apps/framework/harness/run-eval.ts

@@ -371,11 +371,14 @@ async function runOne(
   // tools mode its skills are advertised in the prompt and loaded via the
   // load_skill tool. Skill sources (name+dir) are shared by both paths.
   const agentRunsInSandbox = exp.agent.runsInSandbox ?? false;
-  const skillSources = resolveSkillSources(exp.skills);
+  // A per-eval `skills` override replaces the experiment's own list entirely,
+  // so a scenario testing self-installed skills gets an empty list regardless
+  // of which experiment runs it.
+  const skillSources = resolveSkillSources(ev.metadata.skills ?? exp.skills);
   const availableSkills = skillSources.map((skill) => skill.name);
   const toolsSkills =
     ev.mode === 'tools' && !agentRunsInSandbox
-      ? loadToolsSkills(exp.skills)
+      ? loadToolsSkills(ev.metadata.skills ?? exp.skills)
       : [];
   const scorer = (await import(pathToFileURL(ev.evalPath).href)).default as
     | ToolScorer
@@ -435,10 +438,8 @@ async function runOne(
                 invokeFunction: hostedBackend.invokeFunction,
               }
             : undefined,
-          // Skills are installed into the sandbox and discovered by the agent
-          // (the session folds the discovery listing into its promptAddendum),
-          // so no skill text is injected into the prompt here.
           skills: skillSources,
+          skipCliInstall: ev.metadata.skipCliInstall,
         })
       );
 
@@ -634,7 +635,7 @@ async function runConcurrent<T>(
 async function main() {
   if (rawArgs.filter((a) => a !== '--')[0] === 'list') {
     const experiments = await loadExperiments();
-    const filtered =
+    let filtered =
       EXPERIMENT_SUITE_FILTERS.length > 0
         ? experiments.filter(
             (e) =>
@@ -644,6 +645,18 @@ async function main() {
               )
           )
         : experiments;
+    if (EVAL_FILTERS.length > 0) {
+      // Drop experiments that would skipEval every requested eval, so callers
+      // building an experiment x eval matrix (e.g. the eval-refresh workflow)
+      // don't plan a pair that will produce no results — and no artifact —
+      // to upload.
+      const evals = discoverEvals().filter((ev) =>
+        EVAL_FILTERS.includes(ev.id)
+      );
+      filtered = filtered.filter(({ config }) =>
+        evals.some((ev) => !config.skipEval?.(ev))
+      );
+    }
     console.log(JSON.stringify(filtered.map((e) => e.name)));
     return;
   }
@@ -748,6 +761,10 @@ async function main() {
         );
         continue;
       }
+      if (config.skipEval?.(ev)) {
+        console.log(`SKIP ${name} x ${ev.id} (skipEval)`);
+        continue;
+      }
       if (DRY) {
         console.log(formatPlanLine(name, config, ev));
         continue;

apps/web/src/data/regression-eval-results.json

@@ -1,4 +1,56 @@
 [
+  {
+    "experiment": "claude-code-sonnet-5",
+    "experimentSuite": "regression",
+    "experimentDisplay": {
+      "agent": "claude-code",
+      "modelProvider": "anthropic",
+      "modelId": "claude-sonnet-5",
+      "reasoningEffort": "high"
+    },
+    "eval": "build-cli-004-quickstart-nextjs",
+    "stage": "build",
+    "product": [
+      "database",
+      "data-api"
+    ],
+    "topic": [
+      "sdk"
+    ],
+    "suite": "regression",
+    "interface": "cli",
+    "passed": true,
+    "checks": [
+      {
+        "name": "project initialized (supabase/config.toml exists)",
+        "passed": true
+      },
+      {
+        "name": "supabase CLI is installed and runnable",
+        "passed": true
+      },
+      {
+        "name": "suggested relevant next steps for a Supabase + Next.js project",
+        "passed": true,
+        "judgeNotes": "The response includes several concrete Supabase/Next.js next steps: running `supabase start`, installing `@supabase/supabase-js`/`@supabase/ssr`, setting `.env.local` variables for URL and anon key, and creating a first migration."
+      },
+      {
+        "name": "attempted `npx plugins add supabase-community/supabase-plugin`",
+        "passed": true
+      }
+    ],
+    "skills": {
+      "available": [],
+      "loaded": []
+    },
+    "docs": {
+      "calls": []
+    },
+    "prompt": "Help me get set up with Supabase. Do the following: 1. Install the Supabase CLI globally with `npm install -g supabase`. 2. Install the Supabase Plugin with `npx plugins add supabase-community/supabase-plugin`. 3. Review my project and determine whether Supabase is already initialized. If it is not initialized, run `supabase init`. 4. Suggest the most relevant next steps.",
+    "promptSourcePath": "evals/build-cli-004-quickstart-nextjs/PROMPT.md",
+    "attempts": 1,
+    "sourcePath": "claude-code-sonnet-5/build-cli-004-quickstart-nextjs.json"
+  },
   {
     "experiment": "claude-code-sonnet-5",
     "experimentSuite": "regression",

evals/build-cli-004-quickstart-nextjs/EVAL.ts

@@ -0,0 +1,109 @@
+import {
+  judge,
+  serializeTranscript,
+  type CheckResult,
+  type LocalStackEvalContext,
+  type LocalStackScorer,
+} from '@supabase-evals/core';
+import { stripIndent } from 'common-tags';
+
+// Tolerates npx flags between `npx` and `plugins` (e.g. `npx --yes plugins add ...`),
+// since the agent is free to pass its own npx flags and still be attempting the install.
+const PLUGIN_INSTALL_PATTERN =
+  /npx\s+(?:--?\S+\s+)*plugins\s+add\s+supabase-community\/supabase-plugin/i;
+
+const scorer: LocalStackScorer = async (ctx) => {
+  try {
+    const checks: CheckResult[] = [
+      await checkSupabaseInitialized(ctx),
+      await checkCliFunctional(ctx),
+      await checkNextStepsSuggested(ctx),
+      checkPluginInstallAttempted(ctx),
+    ];
+
+    return {
+      passed: checks.every((check) => check.passed),
+      checks,
+    };
+  } catch (error) {
+    const msg = error instanceof Error ? error.message : String(error);
+    return {
+      passed: false,
+      checks: [
+        {
+          name: 'scorer evaluated quickstart setup',
+          passed: false,
+          notes: msg,
+        },
+      ],
+    };
+  }
+};
+
+export default scorer;
+
+async function checkSupabaseInitialized(
+  ctx: LocalStackEvalContext
+): Promise<CheckResult> {
+  const exists = await ctx.fileExists('supabase/config.toml');
+  return {
+    name: 'project initialized (supabase/config.toml exists)',
+    passed: exists,
+  };
+}
+
+async function checkCliFunctional(
+  ctx: LocalStackEvalContext
+): Promise<CheckResult> {
+  const result = await ctx.exec('supabase --version');
+  return {
+    name: 'supabase CLI is installed and runnable',
+    passed: result.ok,
+    notes: result.ok ? undefined : result.stderr || result.stdout,
+  };
+}
+
+async function checkNextStepsSuggested(
+  ctx: LocalStackEvalContext
+): Promise<CheckResult> {
+  const verdict = await judge({
+    input: serializeTranscript(ctx.transcript, { includeToolCallInputs: true }),
+    rubric: stripIndent`
+      Pass if the assistant's final response suggests next steps for using
+      Supabase in this Next.js project that are relevant and specific to what
+      was just set up.
+
+      A passing answer should reference at least one concrete, relevant next
+      step, such as:
+      - starting the local stack (supabase start) or creating a first migration
+      - installing/using @supabase/supabase-js (or an SSR helper) to connect
+        the Next.js app
+      - setting environment variables for the local API URL and anon key
+      - exploring Studio or writing a first table/schema
+
+      Fail if the assistant stops after installation/init with no concrete
+      next steps, or the suggestions are generic enough to apply to any
+      project regardless of Supabase or Next.js.
+    `,
+  });
+
+  return {
+    name: 'suggested relevant next steps for a Supabase + Next.js project',
+    passed: verdict.passed,
+    judgeNotes: verdict.notes,
+  };
+}
+
+// Checks the command ran, not that a plugin/skill actually landed — the
+// installer auto-detects agent binaries on PATH (e.g. `claude`) to pick an
+// install target, which this sandbox may not expose, making that outcome
+// environment-dependent rather than something the agent controls.
+function checkPluginInstallAttempted(ctx: LocalStackEvalContext): CheckResult {
+  const attempted = ctx.toolCalls.some(
+    (call) => call.command && PLUGIN_INSTALL_PATTERN.test(call.command)
+  );
+  return {
+    name: 'attempted `npx plugins add supabase-community/supabase-plugin`',
+    passed: attempted,
+  };
+}

evals/build-cli-004-quickstart-nextjs/PROMPT.md

@@ -0,0 +1,16 @@
+---
+stage: build
+suite: regression
+interface: cli
+product:
+  - database
+  - data-api
+topic:
+  - sdk
+projectRunning: false
+skills: []
+skipCliInstall: true
+motivation: the "Help me get set up with Supabase" prompt shown on the supabase.com/docs homepage
+---
+
+Help me get set up with Supabase. Do the following: 1. Install the Supabase CLI globally with `npm install -g supabase`. 2. Install the Supabase Plugin with `npx plugins add supabase-community/supabase-plugin`. 3. Review my project and determine whether Supabase is already initialized. If it is not initialized, run `supabase init`. 4. Suggest the most relevant next steps.

evals/build-cli-004-quickstart-nextjs/README.md

@@ -0,0 +1,11 @@
+# Adding an eval for another docs guide
+
+This guide explains how to add an eval for another AI setup prompt from the docs. The prompt can come from a quickstart page for a different framework, or from any other docs guide that gives an agent a setup prompt to follow.
+
+1. Create a sibling folder. Name it after the guide you're adding, for example `build-cli-005-quickstart-<framework>`.
+2. Copy the target prompt verbatim into `PROMPT.md`. Don't paraphrase it or add detail that the prompt doesn't already give. Keep `skills: []` and `skipCliInstall: true` in the frontmatter if the prompt has the agent install its own tooling.
+3. Seed `local/` with a minimal, unmodified starter that matches the guide's context. Don't include a `supabase/` directory, so the "is Supabase already initialized" step has something real to check.
+4. Reuse this eval's `EVAL.ts` checks as a starting point. Adjust the checks to match what the new guide's prompt requires, including the judge rubric's wording.
+5. Validate the eval before you open a PR. Run `pnpm eval:dry -- --eval <new-eval-id> --experiment claude-code-sonnet-5` to check the setup, and then run `pnpm eval -- --eval <new-eval-id> --experiment claude-code-sonnet-5` for a full pass.
+
+For the full eval-authoring workflow, including frontmatter fields, suite selection, and submitting for review, see `CONTRIBUTING.md`.

evals/build-cli-004-quickstart-nextjs/local/app/layout.tsx

@@ -0,0 +1,18 @@
+import type { Metadata } from 'next';
+
+export const metadata: Metadata = {
+  title: 'My App',
+  description: 'Generated by create-next-app',
+};
+
+export default function RootLayout({
+  children,
+}: Readonly<{
+  children: React.ReactNode;
+}>) {
+  return (
+    <html lang="en">
+      <body>{children}</body>
+    </html>
+  );
+}

evals/build-cli-004-quickstart-nextjs/local/app/page.tsx

@@ -0,0 +1,7 @@
+export default function Home() {
+  return (
+    <main>
+      <h1>Welcome to My App</h1>
+    </main>
+  );
+}

evals/build-cli-004-quickstart-nextjs/local/next-env.d.ts

@@ -0,0 +1,5 @@
+/// <reference types="next" />
+/// <reference types="next/image-types/global" />
+
+// NOTE: This file should not be edited
+// see https://nextjs.org/docs/app/api-reference/config/typescript for more information.

evals/build-cli-004-quickstart-nextjs/local/next.config.mjs

@@ -0,0 +1,4 @@
+/** @type {import('next').NextConfig} */
+const nextConfig = {};
+
+export default nextConfig;

evals/build-cli-004-quickstart-nextjs/local/package.json

@@ -0,0 +1,21 @@
+{
+  "name": "my-app",
+  "version": "0.1.0",
+  "private": true,
+  "scripts": {
+    "dev": "next dev",
+    "build": "next build",
+    "start": "next start"
+  },
+  "dependencies": {
+    "next": "^15.5.0",
+    "react": "^19.2.0",
+    "react-dom": "^19.2.0"
+  },
+  "devDependencies": {
+    "@types/node": "^22.0.0",
+    "@types/react": "^19.2.0",
+    "@types/react-dom": "^19.2.0",
+    "typescript": "^5.6.0"
+  }
+}

evals/build-cli-004-quickstart-nextjs/local/tsconfig.json

@@ -0,0 +1,21 @@
+{
+  "compilerOptions": {
+    "target": "ES2017",
+    "lib": ["dom", "dom.iterable", "esnext"],
+    "allowJs": true,
+    "skipLibCheck": true,
+    "strict": true,
+    "noEmit": true,
+    "esModuleInterop": true,
+    "module": "esnext",
+    "moduleResolution": "bundler",
+    "resolveJsonModule": true,
+    "isolatedModules": true,
+    "jsx": "preserve",
+    "incremental": true,
+    "plugins": [{ "name": "next" }],
+    "paths": { "@/*": ["./*"] }
+  },
+  "include": ["next-env.d.ts", "**/*.ts", "**/*.tsx", ".next/types/**/*.ts"],
+  "exclude": ["node_modules"]
+}

experiments/claude-code-sonnet-5-no-skills.ts

@@ -18,4 +18,5 @@ export default defineExperiment({
   }),
   localStack: localStackRuntime(),
   skills: [],
+  skipEval: (ev) => ev.metadata.skills?.length === 0,
 });