diff --git a/.github/workflows/eval-refresh.yml b/.github/workflows/eval-refresh.yml index 98b7e215..9b8162d0 100644 --- a/.github/workflows/eval-refresh.yml +++ b/.github/workflows/eval-refresh.yml @@ -226,7 +226,7 @@ jobs: if [ -n "$experiments_override" ]; then experiments_json="$(jq -Rc 'split(",") | map(gsub("^\\s+|\\s+$"; "")) | map(select(length > 0))' <<< "$experiments_override")" else - experiments_json="$(pnpm --silent eval -- list --experiment-suite "$experiment_suite")" + experiments_json="$(pnpm --silent eval -- list --experiment-suite "$experiment_suite" --eval "$id")" fi while IFS= read -r experiment; do diff --git a/apps/framework/harness/run-eval.ts b/apps/framework/harness/run-eval.ts index 0f758e53..e0e13374 100644 --- a/apps/framework/harness/run-eval.ts +++ b/apps/framework/harness/run-eval.ts @@ -371,11 +371,14 @@ async function runOne( // tools mode its skills are advertised in the prompt and loaded via the // load_skill tool. Skill sources (name+dir) are shared by both paths. const agentRunsInSandbox = exp.agent.runsInSandbox ?? false; - const skillSources = resolveSkillSources(exp.skills); + // A per-eval `skills` override replaces the experiment's own list entirely, + // so a scenario testing self-installed skills gets an empty list regardless + // of which experiment runs it. + const skillSources = resolveSkillSources(ev.metadata.skills ?? exp.skills); const availableSkills = skillSources.map((skill) => skill.name); const toolsSkills = ev.mode === 'tools' && !agentRunsInSandbox - ? loadToolsSkills(exp.skills) + ? loadToolsSkills(ev.metadata.skills ?? exp.skills) : []; const scorer = (await import(pathToFileURL(ev.evalPath).href)).default as | ToolScorer @@ -435,10 +438,8 @@ async function runOne( invokeFunction: hostedBackend.invokeFunction, } : undefined, - // Skills are installed into the sandbox and discovered by the agent - // (the session folds the discovery listing into its promptAddendum), - // so no skill text is injected into the prompt here. skills: skillSources, + skipCliInstall: ev.metadata.skipCliInstall, }) ); @@ -634,7 +635,7 @@ async function runConcurrent( async function main() { if (rawArgs.filter((a) => a !== '--')[0] === 'list') { const experiments = await loadExperiments(); - const filtered = + let filtered = EXPERIMENT_SUITE_FILTERS.length > 0 ? experiments.filter( (e) => @@ -644,6 +645,18 @@ async function main() { ) ) : experiments; + if (EVAL_FILTERS.length > 0) { + // Drop experiments that would skipEval every requested eval, so callers + // building an experiment x eval matrix (e.g. the eval-refresh workflow) + // don't plan a pair that will produce no results — and no artifact — + // to upload. + const evals = discoverEvals().filter((ev) => + EVAL_FILTERS.includes(ev.id) + ); + filtered = filtered.filter(({ config }) => + evals.some((ev) => !config.skipEval?.(ev)) + ); + } console.log(JSON.stringify(filtered.map((e) => e.name))); return; } @@ -748,6 +761,10 @@ async function main() { ); continue; } + if (config.skipEval?.(ev)) { + console.log(`SKIP ${name} x ${ev.id} (skipEval)`); + continue; + } if (DRY) { console.log(formatPlanLine(name, config, ev)); continue; diff --git a/apps/web/src/data/regression-eval-results.json b/apps/web/src/data/regression-eval-results.json index 5ac81bd2..592d9648 100644 --- a/apps/web/src/data/regression-eval-results.json +++ b/apps/web/src/data/regression-eval-results.json @@ -1,4 +1,56 @@ [ + { + "experiment": "claude-code-sonnet-5", + "experimentSuite": "regression", + "experimentDisplay": { + "agent": "claude-code", + "modelProvider": "anthropic", + "modelId": "claude-sonnet-5", + "reasoningEffort": "high" + }, + "eval": "build-cli-004-quickstart-nextjs", + "stage": "build", + "product": [ + "database", + "data-api" + ], + "topic": [ + "sdk" + ], + "suite": "regression", + "interface": "cli", + "passed": true, + "checks": [ + { + "name": "project initialized (supabase/config.toml exists)", + "passed": true + }, + { + "name": "supabase CLI is installed and runnable", + "passed": true + }, + { + "name": "suggested relevant next steps for a Supabase + Next.js project", + "passed": true, + "judgeNotes": "The response includes several concrete Supabase/Next.js next steps: running `supabase start`, installing `@supabase/supabase-js`/`@supabase/ssr`, setting `.env.local` variables for URL and anon key, and creating a first migration." + }, + { + "name": "attempted `npx plugins add supabase-community/supabase-plugin`", + "passed": true + } + ], + "skills": { + "available": [], + "loaded": [] + }, + "docs": { + "calls": [] + }, + "prompt": "Help me get set up with Supabase. Do the following: 1. Install the Supabase CLI globally with `npm install -g supabase`. 2. Install the Supabase Plugin with `npx plugins add supabase-community/supabase-plugin`. 3. Review my project and determine whether Supabase is already initialized. If it is not initialized, run `supabase init`. 4. Suggest the most relevant next steps.", + "promptSourcePath": "evals/build-cli-004-quickstart-nextjs/PROMPT.md", + "attempts": 1, + "sourcePath": "claude-code-sonnet-5/build-cli-004-quickstart-nextjs.json" + }, { "experiment": "claude-code-sonnet-5", "experimentSuite": "regression", diff --git a/evals/build-cli-004-quickstart-nextjs/EVAL.ts b/evals/build-cli-004-quickstart-nextjs/EVAL.ts new file mode 100644 index 00000000..829a1b0f --- /dev/null +++ b/evals/build-cli-004-quickstart-nextjs/EVAL.ts @@ -0,0 +1,109 @@ +import { + judge, + serializeTranscript, + type CheckResult, + type LocalStackEvalContext, + type LocalStackScorer, +} from '@supabase-evals/core'; +import { stripIndent } from 'common-tags'; + +// Tolerates npx flags between `npx` and `plugins` (e.g. `npx --yes plugins add ...`), +// since the agent is free to pass its own npx flags and still be attempting the install. +const PLUGIN_INSTALL_PATTERN = + /npx\s+(?:--?\S+\s+)*plugins\s+add\s+supabase-community\/supabase-plugin/i; + +const scorer: LocalStackScorer = async (ctx) => { + try { + const checks: CheckResult[] = [ + await checkSupabaseInitialized(ctx), + await checkCliFunctional(ctx), + await checkNextStepsSuggested(ctx), + checkPluginInstallAttempted(ctx), + ]; + + return { + passed: checks.every((check) => check.passed), + checks, + }; + } catch (error) { + const msg = error instanceof Error ? error.message : String(error); + return { + passed: false, + checks: [ + { + name: 'scorer evaluated quickstart setup', + passed: false, + notes: msg, + }, + ], + }; + } +}; + +export default scorer; + +async function checkSupabaseInitialized( + ctx: LocalStackEvalContext +): Promise { + const exists = await ctx.fileExists('supabase/config.toml'); + return { + name: 'project initialized (supabase/config.toml exists)', + passed: exists, + }; +} + +async function checkCliFunctional( + ctx: LocalStackEvalContext +): Promise { + const result = await ctx.exec('supabase --version'); + return { + name: 'supabase CLI is installed and runnable', + passed: result.ok, + notes: result.ok ? undefined : result.stderr || result.stdout, + }; +} + +async function checkNextStepsSuggested( + ctx: LocalStackEvalContext +): Promise { + const verdict = await judge({ + input: serializeTranscript(ctx.transcript, { includeToolCallInputs: true }), + rubric: stripIndent` + Pass if the assistant's final response suggests next steps for using + Supabase in this Next.js project that are relevant and specific to what + was just set up. + + A passing answer should reference at least one concrete, relevant next + step, such as: + - starting the local stack (supabase start) or creating a first migration + - installing/using @supabase/supabase-js (or an SSR helper) to connect + the Next.js app + - setting environment variables for the local API URL and anon key + - exploring Studio or writing a first table/schema + + Fail if the assistant stops after installation/init with no concrete + next steps, or the suggestions are generic enough to apply to any + project regardless of Supabase or Next.js. + `, + }); + + return { + name: 'suggested relevant next steps for a Supabase + Next.js project', + passed: verdict.passed, + judgeNotes: verdict.notes, + }; +} + +// Checks the command ran, not that a plugin/skill actually landed — the +// installer auto-detects agent binaries on PATH (e.g. `claude`) to pick an +// install target, which this sandbox may not expose, making that outcome +// environment-dependent rather than something the agent controls. +function checkPluginInstallAttempted(ctx: LocalStackEvalContext): CheckResult { + const attempted = ctx.toolCalls.some( + (call) => call.command && PLUGIN_INSTALL_PATTERN.test(call.command) + ); + return { + name: 'attempted `npx plugins add supabase-community/supabase-plugin`', + passed: attempted, + }; +} diff --git a/evals/build-cli-004-quickstart-nextjs/PROMPT.md b/evals/build-cli-004-quickstart-nextjs/PROMPT.md new file mode 100644 index 00000000..a60f7324 --- /dev/null +++ b/evals/build-cli-004-quickstart-nextjs/PROMPT.md @@ -0,0 +1,16 @@ +--- +stage: build +suite: regression +interface: cli +product: + - database + - data-api +topic: + - sdk +projectRunning: false +skills: [] +skipCliInstall: true +motivation: the "Help me get set up with Supabase" prompt shown on the supabase.com/docs homepage +--- + +Help me get set up with Supabase. Do the following: 1. Install the Supabase CLI globally with `npm install -g supabase`. 2. Install the Supabase Plugin with `npx plugins add supabase-community/supabase-plugin`. 3. Review my project and determine whether Supabase is already initialized. If it is not initialized, run `supabase init`. 4. Suggest the most relevant next steps. diff --git a/evals/build-cli-004-quickstart-nextjs/README.md b/evals/build-cli-004-quickstart-nextjs/README.md new file mode 100644 index 00000000..e4ea702d --- /dev/null +++ b/evals/build-cli-004-quickstart-nextjs/README.md @@ -0,0 +1,11 @@ +# Adding an eval for another docs guide + +This guide explains how to add an eval for another AI setup prompt from the docs. The prompt can come from a quickstart page for a different framework, or from any other docs guide that gives an agent a setup prompt to follow. + +1. Create a sibling folder. Name it after the guide you're adding, for example `build-cli-005-quickstart-`. +2. Copy the target prompt verbatim into `PROMPT.md`. Don't paraphrase it or add detail that the prompt doesn't already give. Keep `skills: []` and `skipCliInstall: true` in the frontmatter if the prompt has the agent install its own tooling. +3. Seed `local/` with a minimal, unmodified starter that matches the guide's context. Don't include a `supabase/` directory, so the "is Supabase already initialized" step has something real to check. +4. Reuse this eval's `EVAL.ts` checks as a starting point. Adjust the checks to match what the new guide's prompt requires, including the judge rubric's wording. +5. Validate the eval before you open a PR. Run `pnpm eval:dry -- --eval --experiment claude-code-sonnet-5` to check the setup, and then run `pnpm eval -- --eval --experiment claude-code-sonnet-5` for a full pass. + +For the full eval-authoring workflow, including frontmatter fields, suite selection, and submitting for review, see `CONTRIBUTING.md`. diff --git a/evals/build-cli-004-quickstart-nextjs/local/app/layout.tsx b/evals/build-cli-004-quickstart-nextjs/local/app/layout.tsx new file mode 100644 index 00000000..ff4380d8 --- /dev/null +++ b/evals/build-cli-004-quickstart-nextjs/local/app/layout.tsx @@ -0,0 +1,18 @@ +import type { Metadata } from 'next'; + +export const metadata: Metadata = { + title: 'My App', + description: 'Generated by create-next-app', +}; + +export default function RootLayout({ + children, +}: Readonly<{ + children: React.ReactNode; +}>) { + return ( + + {children} + + ); +} diff --git a/evals/build-cli-004-quickstart-nextjs/local/app/page.tsx b/evals/build-cli-004-quickstart-nextjs/local/app/page.tsx new file mode 100644 index 00000000..e0d6da53 --- /dev/null +++ b/evals/build-cli-004-quickstart-nextjs/local/app/page.tsx @@ -0,0 +1,7 @@ +export default function Home() { + return ( +
+

Welcome to My App

+
+ ); +} diff --git a/evals/build-cli-004-quickstart-nextjs/local/next-env.d.ts b/evals/build-cli-004-quickstart-nextjs/local/next-env.d.ts new file mode 100644 index 00000000..1b3be084 --- /dev/null +++ b/evals/build-cli-004-quickstart-nextjs/local/next-env.d.ts @@ -0,0 +1,5 @@ +/// +/// + +// NOTE: This file should not be edited +// see https://nextjs.org/docs/app/api-reference/config/typescript for more information. diff --git a/evals/build-cli-004-quickstart-nextjs/local/next.config.mjs b/evals/build-cli-004-quickstart-nextjs/local/next.config.mjs new file mode 100644 index 00000000..4678774e --- /dev/null +++ b/evals/build-cli-004-quickstart-nextjs/local/next.config.mjs @@ -0,0 +1,4 @@ +/** @type {import('next').NextConfig} */ +const nextConfig = {}; + +export default nextConfig; diff --git a/evals/build-cli-004-quickstart-nextjs/local/package.json b/evals/build-cli-004-quickstart-nextjs/local/package.json new file mode 100644 index 00000000..82e7d907 --- /dev/null +++ b/evals/build-cli-004-quickstart-nextjs/local/package.json @@ -0,0 +1,21 @@ +{ + "name": "my-app", + "version": "0.1.0", + "private": true, + "scripts": { + "dev": "next dev", + "build": "next build", + "start": "next start" + }, + "dependencies": { + "next": "^15.5.0", + "react": "^19.2.0", + "react-dom": "^19.2.0" + }, + "devDependencies": { + "@types/node": "^22.0.0", + "@types/react": "^19.2.0", + "@types/react-dom": "^19.2.0", + "typescript": "^5.6.0" + } +} diff --git a/evals/build-cli-004-quickstart-nextjs/local/tsconfig.json b/evals/build-cli-004-quickstart-nextjs/local/tsconfig.json new file mode 100644 index 00000000..6420eed1 --- /dev/null +++ b/evals/build-cli-004-quickstart-nextjs/local/tsconfig.json @@ -0,0 +1,21 @@ +{ + "compilerOptions": { + "target": "ES2017", + "lib": ["dom", "dom.iterable", "esnext"], + "allowJs": true, + "skipLibCheck": true, + "strict": true, + "noEmit": true, + "esModuleInterop": true, + "module": "esnext", + "moduleResolution": "bundler", + "resolveJsonModule": true, + "isolatedModules": true, + "jsx": "preserve", + "incremental": true, + "plugins": [{ "name": "next" }], + "paths": { "@/*": ["./*"] } + }, + "include": ["next-env.d.ts", "**/*.ts", "**/*.tsx", ".next/types/**/*.ts"], + "exclude": ["node_modules"] +} diff --git a/experiments/claude-code-sonnet-5-no-skills.ts b/experiments/claude-code-sonnet-5-no-skills.ts index 9c310c11..e209193e 100644 --- a/experiments/claude-code-sonnet-5-no-skills.ts +++ b/experiments/claude-code-sonnet-5-no-skills.ts @@ -18,4 +18,5 @@ export default defineExperiment({ }), localStack: localStackRuntime(), skills: [], + skipEval: (ev) => ev.metadata.skills?.length === 0, }); diff --git a/packages/core/src/eval-metadata.ts b/packages/core/src/eval-metadata.ts index 59a645cf..af0ec266 100644 --- a/packages/core/src/eval-metadata.ts +++ b/packages/core/src/eval-metadata.ts @@ -129,6 +129,22 @@ export type EvalMetadata = { * evals only). Defaults to false. */ hostedProject?: boolean; + /** + * Overrides the experiment's `skills` list for this eval only. An empty + * list (`skills: []`) tests an agent with no pre-installed Supabase + * skills, regardless of which experiment runs it — for a scenario where + * the prompt asks the agent to install skills itself. Omit the key + * entirely to use the experiment's own skill list. + */ + skills?: string[]; + /** + * Skips installing the real Supabase CLI into the sandbox before the agent + * starts (sandbox evals only). Defaults to false. Set true only for + * scenarios whose prompt has the agent install the CLI itself — the + * harness itself never invokes `supabase` when this is set, so pair it + * with `projectRunning: false`. + */ + skipCliInstall?: boolean; }; export type ParsedEvalMarkdown = { @@ -149,6 +165,8 @@ export const evalMetadataSchema = z.object({ // non-empty string as true. projectRunning: z.union([z.boolean(), z.stringbool()]).optional(), hostedProject: z.union([z.boolean(), z.stringbool()]).optional(), + skills: z.array(z.string().min(1)).optional(), + skipCliInstall: z.union([z.boolean(), z.stringbool()]).optional(), }); // Collapse a YAML scalar into a comparable token: trim, lowercase, and fold @@ -180,12 +198,14 @@ const toTokenList = (value: unknown): unknown => { .filter((item) => typeof item === 'string' && item.length > 0); }; -// `services` are real Supabase CLI service identifiers (e.g. `postgres-meta`, -// `storage-api`, `edge-runtime`), matched verbatim against ALL_SUPABASE_SERVICES -// in @supabase-evals/sandbox. They must NOT go through normalizeToken: folding -// hyphens to underscores would turn `postgres-meta` into `postgres_meta` and -// fail that match. Only trim + lowercase; preserve hyphens. Blanks are dropped. -const toServiceList = (value: unknown): unknown => +// `services` and `skills` are literal identifiers (Supabase CLI service names +// like `postgres-meta`, or skill directory names like +// `supabase-postgres-best-practices`), not enum tokens. They must NOT go +// through normalizeToken: folding hyphens to underscores would turn +// `postgres-meta` into `postgres_meta` and fail the verbatim match against +// ALL_SUPABASE_SERVICES (services) or the skills/ directory (skills). Only +// trim + lowercase; preserve hyphens. Blanks are dropped. +const toIdentifierList = (value: unknown): unknown => (Array.isArray(value) ? value : [value]) .map((item) => typeof item === 'string' || @@ -215,10 +235,17 @@ export const evalFrontmatterSchema = z.preprocess((raw) => { // `services: []` means database only; an omitted key means the full stack. // Only an explicit list is honored — any other value is treated as absent. services: Array.isArray(data.services) - ? toServiceList(data.services) + ? toIdentifierList(data.services) : undefined, projectRunning: data.projectRunning, hostedProject: data.hostedProject, + // `skills: []` means no pre-installed skills for this eval; an omitted + // key means "use the experiment's own skill list" (see resolveSkillSources + // in apps/framework/harness/run-eval.ts). + skills: Array.isArray(data.skills) + ? toIdentifierList(data.skills) + : undefined, + skipCliInstall: data.skipCliInstall, }; }, evalMetadataSchema); diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index caa480d2..cf25d701 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -36,6 +36,7 @@ import { import type { AgentHarnessId, CheckResult, + EvalMetadata, EvalSuite, ExperimentDisplayMetadata, ExperimentSuite, @@ -438,6 +439,13 @@ export type LocalStackSessionArgs = { * instead, so they ignore this. */ skills?: readonly SkillSource[]; + /** + * Skip installing the real Supabase CLI into the sandbox (from + * `skipCliInstall:` frontmatter), for scenarios whose prompt has the agent + * install the CLI itself. Also skips the CLI shim that enforces + * `includeServices`, since there's no real binary yet to resolve and wrap. + */ + skipCliInstall?: boolean; }; /** A mocked hosted project (platform-lite) the sandbox CLI is linked to. */ @@ -514,6 +522,13 @@ export type ExperimentConfig = { /** Local-stack environment (e.g. localStackRuntime() from @supabase-evals/sandbox). */ localStack?: LocalStackRuntime; skills: string[]; + /** + * Skip running specific evals against this experiment, e.g. a `*-no-skills` + * experiment skipping evals whose per-eval `skills` override is already + * `[]` — running those would just duplicate what this experiment's own + * empty skill list already covers. + */ + skipEval?: (ev: { id: string; metadata: EvalMetadata }) => boolean; }; export function getExperimentDisplayMetadata( diff --git a/packages/sandbox/Dockerfile b/packages/sandbox/Dockerfile index 3d9b771b..4885ae3f 100644 --- a/packages/sandbox/Dockerfile +++ b/packages/sandbox/Dockerfile @@ -8,6 +8,14 @@ RUN apt-get update -qq && apt-get install -y -qq --no-install-recommends \ ca-certificates curl git docker.io postgresql-client \ && rm -rf /var/lib/apt/lists/* +# Global npm installs (both the baked-in skills CLI below and the agent's own +# `npm install -g supabase` at setup time) land in a node-owned prefix instead +# of /usr/local, so they work without root/sudo — matching a typical user's +# machine, where global npm installs "just work" unelevated. +USER node +RUN mkdir -p /home/node/.npm-global +ENV NPM_CONFIG_PREFIX=/home/node/.npm-global + # Vercel's agent-skills CLI, used to install agent skills from local sources # into each session's workspace (both modes). Baked in (pinned) so installs are # offline and instant rather than an npx download per session. diff --git a/packages/sandbox/src/agent-environment.ts b/packages/sandbox/src/agent-environment.ts index 4d046395..d17bc08d 100644 --- a/packages/sandbox/src/agent-environment.ts +++ b/packages/sandbox/src/agent-environment.ts @@ -29,6 +29,8 @@ export interface LocalStackSetup { projectRunning?: boolean; /** Link the CLI to a mocked hosted project (platform-lite) at this host port. */ hosted?: { port: number; pgPort?: number; ref: string; accessToken: string }; + /** Skip installing the real CLI, for scenarios where the agent installs it itself. */ + skipCliInstall?: boolean; } export interface AgentEnvironmentOptions { @@ -78,6 +80,7 @@ export async function createAgentEnvironment( localDir: options.localDir, projectRunning: options.localStack.projectRunning, hosted: options.localStack.hosted, + skipCliInstall: options.localStack.skipCliInstall, }); } else if (options.localDir) { await sandbox.copyToContainer(options.localDir, sandbox.workdir); diff --git a/packages/sandbox/src/docker-sandbox.ts b/packages/sandbox/src/docker-sandbox.ts index 42f98052..dbfd0983 100644 --- a/packages/sandbox/src/docker-sandbox.ts +++ b/packages/sandbox/src/docker-sandbox.ts @@ -58,6 +58,7 @@ export const SANDBOX_CONTAINER_LABEL = 'supabase-evals-sandbox'; const CLIENT_TIMEOUT_HEADROOM_MS = 20_000; const SANDBOX_PATH = [ + '/home/node/.npm-global/bin', '/usr/local/sbin', '/usr/local/bin', '/usr/sbin', diff --git a/packages/sandbox/src/local-stack-runtime.ts b/packages/sandbox/src/local-stack-runtime.ts index 5fe61875..f9564bdf 100644 --- a/packages/sandbox/src/local-stack-runtime.ts +++ b/packages/sandbox/src/local-stack-runtime.ts @@ -79,10 +79,8 @@ export function localStackRuntime( projectRunning, hosted, skills, + skipCliInstall, }) { - // Local-stack mode = the shared agent environment with the Supabase local - // stack started. Everything else (image, tooling, skills) is identical to - // tools mode; only the `localStack` component differs. const env = await createAgentEnvironment({ cliVersion: cliVersion ?? options.cliVersion, localDir, @@ -98,17 +96,21 @@ export function localStackRuntime( accessToken: hosted.accessToken, } : undefined, + skipCliInstall, }, }); const sandbox = env.sandbox; const mcpServers = await resolveMcpServers(options, hosted); - const baseAddendum = - 'The Supabase CLI (`supabase`), docker, psql, git, and curl are installed in the workspace. ' + - 'Use the bash tool to run commands (the working directory is always the workspace root) ' + - 'and the files tools to inspect and modify files. ' + - 'Services started with `supabase start` are reachable on their default 127.0.0.1 ports.'; + const baseAddendum = skipCliInstall + ? 'docker, psql, git, and curl are installed in the workspace, but the Supabase CLI is not — install it yourself. ' + + 'Use the bash tool to run commands (the working directory is always the workspace root) ' + + 'and the files tools to inspect and modify files.' + : 'The Supabase CLI (`supabase`), docker, psql, git, and curl are installed in the workspace. ' + + 'Use the bash tool to run commands (the working directory is always the workspace root) ' + + 'and the files tools to inspect and modify files. ' + + 'Services started with `supabase start` are reachable on their default 127.0.0.1 ports.'; return { tools: buildLocalStackTools(sandbox), diff --git a/packages/sandbox/src/supabase.ts b/packages/sandbox/src/supabase.ts index 53c89c68..0f2c3857 100644 --- a/packages/sandbox/src/supabase.ts +++ b/packages/sandbox/src/supabase.ts @@ -155,6 +155,13 @@ export async function installSupabaseCli( export interface SetupSupabaseSandboxOptions { /** Supabase CLI version to install into the sandbox (pinned default). */ cliVersion?: string; + /** + * Skip installing the real Supabase CLI, for scenarios whose prompt has the + * agent install it itself. Also skips the CLI shim (there's no real binary + * to resolve and wrap yet), so service exclusion isn't enforced — pair with + * an eval that omits `services` (full stack) or accepts that limitation. + */ + skipCliInstall?: boolean; /** * Local-stack services this session needs; every other service is excluded * from `supabase start` to keep boots fast. Omitted means the full stack. @@ -221,8 +228,10 @@ export async function setupSupabaseSandbox( ): Promise { // The Supabase CLI is a local-stack component: install it here (not in the // base image) so tools-mode sandboxes don't have it. Everything below needs - // the CLI, so it goes first. - await installSupabaseCli(sandbox, options.cliVersion); + // the CLI, so it goes first — unless the scenario has the agent install it. + if (!options.skipCliInstall) { + await installSupabaseCli(sandbox, options.cliVersion); + } // Let the non-root sandbox user talk to the mounted Docker socket by // joining the socket's group. chmod would also work but mutates the host @@ -276,13 +285,16 @@ export async function setupSupabaseSandbox( // linked DB commands at the hosted wire endpoint, and appends `-x` to any // `supabase start` the agent runs itself (so service exclusions hold even when // the stack is already up and the agent restarts it). No-op when there's - // nothing to exclude and no hosted wire endpoint. - await installSupabaseCliWrapper( - sandbox, - options.includeServices, - poolerUrlPath, - options.hosted !== undefined - ); + // nothing to exclude and no hosted wire endpoint. Skipped when there's no CLI + // installed yet to resolve and wrap (skipCliInstall). + if (!options.skipCliInstall) { + await installSupabaseCliWrapper( + sandbox, + options.includeServices, + poolerUrlPath, + options.hosted !== undefined + ); + } } /** diff --git a/packages/sandbox/test/docker.test.ts b/packages/sandbox/test/docker.test.ts index 7d295c3d..5107b2a3 100644 --- a/packages/sandbox/test/docker.test.ts +++ b/packages/sandbox/test/docker.test.ts @@ -112,6 +112,58 @@ describe.runIf(process.env.SANDBOX_DOCKER_TESTS)( } ); + it( + "skipCliInstall leaves the CLI absent, and the wrapper doesn't crash setup", + { timeout: TEST_TIMEOUT_MS }, + async () => { + const image = await ensureSupabaseSandboxImage(); + const sandbox = await DockerSandbox.create({ image, network: 'host' }); + try { + await setupSupabaseSandbox(sandbox, { + projectRunning: false, + skipCliInstall: true, + }); + + const version = await sandbox.runShell('supabase --version'); + expect(version.ok).toBe(false); + + // The agent installs it itself; the harness never touched the CLI. + const install = await sandbox.runShellAsRoot( + `ARCH="$(dpkg --print-architecture)" && ` + + `curl -fsSL "https://github.com/supabase/cli/releases/download/v${SUPABASE_CLI_VERSION}/supabase_${SUPABASE_CLI_VERSION}_linux_$ARCH.deb" -o /tmp/supabase.deb && ` + + `dpkg -i /tmp/supabase.deb && rm /tmp/supabase.deb` + ); + expect(install.ok).toBe(true); + + const versionAfter = await sandbox.runShell('supabase --version'); + expect(versionAfter.ok).toBe(true); + expect(versionAfter.stdout.trim()).toBe(SUPABASE_CLI_VERSION); + } finally { + await sandbox.stop(); + } + } + ); + + it( + 'npm install -g lands on PATH without sudo/workarounds', + { timeout: TEST_TIMEOUT_MS }, + async () => { + const image = await ensureSupabaseSandboxImage(); + const sandbox = await DockerSandbox.create({ image, network: 'host' }); + try { + const install = await sandbox.runShell( + 'npm install -g supabase 2>&1' + ); + expect(install.ok).toBe(true); + + const version = await sandbox.runShell('supabase --version'); + expect(version.ok).toBe(true); + } finally { + await sandbox.stop(); + } + } + ); + it( 'installs a local skill into the workspace with the skills CLI', { timeout: TEST_TIMEOUT_MS }, diff --git a/packages/sandbox/test/unit.test.ts b/packages/sandbox/test/unit.test.ts index 2713975c..bfbef903 100644 --- a/packages/sandbox/test/unit.test.ts +++ b/packages/sandbox/test/unit.test.ts @@ -402,6 +402,71 @@ describe('cliVersion frontmatter', () => { }); }); +describe('skills frontmatter', () => { + const buildMarkdown = (extra: string) => + [ + '---', + 'stage: build', + 'suite: regression', + 'interface: cli', + 'product: [database]', + 'topic: [sdk]', + extra, + '---', + 'body', + ].join('\n'); + + it('preserves hyphenated skill directory names', () => { + const { metadata } = parseEvalMarkdown( + buildMarkdown('skills: [supabase, supabase-postgres-best-practices]') + ); + expect(metadata.skills).toEqual([ + 'supabase', + 'supabase-postgres-best-practices', + ]); + }); + + it('parses an empty override distinctly from an omitted key', () => { + const overridden = parseEvalMarkdown(buildMarkdown('skills: []')); + expect(overridden.metadata.skills).toEqual([]); + + const omitted = parseEvalMarkdown(buildMarkdown('')); + expect(omitted.metadata.skills).toBeUndefined(); + }); +}); + +describe('skipCliInstall frontmatter', () => { + const buildMarkdown = (extra: string) => + [ + '---', + 'stage: build', + 'suite: regression', + 'interface: cli', + 'product: [database]', + 'topic: [sdk]', + extra, + '---', + 'body', + ].join('\n'); + + it('accepts a real boolean and a quoted string form', () => { + expect( + parseEvalMarkdown(buildMarkdown('skipCliInstall: true')).metadata + .skipCliInstall + ).toBe(true); + expect( + parseEvalMarkdown(buildMarkdown('skipCliInstall: "true"')).metadata + .skipCliInstall + ).toBe(true); + }); + + it('defaults to undefined when omitted', () => { + expect( + parseEvalMarkdown(buildMarkdown('')).metadata.skipCliInstall + ).toBeUndefined(); + }); +}); + describe('resolveSandboxPath', () => { it('accepts and normalizes relative paths', () => { expect(resolveSandboxPath('a/b.txt')).toBe('a/b.txt');