From a9ff80b78a05274081eb3be0824ddece5a71f27a Mon Sep 17 00:00:00 2001 From: Pedro Rodrigues Date: Thu, 2 Jul 2026 16:01:13 -0700 Subject: [PATCH] feat(eval): add Cursor CLI agent harness MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds `cursor-cli` (Cursor's `cursor-agent`) as an eval agent alongside claude-code and codex, on the same per-agent framework (runner + parser + index + experiment): - Runner: pinned standalone-binary install (not npm) — fetches the version's tarball from Cursor's download host and symlinks the binary onto PATH; runs `--print --output-format stream-json --model --trust --force --approve-mcps` with the prompt on stdin; MCP written to ~/.cursor/mcp.json. - Parser: Cursor's stream-json (SDK messages) — tool type is the `*ToolCall` key, started/completed paired by call_id, {success}/{error} result shape, terminal result for the stop reason; skill tracking via the shared extractor. - Model id is a free-form string; provider "cursor". - Experiment cursor-cli.ts: composer-2.5 (Max mode off ⇒ no reasoningEffort), suite benchmark. CURSOR_API_KEY wired into eval-refresh CI + .env.example. - reasoningEffort is agent-scoped: cursor accepts only `max` (Max mode); claude-code/codex now exclude `max` (it's Cursor-specific). Verified end-to-end: cursor-cli (composer-2.5, CLI 2026.05.01-eea359f) passes investigate-db-001 (3/3) — install, MCP, stream parse (29/29 tools paired), skill loading, and scoring all work. Co-Authored-By: Claude Opus 4.8 (1M context) --- .env.example | 1 + .github/workflows/eval-refresh.yml | 2 + apps/web/src/App.tsx | 10 + experiments/cursor-cli.ts | 27 +++ packages/core/src/agents/claude-code/index.ts | 7 +- packages/core/src/agents/codex/index.ts | 7 +- packages/core/src/agents/cursor-cli/index.ts | 46 ++++ .../core/src/agents/cursor-cli/parser.test.ts | 126 +++++++++++ packages/core/src/agents/cursor-cli/parser.ts | 205 ++++++++++++++++++ packages/core/src/agents/cursor-cli/runner.ts | 136 ++++++++++++ packages/core/src/agents/engine.ts | 2 + packages/core/src/agents/registry.ts | 7 +- packages/core/src/eval-metadata.ts | 13 +- packages/core/src/index.ts | 1 + 14 files changed, 583 insertions(+), 7 deletions(-) create mode 100644 experiments/cursor-cli.ts create mode 100644 packages/core/src/agents/cursor-cli/index.ts create mode 100644 packages/core/src/agents/cursor-cli/parser.test.ts create mode 100644 packages/core/src/agents/cursor-cli/parser.ts create mode 100644 packages/core/src/agents/cursor-cli/runner.ts diff --git a/.env.example b/.env.example index 19a0696f..f6246110 100644 --- a/.env.example +++ b/.env.example @@ -1,3 +1,4 @@ # AI SDK Core direct-provider credentials. ANTHROPIC_API_KEY= OPENAI_API_KEY= +CURSOR_API_KEY= diff --git a/.github/workflows/eval-refresh.yml b/.github/workflows/eval-refresh.yml index 3991b135..5816c200 100644 --- a/.github/workflows/eval-refresh.yml +++ b/.github/workflows/eval-refresh.yml @@ -212,6 +212,7 @@ jobs: env: ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + CURSOR_API_KEY: ${{ secrets.CURSOR_API_KEY }} steps: - name: Checkout uses: actions/checkout@9f698171ed81b15d1823a05fc7211befd50c8ae0 # v6.0.3 @@ -239,6 +240,7 @@ jobs: { echo "ANTHROPIC_API_KEY=${ANTHROPIC_API_KEY}" echo "OPENAI_API_KEY=${OPENAI_API_KEY}" + echo "CURSOR_API_KEY=${CURSOR_API_KEY}" } > .env - name: Run evals diff --git a/apps/web/src/App.tsx b/apps/web/src/App.tsx index 741234e7..fc2edad6 100644 --- a/apps/web/src/App.tsx +++ b/apps/web/src/App.tsx @@ -150,6 +150,7 @@ const AGENT_LABELS = { "ai-sdk": "AI SDK", "claude-code": "Claude Code", codex: "Codex", + "cursor-cli": "Cursor CLI", } satisfies Record function capitalize(value: string) { @@ -187,12 +188,21 @@ function formatOpenAiModel(modelId: string) { .join(" ") } +function formatCursorModel(modelId: string) { + // composer-2.5 → Composer 2.5; auto → Auto. Split family from version parts. + const [family = "", ...rest] = modelId.split("-") + if (!family) return modelId + return [capitalize(family), rest.join(".")].filter(Boolean).join(" ") +} + function formatModel(display: ExperimentDisplay) { switch (display.modelProvider) { case "anthropic": return formatAnthropicModel(display.modelId) case "openai": return formatOpenAiModel(display.modelId) + case "cursor": + return formatCursorModel(display.modelId) } } diff --git a/experiments/cursor-cli.ts b/experiments/cursor-cli.ts new file mode 100644 index 00000000..965a1e90 --- /dev/null +++ b/experiments/cursor-cli.ts @@ -0,0 +1,27 @@ +import { + cursorCliAgent, + defineExperiment, + platformLiteRuntime, + supabaseMcpServer, +} from "@supabase-evals/core"; +import { localStackRuntime } from "@supabase-evals/sandbox"; + +// Cursor's CLI (`cursor-agent`) driving Composer 2.5 (Cursor's own model, fast +// mode / Max mode off). Like Claude Code / Codex it runs in both modes: +// `runtime` supplies the MCP servers for tools-mode evals (written into +// ~/.cursor/mcp.json) and `localStack` drives local-stack evals. Which mode an +// eval uses is a property of the eval, not the agent. +export default defineExperiment({ + suite: ["benchmark"], + agent: cursorCliAgent({ + model: "composer-2.5", + // Max mode is off (the default), so no reasoningEffort is recorded. Cursor + // has no low/medium/high dial — set reasoningEffort: "max" only to record + // that Max mode is on. + }), + runtime: platformLiteRuntime({ + mcpServers: [supabaseMcpServer()], + }), + localStack: localStackRuntime(), + skills: ["supabase", "supabase-postgres-best-practices"], +}); diff --git a/packages/core/src/agents/claude-code/index.ts b/packages/core/src/agents/claude-code/index.ts index 076953d0..00a8b4f2 100644 --- a/packages/core/src/agents/claude-code/index.ts +++ b/packages/core/src/agents/claude-code/index.ts @@ -18,8 +18,11 @@ export function claudeCodeAgent( options: { /** Anthropic model id (typed from `@anthropic-ai/sdk`). Defaults to Sonnet. */ model?: AnthropicModel; - /** Reasoning effort (`--effort`). Omit to use Claude Code's own default. */ - reasoningEffort?: ReasoningEffortLevel; + /** + * Reasoning effort (`--effort`). Omit to use Claude Code's own default. + * `max` is excluded — it's Cursor's Max-mode marker, not a Claude effort. + */ + reasoningEffort?: Exclude; /** Override the pinned CLI version. */ cliVersion?: string; } = {}, diff --git a/packages/core/src/agents/codex/index.ts b/packages/core/src/agents/codex/index.ts index ea6792d3..8a92c2cc 100644 --- a/packages/core/src/agents/codex/index.ts +++ b/packages/core/src/agents/codex/index.ts @@ -17,8 +17,11 @@ export function codexAgent( options: { /** OpenAI model id (typed from `openai`; any string accepted). */ model?: CodexModel; - /** Reasoning effort (`model_reasoning_effort`). Omit to use Codex's default. */ - reasoningEffort?: ReasoningEffortLevel; + /** + * Reasoning effort (`model_reasoning_effort`). Omit to use Codex's default. + * `max` is excluded — it's Cursor's Max-mode marker, not a Codex effort. + */ + reasoningEffort?: Exclude; /** Override the pinned CLI version. */ cliVersion?: string; } = {}, diff --git a/packages/core/src/agents/cursor-cli/index.ts b/packages/core/src/agents/cursor-cli/index.ts new file mode 100644 index 00000000..a882284a --- /dev/null +++ b/packages/core/src/agents/cursor-cli/index.ts @@ -0,0 +1,46 @@ +/** + * Cursor CLI agent. Owns everything Cursor-CLI-specific: it wires its own runner + * + parser into the public `cursorCliAgent` factory (via the generic + * `createCliAgent` engine) and exports the registry definition the harness uses + * to parse Cursor CLI transcripts. Runs in both modes, like Claude Code / Codex. + * + * Cursor has no low/medium/high effort dial — only "Max mode" (off by default). + * So `reasoningEffort` here is display-only: set it to `max` to record that Max + * mode is on (for benchmark metadata parity), or omit it for the default fast + * mode. It is recorded, not applied — cursor-agent exposes no flag to toggle it. + */ + +import type { AgentHarness } from "../../index.js"; +import type { ReasoningEffortLevel } from "../../eval-metadata.js"; +import { createCliAgent } from "../engine.js"; +import type { AgentDefinition } from "../types.js"; +import { cursorCliRunner, type CursorCliModel } from "./runner.js"; +import { cursorCliParser } from "./parser.js"; + +/** Cursor's CLI (`cursor-agent`) as an `AgentHarness`. */ +export function cursorCliAgent( + options: { + /** Cursor model id (e.g. `composer-2.5`). */ + model?: CursorCliModel; + /** + * Display-only effort metadata. Cursor has no low/medium/high dial — only + * Max mode — so the only settable value is `max` (Max mode on); omit it for + * the default fast mode. + */ + reasoningEffort?: Extract; + /** Override the pinned CLI version. */ + cliVersion?: string; + } = {}, +): AgentHarness { + return createCliAgent(cursorCliRunner, cursorCliParser, { + model: options.model ?? cursorCliRunner.defaultModel, + reasoningEffort: options.reasoningEffort, + cliVersion: options.cliVersion, + }); +} + +/** Runner + parser pairing for the agent registry (id comes from `runner.id`). */ +export const cursorCliDefinition: AgentDefinition = { + runner: cursorCliRunner, + parser: cursorCliParser, +}; diff --git a/packages/core/src/agents/cursor-cli/parser.test.ts b/packages/core/src/agents/cursor-cli/parser.test.ts new file mode 100644 index 00000000..cc7b6888 --- /dev/null +++ b/packages/core/src/agents/cursor-cli/parser.test.ts @@ -0,0 +1,126 @@ +import { describe, expect, it } from "vitest"; +import { cursorCliParser } from "./parser.js"; +import { buildCursorMcpConfig, cursorCliRunner } from "./runner.js"; +import { adaptTranscript } from "../../parsers/adapt.js"; + +/** A representative `cursor-agent --output-format stream-json` stream. */ +const SESSION = [ + JSON.stringify({ type: "system", subtype: "init", model: "composer-2.5", session_id: "s1" }), + JSON.stringify({ + type: "user", + message: { role: "user", content: [{ type: "text", text: "Count the rows." }] }, + }), + JSON.stringify({ + type: "tool_call", + subtype: "started", + call_id: "c1", + timestamp_ms: 1_780_000_000_000, + tool_call: { shellToolCall: { args: { command: "echo 42" } } }, + }), + JSON.stringify({ + type: "tool_call", + subtype: "completed", + call_id: "c1", + tool_call: { shellToolCall: { args: { command: "echo 42" }, result: { success: { output: "42" } } } }, + }), + JSON.stringify({ + type: "assistant", + message: { role: "assistant", content: [{ type: "text", text: "The count is 42." }] }, + }), + JSON.stringify({ type: "result", subtype: "success", is_error: false, result: "The count is 42." }), +].join("\n"); + +describe("cursorCliParser", () => { + it("maps a shellToolCall to a canonical shell call, paired by call_id", () => { + const { events, errors } = cursorCliParser.parseTranscript(SESSION); + expect(errors).toEqual([]); + + const calls = events.filter((e) => e.type === "tool_call"); + expect(calls.map((e) => e.tool?.name)).toEqual(["shell"]); + expect(calls.map((e) => e.tool?.originalName)).toEqual(["shellToolCall"]); + expect(calls[0].tool?.id).toBe("c1"); + expect(calls[0].tool?.command).toBe("echo 42"); + + const results = events.filter((e) => e.type === "tool_result"); + expect(results[0].tool?.id).toBe("c1"); + expect(results[0].tool?.success).toBe(true); + }); + + it("adapts the transcript: report, one assistant step, paired tool result", () => { + const { events } = cursorCliParser.parseTranscript(SESSION); + const adapted = adaptTranscript(events); + expect(adapted.agentReport).toBe("The count is 42."); + expect(adapted.steps).toBe(1); + expect(adapted.toolCalls).toEqual([ + { + endpoint: "shellToolCall", + body: { command: "echo 42" }, + command: "echo 42", + result: { success: { output: "42" } }, + error: undefined, + ts: 1_780_000_000_000, + }, + ]); + }); + + it("marks a failed tool_result via the error result shape", () => { + const stream = [ + JSON.stringify({ type: "tool_call", subtype: "started", call_id: "t1", tool_call: { shellToolCall: { args: { command: "false" } } } }), + JSON.stringify({ type: "tool_call", subtype: "completed", call_id: "t1", tool_call: { shellToolCall: { args: { command: "false" }, result: { error: { message: "boom" } } } } }), + ].join("\n"); + const { events } = cursorCliParser.parseTranscript(stream); + const result = events.find((e) => e.type === "tool_result"); + expect(result?.tool?.success).toBe(false); + expect(adaptTranscript(events).toolCalls[0].error).toContain("boom"); + }); + + it("identifies a loaded skill from a SKILL.md read", () => { + const stream = JSON.stringify({ + type: "tool_call", + subtype: "started", + call_id: "r1", + tool_call: { readToolCall: { args: { path: "/workspace/.claude/skills/supabase/SKILL.md" } } }, + }); + const call = cursorCliParser.parseTranscript(stream).events.find((e) => e.type === "tool_call"); + expect(call?.tool?.name).toBe("file_read"); + expect(call?.tool?.loadedSkill).toBe("supabase"); + expect(adaptTranscript([call!]).toolCalls[0].loadedSkill).toBe("supabase"); + }); + + it("never throws on malformed lines and records the error", () => { + const { errors } = cursorCliParser.parseTranscript("not json\n{}\n"); + expect(errors.length).toBe(1); + }); +}); + +describe("cursor-cli runner", () => { + const ok = { ok: true, exitCode: 0, stdout: "", stderr: "" }; + + it("deriveStopReason reads the terminal result status", () => { + expect(cursorCliRunner.deriveStopReason!(SESSION, ok)).toBe("stop"); + const errored = JSON.stringify({ type: "result", subtype: "error", is_error: true }); + expect(cursorCliRunner.deriveStopReason!(errored, ok)).toBe("error"); + }); + + it("deriveStopReason falls back to the process result when there's no terminal event", () => { + const timedOut = { ok: false, exitCode: 124, stdout: "", stderr: "timed out" }; + expect(cursorCliRunner.deriveStopReason!("", timedOut)).toBe("timeout"); + }); + + it("reads the key from CURSOR_API_KEY", () => { + expect(cursorCliRunner.apiKeyEnvVar).toBe("CURSOR_API_KEY"); + }); + + it("builds cursor-agent's mcp.json shape", () => { + const config = JSON.parse( + buildCursorMcpConfig({ + supabase: { command: "npx", args: ["-y", "srv"], env: { TOKEN: "t" } }, + docs: { command: "docs-server" }, + }), + ); + expect(config.mcpServers).toEqual({ + supabase: { command: "npx", args: ["-y", "srv"], env: { TOKEN: "t" } }, + docs: { command: "docs-server" }, + }); + }); +}); diff --git a/packages/core/src/agents/cursor-cli/parser.ts b/packages/core/src/agents/cursor-cli/parser.ts new file mode 100644 index 00000000..1cbbc6c9 --- /dev/null +++ b/packages/core/src/agents/cursor-cli/parser.ts @@ -0,0 +1,205 @@ +/** + * Cursor CLI transcript parser — for `cursor-agent --output-format stream-json`. + * + * The stream is newline-delimited SDK-message events: + * {"type":"system","subtype":"init","model":"…","session_id":"…"} + * {"type":"user","message":{"role":"user","content":[{"type":"text","text":"…"}]}} + * {"type":"assistant","message":{"role":"assistant","content":[{"type":"text","text":"…"}]}} + * {"type":"tool_call","subtype":"started","call_id":"…","tool_call":{"shellToolCall":{"args":{…}}}} + * {"type":"tool_call","subtype":"completed","call_id":"…","tool_call":{"shellToolCall":{"args":{…},"result":{"success":{…}}}}} + * {"type":"result","subtype":"success","is_error":false,"result":""} + * + * A tool call's *type* is the single `*ToolCall` key inside `tool_call` (e.g. + * `shellToolCall`, `readToolCall`); `started`/`completed` are correlated by + * `call_id`. Assistant events carry the full message text (not deltas, since we + * don't pass `--stream-partial-output`). Cursor may also emit `thinking` events + * (its hidden reasoning); only the `completed` one carries the full text. + * `system`/`result` carry no transcript content (the runner reads the terminal + * `result` for the stop reason). + * + * Adapted from `@supabase/agent-evals` (packages/agent-eval/src/parsers). + */ + +import { isRecord, parseJsonlRecords } from "../../json.js"; +import type { ParsedTranscript, TranscriptEvent } from "../../transcript/types.js"; +import type { AgentTranscriptParser } from "../../parsers/types.js"; +import { normalizeToolName, type AgentToolMap } from "../../parsers/shared/normalize.js"; +import { + extractArgs, + extractLoadedSkillFromText, + type ArgFieldMap, +} from "../../parsers/shared/extract.js"; + +/** + * cursor-agent's tool-call keys → canonical names. Cursor names built-in tools + * by the `*ToolCall` key wrapping their payload. Owned here, not in shared. + * Unmapped keys fall through to `unknown`. + */ +const CURSOR_CLI_TOOLS: AgentToolMap = { + tools: { + readToolCall: "file_read", + writeToolCall: "file_write", + editToolCall: "file_edit", + deleteToolCall: "file_write", + shellToolCall: "shell", + grepToolCall: "grep", + globToolCall: "glob", + lsToolCall: "list_dir", + searchToolCall: "web_search", + fetchToolCall: "web_fetch", + updateTodosToolCall: "agent_task", + }, +}; + +/** + * cursor-agent tool args → normalized fields (key names observed across Cursor's + * built-in tools). `shell` carries the command in `command`; file tools the path + * in `path`/`file_path`/…; `fetch` the URL in `url`/`uri`. + */ +const CURSOR_CLI_ARG_FIELDS: ArgFieldMap = { + path: ["path", "file_path", "filePath", "file", "filename", "target"], + command: ["command", "cmd"], + url: ["url", "uri", "href"], +}; + +function str(value: unknown): string | undefined { + return typeof value === "string" ? value : undefined; +} + +/** Epoch-ms (or pass-through ISO) → ISO string. */ +function toISO(value: unknown): string | undefined { + if (typeof value === "number") return new Date(value).toISOString(); + if (typeof value === "string") return value; + return undefined; +} + +/** Join the `text` blocks of a Cursor `message.content` array (or a plain string). */ +function extractMessageText(content: unknown): string | undefined { + if (typeof content === "string") return content; + if (!Array.isArray(content)) return undefined; + const texts = content + .filter((c): c is Record => isRecord(c) && c.type === "text") + .map((c) => str(c.text)) + .filter((t): t is string => Boolean(t)); + return texts.length > 0 ? texts.join("") : undefined; +} + +/** The single `*ToolCall` key inside a `tool_call` object, and its payload. */ +function findToolCall( + toolCall: Record, +): { key: string; inner: Record } | undefined { + for (const key of Object.keys(toolCall)) { + if (key.endsWith("ToolCall") && isRecord(toolCall[key])) { + return { key, inner: toolCall[key] as Record }; + } + } + return undefined; +} + +/** Cursor tool result is `{ success: {...} }` on success, `{ error: {...} }` on failure. */ +function resultSuccess(result: unknown): boolean | undefined { + if (!isRecord(result)) return undefined; + if ("success" in result) return true; + if ("error" in result) return false; + return undefined; +} + +function recordToEvents(data: Record): TranscriptEvent[] { + const type = str(data.type); + if (!type) return []; + const timestamp = toISO(data.timestamp_ms); + + switch (type) { + case "user": + case "assistant": { + const message = isRecord(data.message) ? data.message : undefined; + const content = message ? extractMessageText(message.content) : undefined; + if (!content) return []; + return [{ timestamp, type: "message", role: type, content, raw: data }]; + } + case "thinking": { + // Only the completed reasoning block carries the full text; skip deltas. + if (data.subtype !== "completed") return []; + const content = str(data.text); + return content ? [{ timestamp, type: "thinking", content, raw: data }] : []; + } + case "tool_call": { + const toolCall = isRecord(data.tool_call) ? findToolCall(data.tool_call) : undefined; + if (!toolCall) return []; + const id = str(data.call_id); + const name = normalizeToolName(toolCall.key, CURSOR_CLI_TOOLS); + + if (data.subtype === "started") { + const args = isRecord(toolCall.inner.args) ? toolCall.inner.args : {}; + const normalized = extractArgs(args, CURSOR_CLI_ARG_FIELDS); + const tool: NonNullable = { + name, + originalName: toolCall.key, + id, + args, + }; + if (normalized.path) tool.path = normalized.path; + if (normalized.command) tool.command = normalized.command; + if (normalized.url) tool.url = normalized.url; + tool.loadedSkill = loadedSkillFromCursorCall(tool); + return [{ timestamp, type: "tool_call", tool, raw: data }]; + } + if (data.subtype === "completed") { + // Pair with the call by `id`; the adapter reads the name off the + // tool_call, so the result's own name isn't authoritative. + return [ + { + timestamp, + type: "tool_result", + tool: { + name, + originalName: toolCall.key, + id, + result: toolCall.inner.result, + success: resultSuccess(toolCall.inner.result), + }, + raw: data, + }, + ]; + } + return []; + } + case "result": { + // Only surface a terminal error here; the final assistant text already + // arrived as an `assistant` event. Success carries no transcript content. + if (data.is_error !== true && data.subtype !== "error" && data.status !== "error") { + return []; + } + const error = isRecord(data.error) ? data.error : undefined; + const message = (error && str(error.message)) || str(data.result) || JSON.stringify(data); + return [{ timestamp, type: "error", content: message, raw: data }]; + } + // system/init and other metadata events carry no transcript content. + default: + return []; + } +} + +/** Identifies Cursor skill loads from normalized file paths or shell commands. */ +function loadedSkillFromCursorCall( + tool: NonNullable, +): string | undefined { + if (tool.path) return extractLoadedSkillFromText(tool.path); + if (tool.command) return extractLoadedSkillFromText(tool.command); + return undefined; +} + +export const cursorCliParser: AgentTranscriptParser = { + parseTranscript(raw: string): ParsedTranscript { + const { records, errors } = parseJsonlRecords(raw); + const events: TranscriptEvent[] = []; + for (const record of records) { + try { + events.push(...recordToEvents(record)); + } catch (e) { + errors.push(e instanceof Error ? e.message : String(e)); + } + } + return { events, errors }; + }, +}; diff --git a/packages/core/src/agents/cursor-cli/runner.ts b/packages/core/src/agents/cursor-cli/runner.ts new file mode 100644 index 00000000..8de1d005 --- /dev/null +++ b/packages/core/src/agents/cursor-cli/runner.ts @@ -0,0 +1,136 @@ +/** + * Cursor CLI runner. Headless via `cursor-agent --print --output-format + * stream-json` (Cursor's `agent` binary streams newline-delimited SDK events to + * stdout; see ./parser.ts). Like Claude Code / Codex it runs in both modes: the + * sandbox carries its shell/file tools either way, and tools mode routes + * Supabase access through MCP (`~/.cursor/mcp.json`). + * + * Notes: + * - Single provider (Cursor routes the model); the key is read from + * `CURSOR_API_KEY`. + * - Distributed as a standalone binary, NOT npm: `install` fetches the pinned + * version's tarball directly (the public install script only ever pulls + * latest) and symlinks `cursor-agent` onto the PATH. + * - `--force` auto-approves tool/command execution, `--approve-mcps` + * auto-approves MCP servers, `--trust` trusts the workspace (all required + * for a non-interactive run; the eval Docker sandbox is the isolation + * boundary). + * - `cursor-agent` has no system-prompt flag, so — like Codex — the system + * prompt is prepended to the task and the pair is fed on stdin (avoids the + * ARG_MAX / shell-expansion surface of a positional prompt). + */ + +import type { McpServerConfig } from "../../index.js"; +import { parseJsonlRecords } from "../../json.js"; +import type { AgentRunner } from "../types.js"; +import { + NPM_PREFIX, + npmGlobalBin, + processStopReason, + shellQuote, + writeSandboxFile, +} from "../shared.js"; + +/** Cursor model id — a free-form string (e.g. `composer-2.5`). */ +export type CursorCliModel = string & {}; + +/** Where the pinned binary is extracted (outside the scored workspace). */ +const CURSOR_INSTALL_DIR = '"$HOME/.cursor-cli"'; +/** Shell path to cursor-agent's MCP config, in `$HOME/.cursor` (outside the workspace). */ +const CURSOR_MCP_PATH = '"$HOME/.cursor/mcp.json"'; + +export const cursorCliRunner: AgentRunner = { + id: "cursor-cli", + displayName: "Cursor CLI", + apiKeyEnvVar: "CURSOR_API_KEY", + cliPackage: "cursor-cli", + // Pinned: cursor-agent's stream-json (SDK message) schema evolves; bump + // deliberately and re-check the parser. See ./parser.ts. + defaultCliVersion: "2026.05.01-eea359f", + defaultModel: "composer-2.5", + + async install(sandbox, version) { + // Not on npm: fetch the pinned version's tarball from Cursor's download + // host (same URL the install script uses, but version-pinned) and symlink + // the binary into the per-run npm bin dir, which is already on PATH. + const bin = npmGlobalBin("cursor-agent"); + const script = [ + "set -e", + 'ARCH=$(uname -m); case "$ARCH" in x86_64|amd64) ARCH=x64;; arm64|aarch64) ARCH=arm64;; esac', + 'OS=$(uname -s | tr "[:upper:]" "[:lower:]")', + `mkdir -p ${CURSOR_INSTALL_DIR} ${NPM_PREFIX}/bin`, + `curl -fsSL "https://downloads.cursor.com/lab/${version}/$OS/$ARCH/agent-cli-package.tar.gz" | tar --strip-components=1 -xzf - -C ${CURSOR_INSTALL_DIR}`, + `ln -sf ${CURSOR_INSTALL_DIR}/cursor-agent ${bin}`, + ].join("\n"); + const result = await sandbox.exec(script, { timeoutMs: 300_000 }); + if (!result.ok) { + throw new Error(`${this.displayName} install failed: ${result.stderr || result.stdout}`); + } + }, + + async exec({ sandbox, model, apiKey, systemPromptPath, userPromptPath, mcpServers, timeoutSec }) { + const agent = npmGlobalBin("cursor-agent"); + + if (Object.keys(mcpServers).length > 0) { + await sandbox.exec(`mkdir -p "$HOME/.cursor"`); + await writeSandboxFile(sandbox, CURSOR_MCP_PATH, buildCursorMcpConfig(mcpServers)); + } + + const flags = [ + "--print", + // Newline-delimited JSON (SDK message) events on stdout. + "--output-format stream-json", + `--model ${shellQuote(model)}`, + // The sandbox is the isolation boundary: trust the workspace, auto-approve + // commands and MCP servers so nothing blocks on a prompt. + "--trust", + "--force", + "--approve-mcps", + ].join(" "); + + // cursor-agent has no system-prompt flag; prepend the system prompt to the + // task, both staged as files, and feed the pair on stdin (cursor-agent + // reads a piped stdin prompt), mirroring Codex. + const command = await sandbox.exec( + `{ cat ${systemPromptPath}; printf '\\n\\n'; cat ${userPromptPath}; } | ${agent} ${flags}`, + { timeoutMs: timeoutSec * 1000, env: { CURSOR_API_KEY: apiKey } }, + ); + return { command, raw: command.stdout }; + }, + + deriveStopReason(raw, command) { + if (!raw) return processStopReason(command); + const { records } = parseJsonlRecords(raw); + // The terminal `result` event carries the run's status. + for (let i = records.length - 1; i >= 0; i -= 1) { + const record = records[i]; + if (record.type !== "result") continue; + if (record.is_error === true) return "error"; + const subtype = typeof record.subtype === "string" ? record.subtype : undefined; + const status = typeof record.status === "string" ? record.status : undefined; + const outcome = subtype ?? status; + if (outcome === "success") return "stop"; + if (outcome) return outcome; // e.g. "error" / "aborted" — surface verbatim + break; + } + // No terminal result (crash / kill / timeout) — derive from the process. + return processStopReason(command); + }, +}; + +/** + * cursor-agent's `~/.cursor/mcp.json` schema: `{ mcpServers: { name: { command, + * args, env } } }` — the same shape as the Cursor editor. The harness's + * `{command,args,env}` maps directly. + */ +export function buildCursorMcpConfig(servers: Record): string { + const mcpServers: Record = {}; + for (const [name, server] of Object.entries(servers)) { + mcpServers[name] = { + command: server.command, + ...(server.args ? { args: server.args } : {}), + ...(server.env ? { env: server.env } : {}), + }; + } + return JSON.stringify({ mcpServers }, null, 2); +} diff --git a/packages/core/src/agents/engine.ts b/packages/core/src/agents/engine.ts index edff83e8..a86ff2d0 100644 --- a/packages/core/src/agents/engine.ts +++ b/packages/core/src/agents/engine.ts @@ -39,6 +39,8 @@ function modelProviderForAgent(id: AgentRunner["id"]): ModelProvider { return "anthropic"; case "codex": return "openai"; + case "cursor-cli": + return "cursor"; case "ai-sdk": throw new Error("ai-sdk agents are not created through createCliAgent"); } diff --git a/packages/core/src/agents/registry.ts b/packages/core/src/agents/registry.ts index b6b8417f..00a0b92f 100644 --- a/packages/core/src/agents/registry.ts +++ b/packages/core/src/agents/registry.ts @@ -13,8 +13,13 @@ import type { AgentHarnessId } from "../eval-metadata.js"; import type { AgentTranscriptParser } from "../parsers/types.js"; import { claudeCodeDefinition } from "./claude-code/index.js"; import { codexDefinition } from "./codex/index.js"; +import { cursorCliDefinition } from "./cursor-cli/index.js"; -const AGENTS: AgentDefinition[] = [claudeCodeDefinition, codexDefinition]; +const AGENTS: AgentDefinition[] = [ + claudeCodeDefinition, + codexDefinition, + cursorCliDefinition, +]; const byId = new Map(AGENTS.map((agent) => [agent.runner.id, agent])); diff --git a/packages/core/src/eval-metadata.ts b/packages/core/src/eval-metadata.ts index 928bbe59..9e64552b 100644 --- a/packages/core/src/eval-metadata.ts +++ b/packages/core/src/eval-metadata.ts @@ -44,10 +44,19 @@ export const evalSuiteSchema = z.enum(["benchmark", "regression", "other"]); export const EVAL_SUITES = evalSuiteSchema.options; export type EvalSuite = z.infer; -export const agentHarnessIdSchema = z.enum(["ai-sdk", "claude-code", "codex"]); +export const agentHarnessIdSchema = z.enum([ + "ai-sdk", + "claude-code", + "codex", + "cursor-cli", +]); export type AgentHarnessId = z.infer; -export const modelProviderSchema = z.enum(["anthropic", "openai"]); +export const modelProviderSchema = z.enum([ + "anthropic", + "openai", + "cursor", +]); export type ModelProvider = z.infer; export const reasoningEffortSchema = z.enum([ diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index e5e3a447..5012c518 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -91,6 +91,7 @@ export { buildSkillResult } from "./skill-results.js"; export { createCliAgent } from "./agents/engine.js"; export { claudeCodeAgent } from "./agents/claude-code/index.js"; export { codexAgent } from "./agents/codex/index.js"; +export { cursorCliAgent } from "./agents/cursor-cli/index.js"; export type { AgentMetadata, AgentSandbox,