From b88b743f31c09560c1678fdcdca0940442b4772c Mon Sep 17 00:00:00 2001 From: Pedro Rodrigues Date: Wed, 24 Jun 2026 11:18:04 +0100 Subject: [PATCH 1/6] feat(eval): add OpenCode agent harness MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds `opencode` (OpenCode CLI) as an eval agent on the agents framework: an agents/opencode module (runner + parser + factory + registry definition) plus experiments. Orchestration is unchanged; opencode's transcript is parsed into the same surface scorers use. Runs in both modes, like Claude Code / Codex. - agents/opencode/runner.ts: install `opencode-ai`, then `opencode run --format json --dangerously-skip-permissions < /dev/null` (opencode blocks on stdin otherwise, even with the message as an arg). Multi-provider: model ids are `provider/model`, typed from the vendor SDKs — `anthropic/${Model}` | `openai/${ChatModel}` | `google/${GoogleGenerativeAIModelId}` | (string & {}). The runner is built per-model: `apiKeyEnvVar` and `modelProvider` are resolved from the provider prefix (anthropic→ANTHROPIC_API_KEY, openai→OPENAI_API_KEY, google→GOOGLE_GENERATIVE_AI_API_KEY — opencode's google provider reads exactly that), throwing on an unsupported provider. MCP is written to an OPENCODE_CONFIG file in scratch. Stop reason from the terminal step_finish reason (or an error event). - agents/opencode/parser.ts: the 1.15 JSONL schema (text / tool_use / reasoning / error / step_*). A tool_use record is self-contained → paired tool_call + tool_result by callID; OPENCODE_TOOLS map; normalized command/path/url via the shared extractArgs; loadedSkill from the `skill` tool or SKILL.md reads; epoch-ms → ISO. - agents/engine.ts + agents/types.ts: multi-provider CLIs carry an optional `modelProvider` on the runner; the engine prefers it over the per-agent-id mapping. `requireApiKey` now uses the shared `requireEnv` (agents/shared.ts) — node-native env validation with a clear " is not set / set but empty" error. - eval-metadata: `opencode` harness id; `google` model provider. Web app gains the OpenCode agent label and strips the `provider/` prefix when formatting model names. - Experiments: opencode-claude-sonnet-5, opencode-gpt-5.4-mini, opencode-gemini-flash. Adds `@ai-sdk/google` for the Gemini model-id types. .env.example gains GOOGLE_GENERATIVE_AI_API_KEY. Verified e2e (opencode 1.15.7) in the sandbox on tools-mode investigate-db-001-table-row-counts: opencode-claude-sonnet-5 3/3, opencode-gpt-5.4-mini 3/3, and opencode-gemini-flash 3/3 — all stoppedReason "stop", using the Supabase MCP. The Gemini direct-provider tool-loop issue observed on the original branch no longer reproduces. Co-Authored-By: Claude Opus 4.8 (1M context) Co-Authored-By: Claude Fable 5 --- .env.example | 5 +- apps/web/src/App.tsx | 6 +- experiments/opencode-claude-sonnet-5.ts | 22 ++ experiments/opencode-gemini-flash.ts | 24 +++ experiments/opencode-gpt-5.4-mini.ts | 21 ++ packages/core/package.json | 3 +- packages/core/src/agents/engine.ts | 21 +- packages/core/src/agents/opencode/index.ts | 42 ++++ .../core/src/agents/opencode/parser.test.ts | 161 ++++++++++++++ packages/core/src/agents/opencode/parser.ts | 203 ++++++++++++++++++ .../core/src/agents/opencode/runner.test.ts | 64 ++++++ packages/core/src/agents/opencode/runner.ts | 180 ++++++++++++++++ packages/core/src/agents/registry.ts | 7 +- packages/core/src/agents/shared.test.ts | 26 +++ packages/core/src/agents/shared.ts | 25 ++- packages/core/src/agents/types.ts | 6 + packages/core/src/eval-metadata.ts | 9 +- packages/core/src/index.ts | 3 +- pnpm-lock.yaml | 39 ++++ pnpm-workspace.yaml | 1 + 20 files changed, 849 insertions(+), 19 deletions(-) create mode 100644 experiments/opencode-claude-sonnet-5.ts create mode 100644 experiments/opencode-gemini-flash.ts create mode 100644 experiments/opencode-gpt-5.4-mini.ts create mode 100644 packages/core/src/agents/opencode/index.ts create mode 100644 packages/core/src/agents/opencode/parser.test.ts create mode 100644 packages/core/src/agents/opencode/parser.ts create mode 100644 packages/core/src/agents/opencode/runner.test.ts create mode 100644 packages/core/src/agents/opencode/runner.ts create mode 100644 packages/core/src/agents/shared.test.ts diff --git a/.env.example b/.env.example index a7cb97a9..66a7f228 100644 --- a/.env.example +++ b/.env.example @@ -2,8 +2,11 @@ ANTHROPIC_API_KEY= OPENAI_API_KEY= +# Opencode Gemini API key +GOOGLE_GENERATIVE_AI_API_KEY= + # Vercel AI Gateway — one key for every vendor. Direct keys above stay the # default; set RUN_THROUGH_GATEWAY=true (the eval-refresh workflow's # run_through_gateway input) to route the whole run through the gateway. AI_GATEWAY_API_KEY= -RUN_THROUGH_GATEWAY= \ No newline at end of file +RUN_THROUGH_GATEWAY= diff --git a/apps/web/src/App.tsx b/apps/web/src/App.tsx index 55680f33..c9b567ae 100644 --- a/apps/web/src/App.tsx +++ b/apps/web/src/App.tsx @@ -198,6 +198,7 @@ const AGENT_LABELS = { "ai-sdk": "AI SDK", "claude-code": "Claude Code", codex: "Codex", + opencode: "OpenCode", } satisfies Record const EXPERIMENT_SUITES = ["benchmark", "no-skills"] as const @@ -313,13 +314,16 @@ function formatOpenAiModel(modelId: string) { } function formatModel(display: ExperimentDisplay) { - // AI Gateway model ids are `vendor/model` slugs; format just the model part. + // opencode ids are `provider/model` and AI Gateway ids are `vendor/model` + // slugs; either way, format just the model part. const modelId = display.modelId.replace(/^[a-z-]+\//, "") switch (display.modelProvider) { case "anthropic": return formatAnthropicModel(modelId) case "openai": return formatOpenAiModel(modelId) + case "google": + return modelId } } diff --git a/experiments/opencode-claude-sonnet-5.ts b/experiments/opencode-claude-sonnet-5.ts new file mode 100644 index 00000000..6ac0379c --- /dev/null +++ b/experiments/opencode-claude-sonnet-5.ts @@ -0,0 +1,22 @@ +import { + defineExperiment, + opencodeAgent, + platformLiteRuntime, + supabaseMcpServer, +} from "@supabase-evals/core"; +import { localStackRuntime } from "@supabase-evals/sandbox"; + +// OpenCode is a CLI agent driving Claude Sonnet 5. Like Claude Code / Codex it +// runs in both modes: `runtime` supplies the MCP servers for tools-mode evals +// (written into opencode's config) and `localStack` drives local-stack evals. +// Which mode an eval uses is a property of the eval, not the agent. +export default defineExperiment({ + agent: opencodeAgent({ + model: "anthropic/claude-sonnet-5", + }), + runtime: platformLiteRuntime({ + mcpServers: [supabaseMcpServer()], + }), + localStack: localStackRuntime(), + skills: ["supabase", "supabase-postgres-best-practices"], +}); diff --git a/experiments/opencode-gemini-flash.ts b/experiments/opencode-gemini-flash.ts new file mode 100644 index 00000000..d1cd08e1 --- /dev/null +++ b/experiments/opencode-gemini-flash.ts @@ -0,0 +1,24 @@ +import { + defineExperiment, + opencodeAgent, + platformLiteRuntime, + supabaseMcpServer, +} from "@supabase-evals/core"; +import { localStackRuntime } from "@supabase-evals/sandbox"; + +// OpenCode driving Google's latest Gemini Flash (cheapest tier). Runs in both +// modes (see opencode-claude-sonnet-5.ts); the `google/` prefix selects the +// GOOGLE_GENERATIVE_AI_API_KEY credential (Google AI Studio, not Vertex). +// `gemini-flash-latest` tracks the newest Flash — the only Gemini Flash id that +// the AI-Studio key serves end-to-end (pinned 2.5/3.x-flash ids returned no +// output via opencode 1.15.7). +export default defineExperiment({ + agent: opencodeAgent({ + model: "google/gemini-flash-latest", + }), + runtime: platformLiteRuntime({ + mcpServers: [supabaseMcpServer()], + }), + localStack: localStackRuntime(), + skills: ["supabase", "supabase-postgres-best-practices"], +}); diff --git a/experiments/opencode-gpt-5.4-mini.ts b/experiments/opencode-gpt-5.4-mini.ts new file mode 100644 index 00000000..e7a671da --- /dev/null +++ b/experiments/opencode-gpt-5.4-mini.ts @@ -0,0 +1,21 @@ +import { + defineExperiment, + opencodeAgent, + platformLiteRuntime, + supabaseMcpServer, +} from "@supabase-evals/core"; +import { localStackRuntime } from "@supabase-evals/sandbox"; + +// OpenCode driving OpenAI GPT-5.4 mini. Runs in both modes (see opencode-claude- +// sonnet-5.ts); the `openai/` model prefix selects the OPENAI_API_KEY +// credential. +export default defineExperiment({ + agent: opencodeAgent({ + model: "openai/gpt-5.4-mini", + }), + runtime: platformLiteRuntime({ + mcpServers: [supabaseMcpServer()], + }), + localStack: localStackRuntime(), + skills: ["supabase", "supabase-postgres-best-practices"], +}); diff --git a/packages/core/package.json b/packages/core/package.json index 4cce8026..bf84b15f 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -18,7 +18,7 @@ }, "dependencies": { "@anthropic-ai/sdk": "catalog:", - "openai": "catalog:", + "@ai-sdk/google": "catalog:", "@ai-sdk/mcp": "catalog:", "@ai-sdk/openai": "catalog:", "@supabase-evals/platform-lite": "workspace:*", @@ -26,6 +26,7 @@ "ai": "catalog:", "executor": "1.4.29", "gray-matter": "^4.0.3", + "openai": "catalog:", "typescript": "catalog:", "zod": "catalog:" } diff --git a/packages/core/src/agents/engine.ts b/packages/core/src/agents/engine.ts index be081445..e774f431 100644 --- a/packages/core/src/agents/engine.ts +++ b/packages/core/src/agents/engine.ts @@ -34,6 +34,7 @@ import { SYSTEM_PROMPT_PATH, USER_PROMPT_PATH, processStopReason, + requireEnv, rewriteLoopback, writeSandboxFile, } from './shared.js'; @@ -44,6 +45,10 @@ function modelProviderForAgent(id: AgentRunner['id']): ModelProvider { return 'anthropic'; case 'codex': return 'openai'; + case 'opencode': + throw new Error( + 'opencode is multi-provider; its runner sets `modelProvider` from the model id' + ); case 'ai-sdk': throw new Error('ai-sdk agents are not created through createCliAgent'); } @@ -72,10 +77,11 @@ export function createCliAgent( metadata: { agent: runner.id, // Through the gateway the model may be any vendor's; derive the vendor - // from the model slug instead of from the agent. + // from the model slug instead of from the agent. On the direct path a + // multi-provider runner (e.g. opencode) sets its own `modelProvider`. modelProvider: useGateway ? gatewayModelProvider(options.model) - : modelProviderForAgent(runner.id), + : (runner.modelProvider ?? modelProviderForAgent(runner.id)), modelId: options.model, ...(options.reasoningEffort ? { reasoningEffort: options.reasoningEffort } @@ -134,11 +140,8 @@ export function createCliAgent( function requireApiKey(runner: AgentRunner, gateway = false): string { if (gateway) return requireGatewayApiKey(runner.displayName); - const apiKey = process.env[runner.apiKeyEnvVar]; - if (!apiKey) { - throw new Error( - `Missing ${runner.displayName} credentials. Set ${runner.apiKeyEnvVar} before running ${runner.id} evals.` - ); - } - return apiKey; + return requireEnv( + runner.apiKeyEnvVar, + `Set it to run ${runner.displayName} (${runner.id}) evals.` + ); } diff --git a/packages/core/src/agents/opencode/index.ts b/packages/core/src/agents/opencode/index.ts new file mode 100644 index 00000000..344e37e6 --- /dev/null +++ b/packages/core/src/agents/opencode/index.ts @@ -0,0 +1,42 @@ +/** + * OpenCode agent. Owns everything opencode-specific: it wires its own runner + + * parser into the public `opencodeAgent` factory (via the generic + * `createCliAgent` engine) and exports the registry definition the harness uses + * to parse opencode transcripts. Runs in both modes, like Claude Code / Codex. + */ + +import type { AgentHarness } from "../../index.js"; +import { createCliAgent } from "../engine.js"; +import type { AgentDefinition } from "../types.js"; +import { + DEFAULT_OPENCODE_MODEL, + createOpencodeRunner, + type OpenCodeModel, +} from "./runner.js"; +import { opencodeParser } from "./parser.js"; + +/** + * OpenCode as an `AgentHarness`. Multi-provider: the `provider/model` id selects + * the credential (anthropic / openai / google), so the runner is built per-model + * with the matching API-key env var and provider. + */ +export function opencodeAgent( + options: { + /** opencode model id, `provider/model` (e.g. `openai/gpt-5.4`). */ + model?: OpenCodeModel; + /** Override the pinned CLI version. */ + cliVersion?: string; + } = {}, +): AgentHarness { + const model = options.model ?? DEFAULT_OPENCODE_MODEL; + return createCliAgent(createOpencodeRunner(model), opencodeParser, { + model, + cliVersion: options.cliVersion, + }); +} + +/** Runner + parser pairing for the agent registry (id comes from `runner.id`). */ +export const opencodeDefinition: AgentDefinition = { + runner: createOpencodeRunner(DEFAULT_OPENCODE_MODEL), + parser: opencodeParser, +}; diff --git a/packages/core/src/agents/opencode/parser.test.ts b/packages/core/src/agents/opencode/parser.test.ts new file mode 100644 index 00000000..d3ebc751 --- /dev/null +++ b/packages/core/src/agents/opencode/parser.test.ts @@ -0,0 +1,161 @@ +import { describe, expect, it } from "vitest"; +import { opencodeParser } from "./parser.js"; +import { adaptTranscript } from "../../parsers/adapt.js"; + +/** A representative `opencode run --format json` stream (shapes from CLI 1.15.7). */ +const SESSION = [ + JSON.stringify({ type: "step_start", part: { type: "step-start" } }), + JSON.stringify({ + type: "reasoning", + timestamp: 1782295624200, + part: { type: "reasoning", text: "I should list the files." }, + }), + JSON.stringify({ + type: "text", + timestamp: 1782295624232, + part: { type: "text", text: "Listing files." }, + }), + JSON.stringify({ + type: "tool_use", + timestamp: 1782295624290, + part: { + type: "tool", + tool: "bash", + callID: "tool_1", + state: { + status: "completed", + input: { command: "ls -la", description: "List files" }, + output: "file1\nfile2", + metadata: { exit: 0 }, + }, + }, + }), + JSON.stringify({ + type: "tool_use", + timestamp: 1782295624300, + part: { + type: "tool", + tool: "write", + callID: "tool_2", + state: { + status: "completed", + input: { filePath: "/work/note.txt", content: "hi" }, + output: "written", + }, + }, + }), + JSON.stringify({ + type: "text", + timestamp: 1782295624400, + part: { type: "text", text: "Done." }, + }), + JSON.stringify({ + type: "step_finish", + part: { type: "step-finish", reason: "stop", tokens: { input: 3, output: 6 } }, + }), +].join("\n"); + +describe("opencodeParser", () => { + it("maps bash + write to canonical tool calls, paired with results by callID", () => { + const { events, errors } = opencodeParser.parseTranscript(SESSION); + expect(errors).toEqual([]); + + const calls = events.filter((e) => e.type === "tool_call"); + expect(calls.map((e) => e.tool?.name)).toEqual(["shell", "file_write"]); + expect(calls.map((e) => e.tool?.originalName)).toEqual(["bash", "write"]); + expect(calls.map((e) => e.tool?.id)).toEqual(["tool_1", "tool_2"]); + // Normalized views on the event; raw args untouched. + expect(calls[0].tool?.command).toBe("ls -la"); + expect(calls[1].tool?.path).toBe("/work/note.txt"); + + const results = events.filter((e) => e.type === "tool_result"); + expect(results.map((e) => e.tool?.id)).toEqual(["tool_1", "tool_2"]); + expect(results.every((e) => e.tool?.success === true)).toBe(true); + }); + + it("surfaces reasoning + the assistant report via the adapter", () => { + const events = opencodeParser.parseTranscript(SESSION).events; + expect(events.some((e) => e.type === "thinking" && e.content === "I should list the files.")).toBe(true); + + const adapted = adaptTranscript(events); + expect(adapted.agentReport).toBe("Done."); + expect(adapted.steps).toBe(2); // two assistant text turns + expect(adapted.toolCalls).toEqual([ + { + endpoint: "bash", + body: { command: "ls -la", description: "List files" }, + name: "shell", + command: "ls -la", + result: "file1\nfile2", + error: undefined, + ts: 1782295624290, // epoch ms preserved through toISO -> parseTs + }, + { + endpoint: "write", + body: { filePath: "/work/note.txt", content: "hi" }, + name: "file_write", + path: "/work/note.txt", + result: "written", + error: undefined, + ts: 1782295624300, + }, + ]); + }); + + it("surfaces skill loads from the skill tool and from SKILL.md reads", () => { + const stream = [ + JSON.stringify({ + type: "tool_use", + part: { + type: "tool", + tool: "skill", + callID: "s1", + state: { status: "completed", input: { name: "supabase" }, output: "# Supabase" }, + }, + }), + JSON.stringify({ + type: "tool_use", + part: { + type: "tool", + tool: "read", + callID: "s2", + state: { + status: "completed", + input: { filePath: ".claude/skills/supabase-postgres-best-practices/SKILL.md" }, + output: "# Postgres", + }, + }, + }), + ].join("\n"); + const adapted = adaptTranscript(opencodeParser.parseTranscript(stream).events); + expect(adapted.toolCalls.map((call) => call.loadedSkill)).toEqual([ + "supabase", + "supabase-postgres-best-practices", + ]); + }); + + it("marks a non-zero shell exit as failed (error surfaced via adapter)", () => { + const stream = JSON.stringify({ + type: "tool_use", + part: { + type: "tool", + tool: "bash", + callID: "c1", + state: { status: "completed", input: { command: "false" }, output: "nope", metadata: { exit: 1 } }, + }, + }); + const events = opencodeParser.parseTranscript(stream).events; + expect(events.find((e) => e.type === "tool_result")?.tool?.success).toBe(false); + const adapted = adaptTranscript(events); + expect(adapted.toolCalls[0].error).toBe("nope"); + expect(adapted.toolCalls[0].result).toBeUndefined(); + }); + + it("emits an error event and never throws on malformed lines", () => { + const { events, errors } = opencodeParser.parseTranscript( + "not json\n" + JSON.stringify({ type: "error", error: { message: "boom" } }), + ); + expect(events).toEqual([{ timestamp: undefined, type: "error", content: "boom", raw: { type: "error", error: { message: "boom" } } }]); + expect(errors.length).toBe(1); + }); +}); diff --git a/packages/core/src/agents/opencode/parser.ts b/packages/core/src/agents/opencode/parser.ts new file mode 100644 index 00000000..5a9e643a --- /dev/null +++ b/packages/core/src/agents/opencode/parser.ts @@ -0,0 +1,203 @@ +/** + * OpenCode transcript parser — for `opencode run --format json` (CLI ≥ 1.15). + * + * The stream is newline-delimited event records, each `{ type, timestamp, + * sessionID, part }`: + * {"type":"step_start","part":{"type":"step-start"}} + * {"type":"text","part":{"type":"text","text":"…"}} + * {"type":"tool_use","part":{"type":"tool","tool":"bash","callID":"…", + * "state":{"status":"completed","input":{…},"output":"…", + * "metadata":{"exit":0}}}} + * {"type":"reasoning","part":{"type":"reasoning","text":"…"}} + * {"type":"error","error":{"message":"…"}} + * {"type":"step_finish","part":{"type":"step-finish","reason":"stop","tokens":{…}}} + * + * A `tool_use` record is self-contained (input + output + status), so it yields + * a paired tool_call + tool_result correlated by `part.callID`. Step records + * carry token/finish info and produce no transcript event (the runner reads the + * terminal `step_finish` reason for the stop reason). + * + * Adapted from `@supabase/agent-evals` (packages/agent-eval/src/parsers). + */ + +import { isRecord, parseJsonlRecords } from "../../json.js"; +import type { ParsedTranscript, TranscriptEvent } from "../../transcript/types.js"; +import type { AgentTranscriptParser } from "../../parsers/types.js"; +import { normalizeToolName, type AgentToolMap } from "../../parsers/shared/normalize.js"; +import { + extractArgs, + extractLoadedSkillFromText, + type ArgFieldMap, + type ExtractedArgs, +} from "../../parsers/shared/extract.js"; + +/** + * opencode's tool names → canonical names. opencode uses lowercase built-in tool + * names. Owned here, not in shared. MCP tools arrive under their server name and + * fall through to `tool_use`. + */ +const OPENCODE_TOOLS: AgentToolMap = { + caseInsensitive: true, + tools: { + read: "file_read", + write: "file_write", + edit: "file_edit", + multiedit: "file_edit", + patch: "file_edit", + apply_patch: "file_edit", + bash: "shell", + shell: "shell", + webfetch: "web_fetch", + websearch: "web_search", + codesearch: "grep", + glob: "glob", + grep: "grep", + list: "list_dir", + ls: "list_dir", + task: "agent_task", + todowrite: "agent_task", + skill: "tool_use", + }, +}; + +/** + * opencode tool args → normalized fields. `bash` carries the command in + * `command`; file tools the path in `filePath` (or `path`); `webfetch` the URL + * in `url`. The shared extractor reads whichever keys this map names. + */ +const OPENCODE_ARG_FIELDS: ArgFieldMap = { + path: ["filePath", "file_path", "path"], + command: ["command"], + url: ["url"], +}; + +/** Epoch-ms (or pass-through ISO) → ISO string. */ +function toISO(value: unknown): string | undefined { + if (typeof value === "number") return new Date(value).toISOString(); + if (typeof value === "string") return value; + return undefined; +} + +function str(value: unknown): string | undefined { + return typeof value === "string" ? value : undefined; +} + +/** Whether a completed tool call succeeded: shell keys off its exit code. */ +function toolSuccess( + canonical: string, + status: string | undefined, + metadata: Record | undefined, +): boolean | undefined { + if (status === undefined) return undefined; + if (status !== "completed") return false; + if (canonical === "shell") { + const exit = metadata?.exit; + return typeof exit === "number" ? exit === 0 : true; + } + return true; +} + +function partToEvents( + type: string, + part: Record, + timestamp: string | undefined, + raw: unknown, +): TranscriptEvent[] { + switch (type) { + case "text": { + const text = str(part.text); + return text ? [{ timestamp, type: "message", role: "assistant", content: text, raw }] : []; + } + case "reasoning": { + const text = str(part.text); + return text ? [{ timestamp, type: "thinking", content: text, raw }] : []; + } + case "tool_use": { + const originalName = str(part.tool) ?? "unknown"; + const id = str(part.callID); + const state = isRecord(part.state) ? part.state : {}; + const args = isRecord(state.input) ? state.input : {}; + const status = str(state.status); + const metadata = isRecord(state.metadata) ? state.metadata : undefined; + const name = normalizeToolName(originalName, OPENCODE_TOOLS); + const normalized: ExtractedArgs = extractArgs(args, OPENCODE_ARG_FIELDS); + + const tool: NonNullable = { name, originalName, id, args }; + if (normalized.path) tool.path = normalized.path; + if (normalized.command) tool.command = normalized.command; + if (normalized.url) tool.url = normalized.url; + tool.loadedSkill = loadedSkillFromOpencodeCall(tool); + + const events: TranscriptEvent[] = [{ timestamp, type: "tool_call", tool, raw }]; + // The result is in the same record; emit it only once the call completed. + if (status && status !== "running" && status !== "pending") { + events.push({ + timestamp, + type: "tool_result", + tool: { + name, + originalName, + id, + result: state.output ?? (isRecord(state.error) ? state.error : undefined), + success: toolSuccess(name, status, metadata), + }, + raw: state, + }); + } + return events; + } + default: + return []; + } +} + +/** + * Identifies opencode skill loads. opencode's native `skill` tool carries the + * skill name in its args; skills read manually surface as `skills// + * SKILL.md` in a file path or shell command. + */ +function loadedSkillFromOpencodeCall( + tool: NonNullable, +): string | undefined { + if (tool.originalName.toLowerCase() === "skill") { + const name = tool.args?.name ?? tool.args?.skill; + if (typeof name === "string") return name; + } + if (tool.path) return extractLoadedSkillFromText(tool.path); + if (tool.command) return extractLoadedSkillFromText(tool.command); + return undefined; +} + +function recordToEvents(data: Record): TranscriptEvent[] { + const type = str(data.type); + if (!type) return []; + const timestamp = toISO(data.timestamp); + + if (type === "error") { + const error = isRecord(data.error) ? data.error : undefined; + const message = str(error?.message) ?? str(data.message) ?? JSON.stringify(data.error ?? data); + return [{ timestamp, type: "error", content: message, raw: data }]; + } + // step_start / step_finish carry no transcript content (tokens + finish reason + // only; the runner reads the terminal step_finish reason for the stop reason). + if (type === "step_start" || type === "step_finish") return []; + + const part = isRecord(data.part) ? data.part : undefined; + if (!part) return []; + return partToEvents(type, part, timestamp, data); +} + +export const opencodeParser: AgentTranscriptParser = { + parseTranscript(raw: string): ParsedTranscript { + const { records, errors } = parseJsonlRecords(raw); + const events: TranscriptEvent[] = []; + for (const record of records) { + try { + events.push(...recordToEvents(record)); + } catch (e) { + errors.push(e instanceof Error ? e.message : String(e)); + } + } + return { events, errors }; + }, +}; diff --git a/packages/core/src/agents/opencode/runner.test.ts b/packages/core/src/agents/opencode/runner.test.ts new file mode 100644 index 00000000..31c6a5f9 --- /dev/null +++ b/packages/core/src/agents/opencode/runner.test.ts @@ -0,0 +1,64 @@ +import { describe, expect, it } from "vitest"; +import { + buildOpencodeConfig, + createOpencodeRunner, + providerApiKeyEnv, +} from "./runner.js"; + +/** A run's terminal records: a mid-run `step_finish` (tool-calls) then the final one. */ +const SESSION = [ + JSON.stringify({ type: "step_finish", part: { type: "step-finish", reason: "tool-calls" } }), + JSON.stringify({ type: "text", part: { type: "text", text: "Done." } }), + JSON.stringify({ type: "step_finish", part: { type: "step-finish", reason: "stop" } }), +].join("\n"); + +describe("opencode runner", () => { + it("resolves the API-key env var from the model's provider prefix", () => { + expect(providerApiKeyEnv("anthropic/claude-sonnet-5")).toBe("ANTHROPIC_API_KEY"); + expect(providerApiKeyEnv("openai/gpt-5.4")).toBe("OPENAI_API_KEY"); + // opencode's google provider reads GOOGLE_GENERATIVE_AI_API_KEY, not GEMINI_API_KEY. + expect(providerApiKeyEnv("google/gemini-flash-latest")).toBe("GOOGLE_GENERATIVE_AI_API_KEY"); + }); + + it("throws a clear error for an unsupported provider", () => { + expect(() => providerApiKeyEnv("openrouter/some-model")).toThrowError( + /Unsupported opencode provider "openrouter".*Supported: anthropic, openai, google/, + ); + }); + + it("carries the provider on the runner for experiment display metadata", () => { + expect(createOpencodeRunner("openai/gpt-5.4").modelProvider).toBe("openai"); + expect(createOpencodeRunner("google/gemini-flash-latest").modelProvider).toBe("google"); + }); + + it("deriveStopReason reads the terminal step_finish reason", () => { + const runner = createOpencodeRunner("anthropic/claude-sonnet-5"); + const ok = { ok: true, exitCode: 0, stdout: "", stderr: "" }; + expect(runner.deriveStopReason!(SESSION, ok)).toBe("stop"); + // A non-stop terminal reason is surfaced verbatim. + const length = JSON.stringify({ type: "step_finish", part: { reason: "length" } }); + expect(runner.deriveStopReason!(length, ok)).toBe("length"); + // An error event wins regardless of exit code. + const errored = JSON.stringify({ type: "error", error: { message: "model overloaded" } }); + expect(runner.deriveStopReason!(errored, ok)).toBe("error"); + }); + + it("builds opencode's MCP config shape from harness server configs", () => { + const config = JSON.parse( + buildOpencodeConfig({ + supabase: { command: "npx", args: ["-y", "srv"], env: { TOKEN: "t" } }, + docs: { command: "docs-server" }, + }), + ); + expect(config.mcp).toEqual({ + supabase: { + type: "local", + command: ["npx", "-y", "srv"], + enabled: true, + environment: { TOKEN: "t" }, + }, + // No env → no `environment` key. + docs: { type: "local", command: ["docs-server"], enabled: true }, + }); + }); +}); diff --git a/packages/core/src/agents/opencode/runner.ts b/packages/core/src/agents/opencode/runner.ts new file mode 100644 index 00000000..4e82ffd1 --- /dev/null +++ b/packages/core/src/agents/opencode/runner.ts @@ -0,0 +1,180 @@ +/** + * OpenCode runner. Headless via `opencode run --format json` (the CLI + * streams newline-delimited event records to stdout; see ./parser.ts). + * + * Two things are opencode-specific: + * - It is **multi-provider**: model ids are `provider/model` (e.g. + * `anthropic/claude-sonnet-5`, `openai/gpt-5.4-mini`, `google/gemini-3.5-flash`) + * and the credential it reads depends on the provider — so the runner is + * built per-model, with `apiKeyEnvVar` and `modelProvider` resolved from the + * model id (see `createOpencodeRunner`). + * - `opencode run` blocks waiting on stdin even when the message is passed as + * an argument, so we redirect stdin from /dev/null. + * + * Like Claude Code / Codex it runs in both modes: tools mode just drops the + * Supabase CLI + local stack, and Supabase access goes through MCP (written to + * an OPENCODE_CONFIG file outside the scored workspace). + */ + +import type { Model as AnthropicModel } from "@anthropic-ai/sdk/resources/messages"; +import type { ChatModel as OpenAIModel } from "openai/resources/shared"; +import type { GoogleGenerativeAIProvider } from "@ai-sdk/google"; +import type { McpServerConfig } from "../../index.js"; +import type { ModelProvider } from "../../eval-metadata.js"; +import { isRecord, parseJsonlRecords } from "../../json.js"; +import type { AgentRunner } from "../types.js"; +import { + SCRATCH, + npmGlobalBin, + npmInstallGlobal, + processStopReason, + shellQuote, + writeSandboxFile, +} from "../shared.js"; + +/** Gemini model ids, extracted from the exported (callable) provider type. */ +type GeminiModel = Parameters[0]; + +/** + * opencode model id: `provider/model`, where the model name is the original + * vendor's id (opencode passes it straight through to that provider's SDK). The + * three supported providers are typed from their vendor packages; any other + * string is still accepted. + */ +export type OpenCodeModel = + | `anthropic/${AnthropicModel}` + | `openai/${OpenAIModel}` + | `google/${GeminiModel}` + | (string & {}); + +/** Model used when the caller doesn't pick one. */ +export const DEFAULT_OPENCODE_MODEL: OpenCodeModel = "anthropic/claude-sonnet-5"; + +/** + * Provider prefix (`provider/model`) → the env var holding its key. opencode and + * the harness both use this name; Google's is `GOOGLE_GENERATIVE_AI_API_KEY` + * (opencode's google provider reads exactly that — not `GEMINI_API_KEY`). + */ +const PROVIDER_API_KEY_ENV: Record = { + anthropic: "ANTHROPIC_API_KEY", + openai: "OPENAI_API_KEY", + google: "GOOGLE_GENERATIVE_AI_API_KEY", +}; + +/** The provider prefix of a `provider/model` id; throws if unsupported. */ +export function providerForModel(model: string): ModelProvider { + const provider = model.split("/")[0]; + if (!(provider in PROVIDER_API_KEY_ENV)) { + throw new Error( + `Unsupported opencode provider "${provider}" in model "${model}". ` + + `Supported: ${Object.keys(PROVIDER_API_KEY_ENV).join(", ")}.`, + ); + } + return provider as ModelProvider; +} + +/** The API-key env var for a given `provider/model` id; throws if unsupported. */ +export function providerApiKeyEnv(model: string): string { + return PROVIDER_API_KEY_ENV[providerForModel(model)]; +} + +/** + * Shell path to the MCP config, staged in scratch (outside the workspace). Used + * both as the write target and as the `OPENCODE_CONFIG` env value — the shell + * expands `$HOME` in either position. + */ +const OPENCODE_CONFIG_PATH = '"$HOME/.eval/opencode.json"'; + +/** + * Build an opencode runner bound to one model's provider. opencode is + * multi-provider, but a single run targets one model, so the runner resolves + * `apiKeyEnvVar` and `modelProvider` from the model id (the generic layer's + * `requireApiKey` reads `apiKeyEnvVar`, and `exec` injects that same key). + */ +export function createOpencodeRunner(model: OpenCodeModel): AgentRunner { + const modelProvider = providerForModel(model); + return { + id: "opencode", + displayName: "OpenCode", + apiKeyEnvVar: providerApiKeyEnv(model), + modelProvider, + cliPackage: "opencode-ai", + // Pinned: opencode's --format json event schema evolves; bump deliberately + // and re-check the parser. See ./parser.ts. + defaultCliVersion: "1.15.7", + defaultModel: DEFAULT_OPENCODE_MODEL, + + async install(sandbox, version) { + await npmInstallGlobal(sandbox, `${this.cliPackage}@${version}`, this.displayName); + }, + + async exec({ sandbox, model, apiKey, systemPromptPath, userPromptPath, mcpServers, timeoutSec }) { + const opencode = npmGlobalBin("opencode"); + + // opencode has no system-prompt flag, so prepend the system prompt to the + // task; both are staged files, joined via command substitution into the + // single message argument. + const message = `"$(cat ${systemPromptPath}; printf '\\n\\n'; cat ${userPromptPath})"`; + + let configPrefix = ""; + if (Object.keys(mcpServers).length > 0) { + await sandbox.exec(`mkdir -p ${SCRATCH}`); + await writeSandboxFile(sandbox, OPENCODE_CONFIG_PATH, buildOpencodeConfig(mcpServers)); + configPrefix = `OPENCODE_CONFIG=${OPENCODE_CONFIG_PATH} `; + } + + const flags = [ + "run", + message, + `--model ${shellQuote(model)}`, + // Newline-delimited JSON event records on stdout. + "--format json", + // The sandbox is the isolation boundary, so let opencode act freely. + "--dangerously-skip-permissions", + ].join(" "); + + // `< /dev/null`: opencode run blocks on stdin otherwise, even with the + // message passed as an argument. + const command = await sandbox.exec(`${configPrefix}${opencode} ${flags} < /dev/null`, { + timeoutMs: timeoutSec * 1000, + env: { [this.apiKeyEnvVar]: apiKey }, + }); + return { command, raw: command.stdout }; + }, + + deriveStopReason(raw, command) { + if (!raw) return processStopReason(command); + const { records } = parseJsonlRecords(raw); + // An error event means the run failed regardless of exit code. + if (records.some((r) => r.type === "error")) return "error"; + // The terminal `step_finish` carries the model's finish reason. + for (let i = records.length - 1; i >= 0; i -= 1) { + if (records[i].type !== "step_finish") continue; + const part = records[i].part; + const reason = isRecord(part) && typeof part.reason === "string" ? part.reason : undefined; + if (reason === "stop") return "stop"; + if (reason && reason !== "tool-calls") return reason; // e.g. length — surface verbatim + break; + } + return processStopReason(command); + }, + }; +} + +/** + * opencode's `OPENCODE_CONFIG` MCP schema: `{ mcp: { name: { type: "local", + * command: [...], environment } } }`. The harness's `{command,args,env}` maps + * onto a single `command` array plus `environment`. + */ +export function buildOpencodeConfig(servers: Record): string { + const mcp: Record = {}; + for (const [name, server] of Object.entries(servers)) { + mcp[name] = { + type: "local", + command: [server.command, ...(server.args ?? [])], + enabled: true, + ...(server.env ? { environment: server.env } : {}), + }; + } + return JSON.stringify({ $schema: "https://opencode.ai/config.json", mcp }, null, 2); +} diff --git a/packages/core/src/agents/registry.ts b/packages/core/src/agents/registry.ts index 6a91a0d5..58db31b8 100644 --- a/packages/core/src/agents/registry.ts +++ b/packages/core/src/agents/registry.ts @@ -13,8 +13,13 @@ import type { AgentHarnessId } from '../eval-metadata.js'; import type { AgentTranscriptParser } from '../parsers/types.js'; import { claudeCodeDefinition } from './claude-code/index.js'; import { codexDefinition } from './codex/index.js'; +import { opencodeDefinition } from './opencode/index.js'; -const AGENTS: AgentDefinition[] = [claudeCodeDefinition, codexDefinition]; +const AGENTS: AgentDefinition[] = [ + claudeCodeDefinition, + codexDefinition, + opencodeDefinition, +]; const byId = new Map(AGENTS.map((agent) => [agent.runner.id, agent])); diff --git a/packages/core/src/agents/shared.test.ts b/packages/core/src/agents/shared.test.ts new file mode 100644 index 00000000..3dc31f07 --- /dev/null +++ b/packages/core/src/agents/shared.test.ts @@ -0,0 +1,26 @@ +import { afterEach, describe, expect, it } from "vitest"; +import { requireEnv } from "./shared.js"; + +const VAR = "OPENCODE_TEST_ENV_VAR"; + +describe("requireEnv", () => { + afterEach(() => { + delete process.env[VAR]; + }); + + it("returns the value when set", () => { + process.env[VAR] = "secret"; + expect(requireEnv(VAR)).toBe("secret"); + }); + + it("throws a clear, variable-naming error when unset, including the hint", () => { + expect(() => requireEnv(VAR, "Set it to run X.")).toThrowError( + `Environment variable ${VAR} is not set. Set it to run X.`, + ); + }); + + it("distinguishes set-but-empty from unset", () => { + process.env[VAR] = " "; + expect(() => requireEnv(VAR)).toThrowError(`Environment variable ${VAR} is set but empty.`); + }); +}); diff --git a/packages/core/src/agents/shared.ts b/packages/core/src/agents/shared.ts index cf1058cb..219bdb73 100644 --- a/packages/core/src/agents/shared.ts +++ b/packages/core/src/agents/shared.ts @@ -1,12 +1,31 @@ /** - * Helpers shared across CLI runners: sandbox scratch paths, file staging, - * global npm install, loopback rewriting, and the default process-exit-based - * stop reason. + * Helpers shared across CLI runners: env-var validation, sandbox scratch paths, + * file staging, global npm install, loopback rewriting, and the default + * process-exit-based stop reason. */ import type { CommandResult, McpServerConfig } from '../index.js'; import type { AgentSandbox } from './types.js'; +/** + * Read a required environment variable, throwing a clear error that names the + * variable (and distinguishes unset from blank). Node-native — reads + * `process.env` directly, no dependency. Shared so every harness validates its + * key the same way and surfaces the same precise message. + */ +export function requireEnv(name: string, hint?: string): string { + // `in` distinguishes "never set" from "set but empty" for a clearer message. + const isSet = name in process.env; + const value = process.env[name]; + if (!isSet || value === undefined) { + throw new Error(`Environment variable ${name} is not set.${hint ? ` ${hint}` : ""}`); + } + if (value.trim() === "") { + throw new Error(`Environment variable ${name} is set but empty.${hint ? ` ${hint}` : ""}`); + } + return value; +} + /** Scratch dir + staged files, outside the workspace so they're never scored. */ export const SCRATCH = '"$HOME/.eval"'; export const SYSTEM_PROMPT_PATH = '"$HOME/.eval/system-prompt.txt"'; diff --git a/packages/core/src/agents/types.ts b/packages/core/src/agents/types.ts index 69ca0858..45f212bd 100644 --- a/packages/core/src/agents/types.ts +++ b/packages/core/src/agents/types.ts @@ -88,6 +88,12 @@ export interface AgentRunner { displayName: string; /** Env var holding the agent's API key (e.g. `ANTHROPIC_API_KEY`). */ apiKeyEnvVar: string; + /** + * Optional: the model's provider, for multi-provider CLIs whose runner is + * built per-model (e.g. opencode's `provider/model` ids). Single-provider + * agents omit it — the engine derives the provider from the agent id. + */ + modelProvider?: ModelProvider; /** npm package providing the CLI. */ cliPackage: string; /** Pinned CLI version — pinned so transcript-format drift can't silently break parsing. */ diff --git a/packages/core/src/eval-metadata.ts b/packages/core/src/eval-metadata.ts index 1a8d541c..7e9fe927 100644 --- a/packages/core/src/eval-metadata.ts +++ b/packages/core/src/eval-metadata.ts @@ -52,10 +52,15 @@ export const experimentSuiteSchema = z.enum([ export const EXPERIMENT_SUITES = experimentSuiteSchema.options; export type ExperimentSuite = z.infer; -export const agentHarnessIdSchema = z.enum(['ai-sdk', 'claude-code', 'codex']); +export const agentHarnessIdSchema = z.enum([ + 'ai-sdk', + 'claude-code', + 'codex', + 'opencode', +]); export type AgentHarnessId = z.infer; -export const modelProviderSchema = z.enum(['anthropic', 'openai']); +export const modelProviderSchema = z.enum(['anthropic', 'openai', 'google']); export type ModelProvider = z.infer; export const reasoningEffortSchema = z.enum([ diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index e881e0f2..732936d0 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -106,10 +106,11 @@ export { rehydrateTruncatedDocsResults, } from './docs-results.js'; export type { DocsResultSandbox } from './docs-results.js'; -// CLI agent harnesses (Claude Code, Codex, and the framework for adding more). +// CLI agent harnesses (Claude Code, Codex, OpenCode, and the framework for adding more). export { createCliAgent } from './agents/engine.js'; export { claudeCodeAgent } from './agents/claude-code/index.js'; export { codexAgent } from './agents/codex/index.js'; +export { opencodeAgent } from './agents/opencode/index.js'; // Vercel AI Gateway (opt-in alternative to per-vendor keys; see agents/gateway.ts). export { AI_GATEWAY, type GatewayModelId } from './agents/gateway.js'; export type { diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index f7b2f128..2fbfb54f 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -9,6 +9,9 @@ catalogs: '@ai-sdk/anthropic': specifier: ^3.0.71 version: 3.0.82 + '@ai-sdk/google': + specifier: ^3.0.83 + version: 3.0.100 '@ai-sdk/mcp': specifier: ^1.0.39 version: 1.0.46 @@ -245,6 +248,9 @@ importers: packages/core: dependencies: + '@ai-sdk/google': + specifier: 'catalog:' + version: 3.0.100(zod@4.4.3) '@ai-sdk/mcp': specifier: 'catalog:' version: 1.0.46(zod@4.4.3) @@ -368,6 +374,12 @@ packages: peerDependencies: zod: ^3.25.76 || ^4.1.8 + '@ai-sdk/google@3.0.100': + resolution: {integrity: sha512-gmsjuwk+1++/qCsIopfkg9d68nb6TfZiLNlkEOEDd86waawsR4B+FQ4j73p/r3NbVz1NWcnU4IqLz5uHa9Fu9g==} + engines: {node: '>=18'} + peerDependencies: + zod: ^3.25.76 || ^4.1.8 + '@ai-sdk/mcp@1.0.46': resolution: {integrity: sha512-owU0wAP87KzsTzr+2JE9sT9lpsCWKg8ZwHhce/KQmD9D/kbhe69sUZ+lsFF5MoGvV/ZzMujrhAvC1MgmufwMeQ==} engines: {node: '>=18'} @@ -386,10 +398,20 @@ packages: peerDependencies: zod: ^3.25.76 || ^4.1.8 + '@ai-sdk/provider-utils@4.0.40': + resolution: {integrity: sha512-OL5IrpUm9Y8Dwy+w/vvFwPotS6m52O9W0op2oXgXdCROMJIBalBI0oro6OIBYkPxvm5Xg02GSkoQN25RlR0bnw==} + engines: {node: '>=18'} + peerDependencies: + zod: ^3.25.76 || ^4.1.8 + '@ai-sdk/provider@3.0.10': resolution: {integrity: sha512-Q3BZ27qfpYqnCYGvE3vt+Qi6LGOF9R5Nmzn+9JoM1lCRsD9mYaIhfJLkSunN48nfGXJ6n+XNV0J/XVpqGQl7Dw==} engines: {node: '>=18'} + '@ai-sdk/provider@3.0.14': + resolution: {integrity: sha512-5X1k57JBJ4H7H1QjX7CnJYAB1I19r/trVZTMcSms7/kLNZ8RaU4Nt2agcwZzv82Hfx6Q7/TOLU7agAKeFfc8cA==} + engines: {node: '>=18'} + '@anthropic-ai/sdk@0.105.0': resolution: {integrity: sha512-sDyu+aM9cE6uZE+HgRjjHRb+qqb87GHZOx+8bE0YlWetdL1YcVLxn8h9ltxGOflyChTe6PMEo50kMQV4cw0hfg==} hasBin: true @@ -4501,6 +4523,12 @@ snapshots: '@vercel/oidc': 3.2.0 zod: 4.4.3 + '@ai-sdk/google@3.0.100(zod@4.4.3)': + dependencies: + '@ai-sdk/provider': 3.0.14 + '@ai-sdk/provider-utils': 4.0.40(zod@4.4.3) + zod: 4.4.3 + '@ai-sdk/mcp@1.0.46(zod@4.4.3)': dependencies: '@ai-sdk/provider': 3.0.10 @@ -4521,10 +4549,21 @@ snapshots: eventsource-parser: 3.1.0 zod: 4.4.3 + '@ai-sdk/provider-utils@4.0.40(zod@4.4.3)': + dependencies: + '@ai-sdk/provider': 3.0.14 + '@standard-schema/spec': 1.1.0 + eventsource-parser: 3.1.0 + zod: 4.4.3 + '@ai-sdk/provider@3.0.10': dependencies: json-schema: 0.4.0 + '@ai-sdk/provider@3.0.14': + dependencies: + json-schema: 0.4.0 + '@anthropic-ai/sdk@0.105.0(zod@4.4.3)': dependencies: json-schema-to-ts: 3.1.1 diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index 7b181ac5..bbfd992f 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -6,6 +6,7 @@ catalog: '@anthropic-ai/sdk': ^0.105.0 'openai': ^6.44.0 '@ai-sdk/anthropic': ^3.0.71 + '@ai-sdk/google': ^3.0.83 '@ai-sdk/mcp': ^1.0.39 '@ai-sdk/openai': ^3.0.66 '@electric-sql/pglite': 0.4.5 From 2922993fb8b8f6dd37e4fc76ba3e62964aa2de6f Mon Sep 17 00:00:00 2001 From: Pedro Rodrigues Date: Fri, 24 Jul 2026 01:40:26 +0100 Subject: [PATCH 2/6] feat(eval): add Moonshot (Kimi) provider to the opencode harness MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds `moonshotai` as a fourth opencode provider: `moonshotai/` ids resolve to the `MOONSHOT_API_KEY` credential (the env var opencode's moonshot provider reads), with `moonshotai` added to `modelProviderSchema` and the web app's model formatting. New opencode-kimi-k3 experiment. Provider resolution is unit-tested; the experiment is NOT yet verified end-to-end — the available Moonshot account is suspended (insufficient balance) and its key serves only kimi-k2.7-code / kimi-k2.6, not kimi-k3. opencode retries Moonshot quota errors silently, so a quota-dead key presents as an eval timeout, not an auth error. Co-Authored-By: Claude Fable 5 --- .env.example | 3 +++ apps/web/src/App.tsx | 1 + experiments/opencode-kimi-k3.ts | 21 +++++++++++++++++++ .../core/src/agents/opencode/runner.test.ts | 3 ++- packages/core/src/agents/opencode/runner.ts | 4 +++- packages/core/src/eval-metadata.ts | 7 ++++++- 6 files changed, 36 insertions(+), 3 deletions(-) create mode 100644 experiments/opencode-kimi-k3.ts diff --git a/.env.example b/.env.example index 66a7f228..6cf09540 100644 --- a/.env.example +++ b/.env.example @@ -5,6 +5,9 @@ OPENAI_API_KEY= # Opencode Gemini API key GOOGLE_GENERATIVE_AI_API_KEY= +# Opencode Moonshot (Kimi) API key +MOONSHOT_API_KEY= + # Vercel AI Gateway — one key for every vendor. Direct keys above stay the # default; set RUN_THROUGH_GATEWAY=true (the eval-refresh workflow's # run_through_gateway input) to route the whole run through the gateway. diff --git a/apps/web/src/App.tsx b/apps/web/src/App.tsx index c9b567ae..9cfb817c 100644 --- a/apps/web/src/App.tsx +++ b/apps/web/src/App.tsx @@ -323,6 +323,7 @@ function formatModel(display: ExperimentDisplay) { case "openai": return formatOpenAiModel(modelId) case "google": + case "moonshotai": return modelId } } diff --git a/experiments/opencode-kimi-k3.ts b/experiments/opencode-kimi-k3.ts new file mode 100644 index 00000000..ce8f92ad --- /dev/null +++ b/experiments/opencode-kimi-k3.ts @@ -0,0 +1,21 @@ +import { + defineExperiment, + opencodeAgent, + platformLiteRuntime, + supabaseMcpServer, +} from "@supabase-evals/core"; +import { localStackRuntime } from "@supabase-evals/sandbox"; + +// OpenCode driving Moonshot's Kimi K3. Runs in both modes (see +// opencode-claude-sonnet-5.ts); the `moonshotai/` prefix selects the +// MOONSHOT_API_KEY credential. +export default defineExperiment({ + agent: opencodeAgent({ + model: "moonshotai/kimi-k3", + }), + runtime: platformLiteRuntime({ + mcpServers: [supabaseMcpServer()], + }), + localStack: localStackRuntime(), + skills: ["supabase", "supabase-postgres-best-practices"], +}); diff --git a/packages/core/src/agents/opencode/runner.test.ts b/packages/core/src/agents/opencode/runner.test.ts index 31c6a5f9..31045096 100644 --- a/packages/core/src/agents/opencode/runner.test.ts +++ b/packages/core/src/agents/opencode/runner.test.ts @@ -18,11 +18,12 @@ describe("opencode runner", () => { expect(providerApiKeyEnv("openai/gpt-5.4")).toBe("OPENAI_API_KEY"); // opencode's google provider reads GOOGLE_GENERATIVE_AI_API_KEY, not GEMINI_API_KEY. expect(providerApiKeyEnv("google/gemini-flash-latest")).toBe("GOOGLE_GENERATIVE_AI_API_KEY"); + expect(providerApiKeyEnv("moonshotai/kimi-k3")).toBe("MOONSHOT_API_KEY"); }); it("throws a clear error for an unsupported provider", () => { expect(() => providerApiKeyEnv("openrouter/some-model")).toThrowError( - /Unsupported opencode provider "openrouter".*Supported: anthropic, openai, google/, + /Unsupported opencode provider "openrouter".*Supported: anthropic, openai, google, moonshotai/, ); }); diff --git a/packages/core/src/agents/opencode/runner.ts b/packages/core/src/agents/opencode/runner.ts index 4e82ffd1..989d61ce 100644 --- a/packages/core/src/agents/opencode/runner.ts +++ b/packages/core/src/agents/opencode/runner.ts @@ -53,12 +53,14 @@ export const DEFAULT_OPENCODE_MODEL: OpenCodeModel = "anthropic/claude-sonnet-5" /** * Provider prefix (`provider/model`) → the env var holding its key. opencode and * the harness both use this name; Google's is `GOOGLE_GENERATIVE_AI_API_KEY` - * (opencode's google provider reads exactly that — not `GEMINI_API_KEY`). + * (opencode's google provider reads exactly that — not `GEMINI_API_KEY`), and + * Moonshot's (`moonshotai/` ids, e.g. Kimi) is `MOONSHOT_API_KEY`. */ const PROVIDER_API_KEY_ENV: Record = { anthropic: "ANTHROPIC_API_KEY", openai: "OPENAI_API_KEY", google: "GOOGLE_GENERATIVE_AI_API_KEY", + moonshotai: "MOONSHOT_API_KEY", }; /** The provider prefix of a `provider/model` id; throws if unsupported. */ diff --git a/packages/core/src/eval-metadata.ts b/packages/core/src/eval-metadata.ts index 7e9fe927..159153e4 100644 --- a/packages/core/src/eval-metadata.ts +++ b/packages/core/src/eval-metadata.ts @@ -60,7 +60,12 @@ export const agentHarnessIdSchema = z.enum([ ]); export type AgentHarnessId = z.infer; -export const modelProviderSchema = z.enum(['anthropic', 'openai', 'google']); +export const modelProviderSchema = z.enum([ + 'anthropic', + 'openai', + 'google', + 'moonshotai', +]); export type ModelProvider = z.infer; export const reasoningEffortSchema = z.enum([ From 36927f2b21d1f98032d93b3873276e3a53b3fa35 Mon Sep 17 00:00:00 2001 From: Pedro Rodrigues Date: Fri, 24 Jul 2026 15:53:15 +0100 Subject: [PATCH 3/6] style(opencode): apply repo-wide Biome formatting The opencode harness and its experiments were authored before the repo-wide Biome pass (AI-831), which now sits in this branch's base after rebasing onto explore/ai-gateway-vendor-provider. Reformat those files (single quotes, wrapping) so the branch introduces no lint debt. No behavior change. Co-Authored-By: Claude Opus 4.8 (1M context) --- experiments/opencode-claude-sonnet-5.ts | 8 +- experiments/opencode-gemini-flash.ts | 8 +- experiments/opencode-gpt-5.4-mini.ts | 8 +- .../core/src/agents/opencode/parser.test.ts | 188 ++++++++++-------- packages/core/src/agents/opencode/parser.ts | 131 +++++++----- packages/core/src/agents/shared.test.ts | 28 +-- packages/core/src/agents/shared.ts | 10 +- 7 files changed, 223 insertions(+), 158 deletions(-) diff --git a/experiments/opencode-claude-sonnet-5.ts b/experiments/opencode-claude-sonnet-5.ts index 6ac0379c..89e9d10c 100644 --- a/experiments/opencode-claude-sonnet-5.ts +++ b/experiments/opencode-claude-sonnet-5.ts @@ -3,8 +3,8 @@ import { opencodeAgent, platformLiteRuntime, supabaseMcpServer, -} from "@supabase-evals/core"; -import { localStackRuntime } from "@supabase-evals/sandbox"; +} from '@supabase-evals/core'; +import { localStackRuntime } from '@supabase-evals/sandbox'; // OpenCode is a CLI agent driving Claude Sonnet 5. Like Claude Code / Codex it // runs in both modes: `runtime` supplies the MCP servers for tools-mode evals @@ -12,11 +12,11 @@ import { localStackRuntime } from "@supabase-evals/sandbox"; // Which mode an eval uses is a property of the eval, not the agent. export default defineExperiment({ agent: opencodeAgent({ - model: "anthropic/claude-sonnet-5", + model: 'anthropic/claude-sonnet-5', }), runtime: platformLiteRuntime({ mcpServers: [supabaseMcpServer()], }), localStack: localStackRuntime(), - skills: ["supabase", "supabase-postgres-best-practices"], + skills: ['supabase', 'supabase-postgres-best-practices'], }); diff --git a/experiments/opencode-gemini-flash.ts b/experiments/opencode-gemini-flash.ts index d1cd08e1..f2f68c8d 100644 --- a/experiments/opencode-gemini-flash.ts +++ b/experiments/opencode-gemini-flash.ts @@ -3,8 +3,8 @@ import { opencodeAgent, platformLiteRuntime, supabaseMcpServer, -} from "@supabase-evals/core"; -import { localStackRuntime } from "@supabase-evals/sandbox"; +} from '@supabase-evals/core'; +import { localStackRuntime } from '@supabase-evals/sandbox'; // OpenCode driving Google's latest Gemini Flash (cheapest tier). Runs in both // modes (see opencode-claude-sonnet-5.ts); the `google/` prefix selects the @@ -14,11 +14,11 @@ import { localStackRuntime } from "@supabase-evals/sandbox"; // output via opencode 1.15.7). export default defineExperiment({ agent: opencodeAgent({ - model: "google/gemini-flash-latest", + model: 'google/gemini-flash-latest', }), runtime: platformLiteRuntime({ mcpServers: [supabaseMcpServer()], }), localStack: localStackRuntime(), - skills: ["supabase", "supabase-postgres-best-practices"], + skills: ['supabase', 'supabase-postgres-best-practices'], }); diff --git a/experiments/opencode-gpt-5.4-mini.ts b/experiments/opencode-gpt-5.4-mini.ts index e7a671da..4937744c 100644 --- a/experiments/opencode-gpt-5.4-mini.ts +++ b/experiments/opencode-gpt-5.4-mini.ts @@ -3,19 +3,19 @@ import { opencodeAgent, platformLiteRuntime, supabaseMcpServer, -} from "@supabase-evals/core"; -import { localStackRuntime } from "@supabase-evals/sandbox"; +} from '@supabase-evals/core'; +import { localStackRuntime } from '@supabase-evals/sandbox'; // OpenCode driving OpenAI GPT-5.4 mini. Runs in both modes (see opencode-claude- // sonnet-5.ts); the `openai/` model prefix selects the OPENAI_API_KEY // credential. export default defineExperiment({ agent: opencodeAgent({ - model: "openai/gpt-5.4-mini", + model: 'openai/gpt-5.4-mini', }), runtime: platformLiteRuntime({ mcpServers: [supabaseMcpServer()], }), localStack: localStackRuntime(), - skills: ["supabase", "supabase-postgres-best-practices"], + skills: ['supabase', 'supabase-postgres-best-practices'], }); diff --git a/packages/core/src/agents/opencode/parser.test.ts b/packages/core/src/agents/opencode/parser.test.ts index d3ebc751..c2760ee8 100644 --- a/packages/core/src/agents/opencode/parser.test.ts +++ b/packages/core/src/agents/opencode/parser.test.ts @@ -1,161 +1,193 @@ -import { describe, expect, it } from "vitest"; -import { opencodeParser } from "./parser.js"; -import { adaptTranscript } from "../../parsers/adapt.js"; +import { describe, expect, it } from 'vitest'; +import { opencodeParser } from './parser.js'; +import { adaptTranscript } from '../../parsers/adapt.js'; /** A representative `opencode run --format json` stream (shapes from CLI 1.15.7). */ const SESSION = [ - JSON.stringify({ type: "step_start", part: { type: "step-start" } }), + JSON.stringify({ type: 'step_start', part: { type: 'step-start' } }), JSON.stringify({ - type: "reasoning", + type: 'reasoning', timestamp: 1782295624200, - part: { type: "reasoning", text: "I should list the files." }, + part: { type: 'reasoning', text: 'I should list the files.' }, }), JSON.stringify({ - type: "text", + type: 'text', timestamp: 1782295624232, - part: { type: "text", text: "Listing files." }, + part: { type: 'text', text: 'Listing files.' }, }), JSON.stringify({ - type: "tool_use", + type: 'tool_use', timestamp: 1782295624290, part: { - type: "tool", - tool: "bash", - callID: "tool_1", + type: 'tool', + tool: 'bash', + callID: 'tool_1', state: { - status: "completed", - input: { command: "ls -la", description: "List files" }, - output: "file1\nfile2", + status: 'completed', + input: { command: 'ls -la', description: 'List files' }, + output: 'file1\nfile2', metadata: { exit: 0 }, }, }, }), JSON.stringify({ - type: "tool_use", + type: 'tool_use', timestamp: 1782295624300, part: { - type: "tool", - tool: "write", - callID: "tool_2", + type: 'tool', + tool: 'write', + callID: 'tool_2', state: { - status: "completed", - input: { filePath: "/work/note.txt", content: "hi" }, - output: "written", + status: 'completed', + input: { filePath: '/work/note.txt', content: 'hi' }, + output: 'written', }, }, }), JSON.stringify({ - type: "text", + type: 'text', timestamp: 1782295624400, - part: { type: "text", text: "Done." }, + part: { type: 'text', text: 'Done.' }, }), JSON.stringify({ - type: "step_finish", - part: { type: "step-finish", reason: "stop", tokens: { input: 3, output: 6 } }, + type: 'step_finish', + part: { + type: 'step-finish', + reason: 'stop', + tokens: { input: 3, output: 6 }, + }, }), -].join("\n"); +].join('\n'); -describe("opencodeParser", () => { - it("maps bash + write to canonical tool calls, paired with results by callID", () => { +describe('opencodeParser', () => { + it('maps bash + write to canonical tool calls, paired with results by callID', () => { const { events, errors } = opencodeParser.parseTranscript(SESSION); expect(errors).toEqual([]); - const calls = events.filter((e) => e.type === "tool_call"); - expect(calls.map((e) => e.tool?.name)).toEqual(["shell", "file_write"]); - expect(calls.map((e) => e.tool?.originalName)).toEqual(["bash", "write"]); - expect(calls.map((e) => e.tool?.id)).toEqual(["tool_1", "tool_2"]); + const calls = events.filter((e) => e.type === 'tool_call'); + expect(calls.map((e) => e.tool?.name)).toEqual(['shell', 'file_write']); + expect(calls.map((e) => e.tool?.originalName)).toEqual(['bash', 'write']); + expect(calls.map((e) => e.tool?.id)).toEqual(['tool_1', 'tool_2']); // Normalized views on the event; raw args untouched. - expect(calls[0].tool?.command).toBe("ls -la"); - expect(calls[1].tool?.path).toBe("/work/note.txt"); + expect(calls[0].tool?.command).toBe('ls -la'); + expect(calls[1].tool?.path).toBe('/work/note.txt'); - const results = events.filter((e) => e.type === "tool_result"); - expect(results.map((e) => e.tool?.id)).toEqual(["tool_1", "tool_2"]); + const results = events.filter((e) => e.type === 'tool_result'); + expect(results.map((e) => e.tool?.id)).toEqual(['tool_1', 'tool_2']); expect(results.every((e) => e.tool?.success === true)).toBe(true); }); - it("surfaces reasoning + the assistant report via the adapter", () => { + it('surfaces reasoning + the assistant report via the adapter', () => { const events = opencodeParser.parseTranscript(SESSION).events; - expect(events.some((e) => e.type === "thinking" && e.content === "I should list the files.")).toBe(true); + expect( + events.some( + (e) => e.type === 'thinking' && e.content === 'I should list the files.' + ) + ).toBe(true); const adapted = adaptTranscript(events); - expect(adapted.agentReport).toBe("Done."); + expect(adapted.agentReport).toBe('Done.'); expect(adapted.steps).toBe(2); // two assistant text turns expect(adapted.toolCalls).toEqual([ { - endpoint: "bash", - body: { command: "ls -la", description: "List files" }, - name: "shell", - command: "ls -la", - result: "file1\nfile2", + endpoint: 'bash', + body: { command: 'ls -la', description: 'List files' }, + name: 'shell', + command: 'ls -la', + result: 'file1\nfile2', error: undefined, ts: 1782295624290, // epoch ms preserved through toISO -> parseTs }, { - endpoint: "write", - body: { filePath: "/work/note.txt", content: "hi" }, - name: "file_write", - path: "/work/note.txt", - result: "written", + endpoint: 'write', + body: { filePath: '/work/note.txt', content: 'hi' }, + name: 'file_write', + path: '/work/note.txt', + result: 'written', error: undefined, ts: 1782295624300, }, ]); }); - it("surfaces skill loads from the skill tool and from SKILL.md reads", () => { + it('surfaces skill loads from the skill tool and from SKILL.md reads', () => { const stream = [ JSON.stringify({ - type: "tool_use", + type: 'tool_use', part: { - type: "tool", - tool: "skill", - callID: "s1", - state: { status: "completed", input: { name: "supabase" }, output: "# Supabase" }, + type: 'tool', + tool: 'skill', + callID: 's1', + state: { + status: 'completed', + input: { name: 'supabase' }, + output: '# Supabase', + }, }, }), JSON.stringify({ - type: "tool_use", + type: 'tool_use', part: { - type: "tool", - tool: "read", - callID: "s2", + type: 'tool', + tool: 'read', + callID: 's2', state: { - status: "completed", - input: { filePath: ".claude/skills/supabase-postgres-best-practices/SKILL.md" }, - output: "# Postgres", + status: 'completed', + input: { + filePath: + '.claude/skills/supabase-postgres-best-practices/SKILL.md', + }, + output: '# Postgres', }, }, }), - ].join("\n"); - const adapted = adaptTranscript(opencodeParser.parseTranscript(stream).events); + ].join('\n'); + const adapted = adaptTranscript( + opencodeParser.parseTranscript(stream).events + ); expect(adapted.toolCalls.map((call) => call.loadedSkill)).toEqual([ - "supabase", - "supabase-postgres-best-practices", + 'supabase', + 'supabase-postgres-best-practices', ]); }); - it("marks a non-zero shell exit as failed (error surfaced via adapter)", () => { + it('marks a non-zero shell exit as failed (error surfaced via adapter)', () => { const stream = JSON.stringify({ - type: "tool_use", + type: 'tool_use', part: { - type: "tool", - tool: "bash", - callID: "c1", - state: { status: "completed", input: { command: "false" }, output: "nope", metadata: { exit: 1 } }, + type: 'tool', + tool: 'bash', + callID: 'c1', + state: { + status: 'completed', + input: { command: 'false' }, + output: 'nope', + metadata: { exit: 1 }, + }, }, }); const events = opencodeParser.parseTranscript(stream).events; - expect(events.find((e) => e.type === "tool_result")?.tool?.success).toBe(false); + expect(events.find((e) => e.type === 'tool_result')?.tool?.success).toBe( + false + ); const adapted = adaptTranscript(events); - expect(adapted.toolCalls[0].error).toBe("nope"); + expect(adapted.toolCalls[0].error).toBe('nope'); expect(adapted.toolCalls[0].result).toBeUndefined(); }); - it("emits an error event and never throws on malformed lines", () => { + it('emits an error event and never throws on malformed lines', () => { const { events, errors } = opencodeParser.parseTranscript( - "not json\n" + JSON.stringify({ type: "error", error: { message: "boom" } }), + 'not json\n' + + JSON.stringify({ type: 'error', error: { message: 'boom' } }) ); - expect(events).toEqual([{ timestamp: undefined, type: "error", content: "boom", raw: { type: "error", error: { message: "boom" } } }]); + expect(events).toEqual([ + { + timestamp: undefined, + type: 'error', + content: 'boom', + raw: { type: 'error', error: { message: 'boom' } }, + }, + ]); expect(errors.length).toBe(1); }); }); diff --git a/packages/core/src/agents/opencode/parser.ts b/packages/core/src/agents/opencode/parser.ts index 5a9e643a..628d8120 100644 --- a/packages/core/src/agents/opencode/parser.ts +++ b/packages/core/src/agents/opencode/parser.ts @@ -20,16 +20,22 @@ * Adapted from `@supabase/agent-evals` (packages/agent-eval/src/parsers). */ -import { isRecord, parseJsonlRecords } from "../../json.js"; -import type { ParsedTranscript, TranscriptEvent } from "../../transcript/types.js"; -import type { AgentTranscriptParser } from "../../parsers/types.js"; -import { normalizeToolName, type AgentToolMap } from "../../parsers/shared/normalize.js"; +import { isRecord, parseJsonlRecords } from '../../json.js'; +import type { + ParsedTranscript, + TranscriptEvent, +} from '../../transcript/types.js'; +import type { AgentTranscriptParser } from '../../parsers/types.js'; +import { + normalizeToolName, + type AgentToolMap, +} from '../../parsers/shared/normalize.js'; import { extractArgs, extractLoadedSkillFromText, type ArgFieldMap, type ExtractedArgs, -} from "../../parsers/shared/extract.js"; +} from '../../parsers/shared/extract.js'; /** * opencode's tool names → canonical names. opencode uses lowercase built-in tool @@ -39,24 +45,24 @@ import { const OPENCODE_TOOLS: AgentToolMap = { caseInsensitive: true, tools: { - read: "file_read", - write: "file_write", - edit: "file_edit", - multiedit: "file_edit", - patch: "file_edit", - apply_patch: "file_edit", - bash: "shell", - shell: "shell", - webfetch: "web_fetch", - websearch: "web_search", - codesearch: "grep", - glob: "glob", - grep: "grep", - list: "list_dir", - ls: "list_dir", - task: "agent_task", - todowrite: "agent_task", - skill: "tool_use", + read: 'file_read', + write: 'file_write', + edit: 'file_edit', + multiedit: 'file_edit', + patch: 'file_edit', + apply_patch: 'file_edit', + bash: 'shell', + shell: 'shell', + webfetch: 'web_fetch', + websearch: 'web_search', + codesearch: 'grep', + glob: 'glob', + grep: 'grep', + list: 'list_dir', + ls: 'list_dir', + task: 'agent_task', + todowrite: 'agent_task', + skill: 'tool_use', }, }; @@ -66,33 +72,33 @@ const OPENCODE_TOOLS: AgentToolMap = { * in `url`. The shared extractor reads whichever keys this map names. */ const OPENCODE_ARG_FIELDS: ArgFieldMap = { - path: ["filePath", "file_path", "path"], - command: ["command"], - url: ["url"], + path: ['filePath', 'file_path', 'path'], + command: ['command'], + url: ['url'], }; /** Epoch-ms (or pass-through ISO) → ISO string. */ function toISO(value: unknown): string | undefined { - if (typeof value === "number") return new Date(value).toISOString(); - if (typeof value === "string") return value; + if (typeof value === 'number') return new Date(value).toISOString(); + if (typeof value === 'string') return value; return undefined; } function str(value: unknown): string | undefined { - return typeof value === "string" ? value : undefined; + return typeof value === 'string' ? value : undefined; } /** Whether a completed tool call succeeded: shell keys off its exit code. */ function toolSuccess( canonical: string, status: string | undefined, - metadata: Record | undefined, + metadata: Record | undefined ): boolean | undefined { if (status === undefined) return undefined; - if (status !== "completed") return false; - if (canonical === "shell") { + if (status !== 'completed') return false; + if (canonical === 'shell') { const exit = metadata?.exit; - return typeof exit === "number" ? exit === 0 : true; + return typeof exit === 'number' ? exit === 0 : true; } return true; } @@ -101,19 +107,29 @@ function partToEvents( type: string, part: Record, timestamp: string | undefined, - raw: unknown, + raw: unknown ): TranscriptEvent[] { switch (type) { - case "text": { + case 'text': { const text = str(part.text); - return text ? [{ timestamp, type: "message", role: "assistant", content: text, raw }] : []; + return text + ? [ + { + timestamp, + type: 'message', + role: 'assistant', + content: text, + raw, + }, + ] + : []; } - case "reasoning": { + case 'reasoning': { const text = str(part.text); - return text ? [{ timestamp, type: "thinking", content: text, raw }] : []; + return text ? [{ timestamp, type: 'thinking', content: text, raw }] : []; } - case "tool_use": { - const originalName = str(part.tool) ?? "unknown"; + case 'tool_use': { + const originalName = str(part.tool) ?? 'unknown'; const id = str(part.callID); const state = isRecord(part.state) ? part.state : {}; const args = isRecord(state.input) ? state.input : {}; @@ -122,23 +138,31 @@ function partToEvents( const name = normalizeToolName(originalName, OPENCODE_TOOLS); const normalized: ExtractedArgs = extractArgs(args, OPENCODE_ARG_FIELDS); - const tool: NonNullable = { name, originalName, id, args }; + const tool: NonNullable = { + name, + originalName, + id, + args, + }; if (normalized.path) tool.path = normalized.path; if (normalized.command) tool.command = normalized.command; if (normalized.url) tool.url = normalized.url; tool.loadedSkill = loadedSkillFromOpencodeCall(tool); - const events: TranscriptEvent[] = [{ timestamp, type: "tool_call", tool, raw }]; + const events: TranscriptEvent[] = [ + { timestamp, type: 'tool_call', tool, raw }, + ]; // The result is in the same record; emit it only once the call completed. - if (status && status !== "running" && status !== "pending") { + if (status && status !== 'running' && status !== 'pending') { events.push({ timestamp, - type: "tool_result", + type: 'tool_result', tool: { name, originalName, id, - result: state.output ?? (isRecord(state.error) ? state.error : undefined), + result: + state.output ?? (isRecord(state.error) ? state.error : undefined), success: toolSuccess(name, status, metadata), }, raw: state, @@ -157,11 +181,11 @@ function partToEvents( * SKILL.md` in a file path or shell command. */ function loadedSkillFromOpencodeCall( - tool: NonNullable, + tool: NonNullable ): string | undefined { - if (tool.originalName.toLowerCase() === "skill") { + if (tool.originalName.toLowerCase() === 'skill') { const name = tool.args?.name ?? tool.args?.skill; - if (typeof name === "string") return name; + if (typeof name === 'string') return name; } if (tool.path) return extractLoadedSkillFromText(tool.path); if (tool.command) return extractLoadedSkillFromText(tool.command); @@ -173,14 +197,17 @@ function recordToEvents(data: Record): TranscriptEvent[] { if (!type) return []; const timestamp = toISO(data.timestamp); - if (type === "error") { + if (type === 'error') { const error = isRecord(data.error) ? data.error : undefined; - const message = str(error?.message) ?? str(data.message) ?? JSON.stringify(data.error ?? data); - return [{ timestamp, type: "error", content: message, raw: data }]; + const message = + str(error?.message) ?? + str(data.message) ?? + JSON.stringify(data.error ?? data); + return [{ timestamp, type: 'error', content: message, raw: data }]; } // step_start / step_finish carry no transcript content (tokens + finish reason // only; the runner reads the terminal step_finish reason for the stop reason). - if (type === "step_start" || type === "step_finish") return []; + if (type === 'step_start' || type === 'step_finish') return []; const part = isRecord(data.part) ? data.part : undefined; if (!part) return []; diff --git a/packages/core/src/agents/shared.test.ts b/packages/core/src/agents/shared.test.ts index 3dc31f07..06ab423b 100644 --- a/packages/core/src/agents/shared.test.ts +++ b/packages/core/src/agents/shared.test.ts @@ -1,26 +1,28 @@ -import { afterEach, describe, expect, it } from "vitest"; -import { requireEnv } from "./shared.js"; +import { afterEach, describe, expect, it } from 'vitest'; +import { requireEnv } from './shared.js'; -const VAR = "OPENCODE_TEST_ENV_VAR"; +const VAR = 'OPENCODE_TEST_ENV_VAR'; -describe("requireEnv", () => { +describe('requireEnv', () => { afterEach(() => { delete process.env[VAR]; }); - it("returns the value when set", () => { - process.env[VAR] = "secret"; - expect(requireEnv(VAR)).toBe("secret"); + it('returns the value when set', () => { + process.env[VAR] = 'secret'; + expect(requireEnv(VAR)).toBe('secret'); }); - it("throws a clear, variable-naming error when unset, including the hint", () => { - expect(() => requireEnv(VAR, "Set it to run X.")).toThrowError( - `Environment variable ${VAR} is not set. Set it to run X.`, + it('throws a clear, variable-naming error when unset, including the hint', () => { + expect(() => requireEnv(VAR, 'Set it to run X.')).toThrowError( + `Environment variable ${VAR} is not set. Set it to run X.` ); }); - it("distinguishes set-but-empty from unset", () => { - process.env[VAR] = " "; - expect(() => requireEnv(VAR)).toThrowError(`Environment variable ${VAR} is set but empty.`); + it('distinguishes set-but-empty from unset', () => { + process.env[VAR] = ' '; + expect(() => requireEnv(VAR)).toThrowError( + `Environment variable ${VAR} is set but empty.` + ); }); }); diff --git a/packages/core/src/agents/shared.ts b/packages/core/src/agents/shared.ts index 219bdb73..63d918d9 100644 --- a/packages/core/src/agents/shared.ts +++ b/packages/core/src/agents/shared.ts @@ -18,10 +18,14 @@ export function requireEnv(name: string, hint?: string): string { const isSet = name in process.env; const value = process.env[name]; if (!isSet || value === undefined) { - throw new Error(`Environment variable ${name} is not set.${hint ? ` ${hint}` : ""}`); + throw new Error( + `Environment variable ${name} is not set.${hint ? ` ${hint}` : ''}` + ); } - if (value.trim() === "") { - throw new Error(`Environment variable ${name} is set but empty.${hint ? ` ${hint}` : ""}`); + if (value.trim() === '') { + throw new Error( + `Environment variable ${name} is set but empty.${hint ? ` ${hint}` : ''}` + ); } return value; } From a19a53a3da35f926aa9b7874ce628171894bf173 Mon Sep 17 00:00:00 2001 From: Pedro Rodrigues Date: Fri, 24 Jul 2026 15:53:29 +0100 Subject: [PATCH 4/6] feat(eval): route the opencode Kimi K3 experiment through the AI Gateway MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The opencode harness was added after the Vercel AI Gateway work, so it was the one harness without a gateway route. Wire it up the same opt-in way as the others: opencodeAgent gains a `gateway` flag (the engine already threads `useGateway` into the runner), and in gateway mode the runner writes a custom OpenAI-compatible provider into OPENCODE_CONFIG pointed at the gateway's /v1 surface and addresses the model under it (`vercel-ai-gateway//`, keeping the gateway slug intact). The gateway key rides in the provider config so no vendor key env var is set for the run. opencode-kimi-k3 now runs `moonshotai/kimi-k3` through the gateway (`gateway: true`) — the model the direct Moonshot account couldn't serve. The gateway catalog confirms the slug is available; the run needs AI_GATEWAY_API_KEY instead of MOONSHOT_API_KEY. Unit tests cover both the config provider block and exec routing (model flag + env) for gateway vs direct. Not yet verified end-to-end (no local gateway key); CI holds the secret. Co-Authored-By: Claude Opus 4.8 (1M context) --- experiments/opencode-kimi-k3.ts | 17 +- packages/core/src/agents/opencode/index.ts | 21 ++- .../core/src/agents/opencode/runner.test.ts | 173 ++++++++++++++---- packages/core/src/agents/opencode/runner.ts | 173 +++++++++++++----- 4 files changed, 289 insertions(+), 95 deletions(-) diff --git a/experiments/opencode-kimi-k3.ts b/experiments/opencode-kimi-k3.ts index ce8f92ad..f8354ec4 100644 --- a/experiments/opencode-kimi-k3.ts +++ b/experiments/opencode-kimi-k3.ts @@ -3,19 +3,22 @@ import { opencodeAgent, platformLiteRuntime, supabaseMcpServer, -} from "@supabase-evals/core"; -import { localStackRuntime } from "@supabase-evals/sandbox"; +} from '@supabase-evals/core'; +import { localStackRuntime } from '@supabase-evals/sandbox'; -// OpenCode driving Moonshot's Kimi K3. Runs in both modes (see -// opencode-claude-sonnet-5.ts); the `moonshotai/` prefix selects the -// MOONSHOT_API_KEY credential. +// OpenCode driving Moonshot's Kimi K3 through the Vercel AI Gateway. `gateway: +// true` writes a gateway provider into opencode's config (see agents/opencode/ +// runner.ts) and routes on the `moonshotai/kimi-k3` catalog slug, so the run +// needs AI_GATEWAY_API_KEY rather than a direct MOONSHOT_API_KEY. Runs in both +// modes like the other opencode experiments (see opencode-claude-sonnet-5.ts). export default defineExperiment({ agent: opencodeAgent({ - model: "moonshotai/kimi-k3", + model: 'moonshotai/kimi-k3', + gateway: true, }), runtime: platformLiteRuntime({ mcpServers: [supabaseMcpServer()], }), localStack: localStackRuntime(), - skills: ["supabase", "supabase-postgres-best-practices"], + skills: ['supabase', 'supabase-postgres-best-practices'], }); diff --git a/packages/core/src/agents/opencode/index.ts b/packages/core/src/agents/opencode/index.ts index 344e37e6..562b6a7b 100644 --- a/packages/core/src/agents/opencode/index.ts +++ b/packages/core/src/agents/opencode/index.ts @@ -5,15 +5,15 @@ * to parse opencode transcripts. Runs in both modes, like Claude Code / Codex. */ -import type { AgentHarness } from "../../index.js"; -import { createCliAgent } from "../engine.js"; -import type { AgentDefinition } from "../types.js"; +import type { AgentHarness } from '../../index.js'; +import { createCliAgent } from '../engine.js'; +import type { AgentDefinition } from '../types.js'; import { DEFAULT_OPENCODE_MODEL, createOpencodeRunner, type OpenCodeModel, -} from "./runner.js"; -import { opencodeParser } from "./parser.js"; +} from './runner.js'; +import { opencodeParser } from './parser.js'; /** * OpenCode as an `AgentHarness`. Multi-provider: the `provider/model` id selects @@ -22,16 +22,23 @@ import { opencodeParser } from "./parser.js"; */ export function opencodeAgent( options: { - /** opencode model id, `provider/model` (e.g. `openai/gpt-5.4`). */ + /** + * opencode model id, `provider/model` (e.g. `openai/gpt-5.4`). With + * `gateway`, this is the AI Gateway `vendor/model` slug (e.g. + * `moonshotai/kimi-k3`) — see `./runner.ts`. + */ model?: OpenCodeModel; /** Override the pinned CLI version. */ cliVersion?: string; - } = {}, + /** Route through the Vercel AI Gateway instead of the vendor's own key. */ + gateway?: boolean; + } = {} ): AgentHarness { const model = options.model ?? DEFAULT_OPENCODE_MODEL; return createCliAgent(createOpencodeRunner(model), opencodeParser, { model, cliVersion: options.cliVersion, + gateway: options.gateway, }); } diff --git a/packages/core/src/agents/opencode/runner.test.ts b/packages/core/src/agents/opencode/runner.test.ts index 31045096..37f27365 100644 --- a/packages/core/src/agents/opencode/runner.test.ts +++ b/packages/core/src/agents/opencode/runner.test.ts @@ -1,65 +1,174 @@ -import { describe, expect, it } from "vitest"; +import { describe, expect, it } from 'vitest'; +import type { CommandResult } from '../../index.js'; +import { AI_GATEWAY } from '../gateway.js'; import { buildOpencodeConfig, createOpencodeRunner, providerApiKeyEnv, -} from "./runner.js"; +} from './runner.js'; /** A run's terminal records: a mid-run `step_finish` (tool-calls) then the final one. */ const SESSION = [ - JSON.stringify({ type: "step_finish", part: { type: "step-finish", reason: "tool-calls" } }), - JSON.stringify({ type: "text", part: { type: "text", text: "Done." } }), - JSON.stringify({ type: "step_finish", part: { type: "step-finish", reason: "stop" } }), -].join("\n"); + JSON.stringify({ + type: 'step_finish', + part: { type: 'step-finish', reason: 'tool-calls' }, + }), + JSON.stringify({ type: 'text', part: { type: 'text', text: 'Done.' } }), + JSON.stringify({ + type: 'step_finish', + part: { type: 'step-finish', reason: 'stop' }, + }), +].join('\n'); -describe("opencode runner", () => { +describe('opencode runner', () => { it("resolves the API-key env var from the model's provider prefix", () => { - expect(providerApiKeyEnv("anthropic/claude-sonnet-5")).toBe("ANTHROPIC_API_KEY"); - expect(providerApiKeyEnv("openai/gpt-5.4")).toBe("OPENAI_API_KEY"); + expect(providerApiKeyEnv('anthropic/claude-sonnet-5')).toBe( + 'ANTHROPIC_API_KEY' + ); + expect(providerApiKeyEnv('openai/gpt-5.4')).toBe('OPENAI_API_KEY'); // opencode's google provider reads GOOGLE_GENERATIVE_AI_API_KEY, not GEMINI_API_KEY. - expect(providerApiKeyEnv("google/gemini-flash-latest")).toBe("GOOGLE_GENERATIVE_AI_API_KEY"); - expect(providerApiKeyEnv("moonshotai/kimi-k3")).toBe("MOONSHOT_API_KEY"); + expect(providerApiKeyEnv('google/gemini-flash-latest')).toBe( + 'GOOGLE_GENERATIVE_AI_API_KEY' + ); + expect(providerApiKeyEnv('moonshotai/kimi-k3')).toBe('MOONSHOT_API_KEY'); }); - it("throws a clear error for an unsupported provider", () => { - expect(() => providerApiKeyEnv("openrouter/some-model")).toThrowError( - /Unsupported opencode provider "openrouter".*Supported: anthropic, openai, google, moonshotai/, + it('throws a clear error for an unsupported provider', () => { + expect(() => providerApiKeyEnv('openrouter/some-model')).toThrowError( + /Unsupported opencode provider "openrouter".*Supported: anthropic, openai, google, moonshotai/ ); }); - it("carries the provider on the runner for experiment display metadata", () => { - expect(createOpencodeRunner("openai/gpt-5.4").modelProvider).toBe("openai"); - expect(createOpencodeRunner("google/gemini-flash-latest").modelProvider).toBe("google"); + it('carries the provider on the runner for experiment display metadata', () => { + expect(createOpencodeRunner('openai/gpt-5.4').modelProvider).toBe('openai'); + expect( + createOpencodeRunner('google/gemini-flash-latest').modelProvider + ).toBe('google'); }); - it("deriveStopReason reads the terminal step_finish reason", () => { - const runner = createOpencodeRunner("anthropic/claude-sonnet-5"); - const ok = { ok: true, exitCode: 0, stdout: "", stderr: "" }; - expect(runner.deriveStopReason!(SESSION, ok)).toBe("stop"); + it('deriveStopReason reads the terminal step_finish reason', () => { + const runner = createOpencodeRunner('anthropic/claude-sonnet-5'); + const ok = { ok: true, exitCode: 0, stdout: '', stderr: '' }; + expect(runner.deriveStopReason!(SESSION, ok)).toBe('stop'); // A non-stop terminal reason is surfaced verbatim. - const length = JSON.stringify({ type: "step_finish", part: { reason: "length" } }); - expect(runner.deriveStopReason!(length, ok)).toBe("length"); + const length = JSON.stringify({ + type: 'step_finish', + part: { reason: 'length' }, + }); + expect(runner.deriveStopReason!(length, ok)).toBe('length'); // An error event wins regardless of exit code. - const errored = JSON.stringify({ type: "error", error: { message: "model overloaded" } }); - expect(runner.deriveStopReason!(errored, ok)).toBe("error"); + const errored = JSON.stringify({ + type: 'error', + error: { message: 'model overloaded' }, + }); + expect(runner.deriveStopReason!(errored, ok)).toBe('error'); }); it("builds opencode's MCP config shape from harness server configs", () => { const config = JSON.parse( buildOpencodeConfig({ - supabase: { command: "npx", args: ["-y", "srv"], env: { TOKEN: "t" } }, - docs: { command: "docs-server" }, - }), + supabase: { command: 'npx', args: ['-y', 'srv'], env: { TOKEN: 't' } }, + docs: { command: 'docs-server' }, + }) ); expect(config.mcp).toEqual({ supabase: { - type: "local", - command: ["npx", "-y", "srv"], + type: 'local', + command: ['npx', '-y', 'srv'], enabled: true, - environment: { TOKEN: "t" }, + environment: { TOKEN: 't' }, }, // No env → no `environment` key. - docs: { type: "local", command: ["docs-server"], enabled: true }, + docs: { type: 'local', command: ['docs-server'], enabled: true }, + }); + // No gateway → no custom provider block. + expect(config.provider).toBeUndefined(); + }); + + it('adds a Vercel AI Gateway provider block when routing through the gateway', () => { + const config = JSON.parse( + buildOpencodeConfig({}, { model: 'moonshotai/kimi-k3', apiKey: 'gw-key' }) + ); + expect(config.provider['vercel-ai-gateway']).toEqual({ + npm: '@ai-sdk/openai-compatible', + name: 'Vercel AI Gateway', + options: { baseURL: AI_GATEWAY.openAiBaseUrl, apiKey: 'gw-key' }, + // The gateway `vendor/model` slug is the model id under the provider. + models: { 'moonshotai/kimi-k3': {} }, + }); + }); +}); + +/** Capture the `--model` flag, run env, and written config from one exec. */ +async function captureExec( + model: string, + opts: { gateway?: boolean; mcp?: boolean } +): Promise<{ + runCommand: string; + runEnv: Record | undefined; + config: Record | undefined; +}> { + const ok: CommandResult = { ok: true, exitCode: 0, stdout: '', stderr: '' }; + let runCommand = ''; + let runEnv: Record | undefined; + let config: Record | undefined; + await createOpencodeRunner(model).exec({ + sandbox: { + workspace: '/w', + exec: async (cmd, options) => { + const write = /^printf %s '([^']+)'/.exec(cmd); + if (write) { + config = JSON.parse(Buffer.from(write[1], 'base64').toString('utf8')); + } else if (cmd.includes(' run ')) { + runCommand = cmd; + runEnv = options?.env; + } + return ok; + }, + readFile: async () => '', + }, + model, + apiKey: opts.gateway ? 'gw-key' : 'vendor-key', + gateway: opts.gateway, + systemPromptPath: '/s', + userPromptPath: '/u', + mcpServers: opts.mcp ? { supabase: { command: 'srv' } } : {}, + timeoutSec: 1, + }); + return { runCommand, runEnv, config }; +} + +describe('opencode runner exec routing', () => { + it('routes the model through the gateway provider and drops the vendor key', async () => { + const { runCommand, runEnv, config } = await captureExec( + 'moonshotai/kimi-k3', + { gateway: true, mcp: true } + ); + // Model is addressed under the custom provider; the gateway slug stays intact. + expect(runCommand).toContain( + "--model 'vercel-ai-gateway/moonshotai/kimi-k3'" + ); + // Key rides in the config, so no vendor key env var is set for the run. + expect(runEnv).toEqual({}); + // Config carries both MCP servers and the gateway provider. + expect(config?.mcp).toHaveProperty('supabase'); + expect(config?.provider).toHaveProperty('vercel-ai-gateway'); + }); + + it('keeps the direct provider/model id and vendor key otherwise', async () => { + const { runCommand, runEnv, config } = await captureExec( + 'moonshotai/kimi-k3', + { mcp: true } + ); + expect(runCommand).toContain("--model 'moonshotai/kimi-k3'"); + expect(runEnv).toEqual({ MOONSHOT_API_KEY: 'vendor-key' }); + expect(config?.provider).toBeUndefined(); + }); + + it('writes a config for the gateway provider even without MCP servers', async () => { + const { config } = await captureExec('moonshotai/kimi-k3', { + gateway: true, }); + expect(config?.provider).toHaveProperty('vercel-ai-gateway'); }); }); diff --git a/packages/core/src/agents/opencode/runner.ts b/packages/core/src/agents/opencode/runner.ts index 989d61ce..54bce76a 100644 --- a/packages/core/src/agents/opencode/runner.ts +++ b/packages/core/src/agents/opencode/runner.ts @@ -16,13 +16,14 @@ * an OPENCODE_CONFIG file outside the scored workspace). */ -import type { Model as AnthropicModel } from "@anthropic-ai/sdk/resources/messages"; -import type { ChatModel as OpenAIModel } from "openai/resources/shared"; -import type { GoogleGenerativeAIProvider } from "@ai-sdk/google"; -import type { McpServerConfig } from "../../index.js"; -import type { ModelProvider } from "../../eval-metadata.js"; -import { isRecord, parseJsonlRecords } from "../../json.js"; -import type { AgentRunner } from "../types.js"; +import type { Model as AnthropicModel } from '@anthropic-ai/sdk/resources/messages'; +import type { ChatModel as OpenAIModel } from 'openai/resources/shared'; +import type { GoogleGenerativeAIProvider } from '@ai-sdk/google'; +import type { McpServerConfig } from '../../index.js'; +import type { ModelProvider } from '../../eval-metadata.js'; +import { isRecord, parseJsonlRecords } from '../../json.js'; +import type { AgentRunner } from '../types.js'; +import { AI_GATEWAY } from '../gateway.js'; import { SCRATCH, npmGlobalBin, @@ -30,7 +31,7 @@ import { processStopReason, shellQuote, writeSandboxFile, -} from "../shared.js"; +} from '../shared.js'; /** Gemini model ids, extracted from the exported (callable) provider type. */ type GeminiModel = Parameters[0]; @@ -48,7 +49,8 @@ export type OpenCodeModel = | (string & {}); /** Model used when the caller doesn't pick one. */ -export const DEFAULT_OPENCODE_MODEL: OpenCodeModel = "anthropic/claude-sonnet-5"; +export const DEFAULT_OPENCODE_MODEL: OpenCodeModel = + 'anthropic/claude-sonnet-5'; /** * Provider prefix (`provider/model`) → the env var holding its key. opencode and @@ -57,19 +59,19 @@ export const DEFAULT_OPENCODE_MODEL: OpenCodeModel = "anthropic/claude-sonnet-5" * Moonshot's (`moonshotai/` ids, e.g. Kimi) is `MOONSHOT_API_KEY`. */ const PROVIDER_API_KEY_ENV: Record = { - anthropic: "ANTHROPIC_API_KEY", - openai: "OPENAI_API_KEY", - google: "GOOGLE_GENERATIVE_AI_API_KEY", - moonshotai: "MOONSHOT_API_KEY", + anthropic: 'ANTHROPIC_API_KEY', + openai: 'OPENAI_API_KEY', + google: 'GOOGLE_GENERATIVE_AI_API_KEY', + moonshotai: 'MOONSHOT_API_KEY', }; /** The provider prefix of a `provider/model` id; throws if unsupported. */ export function providerForModel(model: string): ModelProvider { - const provider = model.split("/")[0]; + const provider = model.split('/')[0]; if (!(provider in PROVIDER_API_KEY_ENV)) { throw new Error( `Unsupported opencode provider "${provider}" in model "${model}". ` + - `Supported: ${Object.keys(PROVIDER_API_KEY_ENV).join(", ")}.`, + `Supported: ${Object.keys(PROVIDER_API_KEY_ENV).join(', ')}.` ); } return provider as ModelProvider; @@ -81,66 +83,109 @@ export function providerApiKeyEnv(model: string): string { } /** - * Shell path to the MCP config, staged in scratch (outside the workspace). Used - * both as the write target and as the `OPENCODE_CONFIG` env value — the shell - * expands `$HOME` in either position. + * Shell path to the config, staged in scratch (outside the workspace). Used both + * as the write target and as the `OPENCODE_CONFIG` env value — the shell expands + * `$HOME` in either position. Holds the MCP servers and, in gateway mode, the + * custom AI Gateway provider block. */ const OPENCODE_CONFIG_PATH = '"$HOME/.eval/opencode.json"'; +/** + * Config provider id for the Vercel AI Gateway route (see `buildOpencodeConfig`). + * opencode addresses a model as `provider/model`, splitting on the first `/`, so + * a gateway run's `--model` is `${GATEWAY_PROVIDER_ID}//` and the + * gateway `vendor/model` slug stays intact as the model id. + */ +const GATEWAY_PROVIDER_ID = 'vercel-ai-gateway'; + /** * Build an opencode runner bound to one model's provider. opencode is * multi-provider, but a single run targets one model, so the runner resolves * `apiKeyEnvVar` and `modelProvider` from the model id (the generic layer's * `requireApiKey` reads `apiKeyEnvVar`, and `exec` injects that same key). */ -export function createOpencodeRunner(model: OpenCodeModel): AgentRunner { +export function createOpencodeRunner( + model: OpenCodeModel +): AgentRunner { const modelProvider = providerForModel(model); return { - id: "opencode", - displayName: "OpenCode", + id: 'opencode', + displayName: 'OpenCode', apiKeyEnvVar: providerApiKeyEnv(model), modelProvider, - cliPackage: "opencode-ai", + cliPackage: 'opencode-ai', // Pinned: opencode's --format json event schema evolves; bump deliberately // and re-check the parser. See ./parser.ts. - defaultCliVersion: "1.15.7", + defaultCliVersion: '1.15.7', defaultModel: DEFAULT_OPENCODE_MODEL, async install(sandbox, version) { - await npmInstallGlobal(sandbox, `${this.cliPackage}@${version}`, this.displayName); + await npmInstallGlobal( + sandbox, + `${this.cliPackage}@${version}`, + this.displayName + ); }, - async exec({ sandbox, model, apiKey, systemPromptPath, userPromptPath, mcpServers, timeoutSec }) { - const opencode = npmGlobalBin("opencode"); + async exec({ + sandbox, + model, + apiKey, + gateway, + systemPromptPath, + userPromptPath, + mcpServers, + timeoutSec, + }) { + const opencode = npmGlobalBin('opencode'); // opencode has no system-prompt flag, so prepend the system prompt to the // task; both are staged files, joined via command substitution into the // single message argument. const message = `"$(cat ${systemPromptPath}; printf '\\n\\n'; cat ${userPromptPath})"`; - let configPrefix = ""; - if (Object.keys(mcpServers).length > 0) { + // A config file is needed for MCP servers (both modes) and for the gateway + // provider block (gateway mode) — write it whenever either applies. + let configPrefix = ''; + if (Object.keys(mcpServers).length > 0 || gateway) { await sandbox.exec(`mkdir -p ${SCRATCH}`); - await writeSandboxFile(sandbox, OPENCODE_CONFIG_PATH, buildOpencodeConfig(mcpServers)); + await writeSandboxFile( + sandbox, + OPENCODE_CONFIG_PATH, + buildOpencodeConfig( + mcpServers, + gateway ? { model, apiKey } : undefined + ) + ); configPrefix = `OPENCODE_CONFIG=${OPENCODE_CONFIG_PATH} `; } + // Gateway mode routes the model through the custom provider defined in the + // config; the gateway slug (e.g. `moonshotai/kimi-k3`) becomes the model id + // under it. Direct mode passes the `provider/model` id through unchanged. + const runModel = gateway ? `${GATEWAY_PROVIDER_ID}/${model}` : model; + const flags = [ - "run", + 'run', message, - `--model ${shellQuote(model)}`, + `--model ${shellQuote(runModel)}`, // Newline-delimited JSON event records on stdout. - "--format json", + '--format json', // The sandbox is the isolation boundary, so let opencode act freely. - "--dangerously-skip-permissions", - ].join(" "); + '--dangerously-skip-permissions', + ].join(' '); // `< /dev/null`: opencode run blocks on stdin otherwise, even with the // message passed as an argument. - const command = await sandbox.exec(`${configPrefix}${opencode} ${flags} < /dev/null`, { - timeoutMs: timeoutSec * 1000, - env: { [this.apiKeyEnvVar]: apiKey }, - }); + const command = await sandbox.exec( + `${configPrefix}${opencode} ${flags} < /dev/null`, + { + timeoutMs: timeoutSec * 1000, + // Direct: the vendor's own key env var. Gateway: the key is embedded + // in the provider config, so no key env var is set. + env: gateway ? {} : { [this.apiKeyEnvVar]: apiKey }, + } + ); return { command, raw: command.stdout }; }, @@ -148,14 +193,17 @@ export function createOpencodeRunner(model: OpenCodeModel): AgentRunner r.type === "error")) return "error"; + if (records.some((r) => r.type === 'error')) return 'error'; // The terminal `step_finish` carries the model's finish reason. for (let i = records.length - 1; i >= 0; i -= 1) { - if (records[i].type !== "step_finish") continue; + if (records[i].type !== 'step_finish') continue; const part = records[i].part; - const reason = isRecord(part) && typeof part.reason === "string" ? part.reason : undefined; - if (reason === "stop") return "stop"; - if (reason && reason !== "tool-calls") return reason; // e.g. length — surface verbatim + const reason = + isRecord(part) && typeof part.reason === 'string' + ? part.reason + : undefined; + if (reason === 'stop') return 'stop'; + if (reason && reason !== 'tool-calls') return reason; // e.g. length — surface verbatim break; } return processStopReason(command); @@ -164,19 +212,46 @@ export function createOpencodeRunner(model: OpenCodeModel): AgentRunner): string { +export function buildOpencodeConfig( + servers: Record, + gateway?: { model: string; apiKey: string } +): string { const mcp: Record = {}; for (const [name, server] of Object.entries(servers)) { mcp[name] = { - type: "local", + type: 'local', command: [server.command, ...(server.args ?? [])], enabled: true, ...(server.env ? { environment: server.env } : {}), }; } - return JSON.stringify({ $schema: "https://opencode.ai/config.json", mcp }, null, 2); + const config: Record = { + $schema: 'https://opencode.ai/config.json', + mcp, + }; + if (gateway) { + config.provider = { + [GATEWAY_PROVIDER_ID]: { + npm: '@ai-sdk/openai-compatible', + name: 'Vercel AI Gateway', + options: { + baseURL: AI_GATEWAY.openAiBaseUrl, + apiKey: gateway.apiKey, + }, + models: { [gateway.model]: {} }, + }, + }; + } + return JSON.stringify(config, null, 2); } From 88cb1839eff80b9e0273d345a68ab9945ef3cf0b Mon Sep 17 00:00:00 2001 From: Pedro Rodrigues Date: Fri, 24 Jul 2026 16:09:04 +0100 Subject: [PATCH 5/6] fix(eval): register opencode-kimi-k3 in the benchmark suite Without a `suite`, the experiment is invisible to the CI eval-refresh workflow, which filters experiments by `--experiment-suite` ("no experiments matched experiment=opencode-kimi-k3"). Add `suite: ['benchmark']` so Kimi K3 is part of the benchmark eval suite, matching the other benchmark experiments. Co-Authored-By: Claude Opus 4.8 (1M context) --- experiments/opencode-kimi-k3.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/experiments/opencode-kimi-k3.ts b/experiments/opencode-kimi-k3.ts index f8354ec4..dc00512c 100644 --- a/experiments/opencode-kimi-k3.ts +++ b/experiments/opencode-kimi-k3.ts @@ -12,6 +12,7 @@ import { localStackRuntime } from '@supabase-evals/sandbox'; // needs AI_GATEWAY_API_KEY rather than a direct MOONSHOT_API_KEY. Runs in both // modes like the other opencode experiments (see opencode-claude-sonnet-5.ts). export default defineExperiment({ + suite: ['benchmark'], agent: opencodeAgent({ model: 'moonshotai/kimi-k3', gateway: true, From cead22f517183ae911801417768ac20b74e5b746 Mon Sep 17 00:00:00 2001 From: Pedro Rodrigues Date: Fri, 24 Jul 2026 18:14:36 +0100 Subject: [PATCH 6/6] refactor(opencode): type the config against @opencode-ai/sdk MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add @opencode-ai/sdk as a type-only devDependency, pinned (via the pnpm catalog) to the exact CLI version the runner installs (1.15.7), and type buildOpencodeConfig's output against opencode's own `Config` schema (`McpLocalConfig` for MCP servers, `ProviderConfig` for the gateway provider block). A config-layout change on a CLI bump now fails to compile instead of breaking silently at runtime. Deliberately scoped to the config only: - Model ids stay `string` — opencode's model catalog is dynamic (models.dev), so there is no model-name union to import the way the vendor SDKs and the AI Gateway (GatewayModelId) provide one. - The transcript parser is NOT typed from this SDK. `run --format json` emits a reduced, differently-shaped record than the SDK's server-API Part/Event entities (the CLI's outer discriminants `tool_use`/`step_finish` don't exist in the SDK; inner parts omit the id/sessionID/messageID the SDK marks required), so the parser stays schema-defensive. Documented at both sites. Co-Authored-By: Claude Opus 4.8 (1M context) --- packages/core/package.json | 1 + packages/core/src/agents/opencode/runner.ts | 33 ++++++++++++++------- pnpm-lock.yaml | 13 ++++++++ pnpm-workspace.yaml | 4 +++ 4 files changed, 40 insertions(+), 11 deletions(-) diff --git a/packages/core/package.json b/packages/core/package.json index bf84b15f..043fbe45 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -14,6 +14,7 @@ "test": "vitest run" }, "devDependencies": { + "@opencode-ai/sdk": "catalog:", "vitest": "catalog:" }, "dependencies": { diff --git a/packages/core/src/agents/opencode/runner.ts b/packages/core/src/agents/opencode/runner.ts index 54bce76a..a6cdec2b 100644 --- a/packages/core/src/agents/opencode/runner.ts +++ b/packages/core/src/agents/opencode/runner.ts @@ -19,6 +19,14 @@ import type { Model as AnthropicModel } from '@anthropic-ai/sdk/resources/messages'; import type { ChatModel as OpenAIModel } from 'openai/resources/shared'; import type { GoogleGenerativeAIProvider } from '@ai-sdk/google'; +// opencode's own config schema (type-only; pinned to the installed CLI version +// via the catalog). Model ids stay `string` — opencode's catalog is dynamic +// (models.dev), so there is no model-name union to import, unlike the vendor +// SDKs above. The transcript stream is deliberately NOT typed from this SDK: +// `run --format json` emits a reduced, differently-shaped record than the SDK's +// server-API `Part`/`Event` entities (no id/sessionID/messageID; different +// discriminants), so the parser stays schema-defensive — see ./parser.ts. +import type { Config, McpLocalConfig, ProviderConfig } from '@opencode-ai/sdk'; import type { McpServerConfig } from '../../index.js'; import type { ModelProvider } from '../../eval-metadata.js'; import { isRecord, parseJsonlRecords } from '../../json.js'; @@ -227,7 +235,7 @@ export function buildOpencodeConfig( servers: Record, gateway?: { model: string; apiKey: string } ): string { - const mcp: Record = {}; + const mcp: Record = {}; for (const [name, server] of Object.entries(servers)) { mcp[name] = { type: 'local', @@ -236,22 +244,25 @@ export function buildOpencodeConfig( ...(server.env ? { environment: server.env } : {}), }; } - const config: Record = { + // Typed against opencode's own `Config` schema, so a config-shape change on a + // CLI bump (mcp/provider layout) fails to compile instead of silently at runtime. + const config: Config = { $schema: 'https://opencode.ai/config.json', mcp, }; if (gateway) { - config.provider = { - [GATEWAY_PROVIDER_ID]: { - npm: '@ai-sdk/openai-compatible', - name: 'Vercel AI Gateway', - options: { - baseURL: AI_GATEWAY.openAiBaseUrl, - apiKey: gateway.apiKey, - }, - models: { [gateway.model]: {} }, + // A custom OpenAI-compatible provider pointed at the gateway's /v1 surface; + // the one gateway `vendor/model` slug is the model id under it. + const gatewayProvider: ProviderConfig = { + npm: '@ai-sdk/openai-compatible', + name: 'Vercel AI Gateway', + options: { + baseURL: AI_GATEWAY.openAiBaseUrl, + apiKey: gateway.apiKey, }, + models: { [gateway.model]: {} }, }; + config.provider = { [GATEWAY_PROVIDER_ID]: gatewayProvider }; } return JSON.stringify(config, null, 2); } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 2fbfb54f..00b7176a 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -27,6 +27,9 @@ catalogs: '@electric-sql/pglite-socket': specifier: 0.1.5 version: 0.1.5 + '@opencode-ai/sdk': + specifier: 1.15.7 + version: 1.15.7 '@supabase/lite': specifier: 0.7.1-next.3 version: 0.7.1-next.3 @@ -285,6 +288,9 @@ importers: specifier: 'catalog:' version: 4.4.3 devDependencies: + '@opencode-ai/sdk': + specifier: 'catalog:' + version: 1.15.7 vitest: specifier: 'catalog:' version: 4.1.8(@opentelemetry/api@1.9.1)(@types/node@25.9.3)(happy-dom@20.10.2)(vite@7.3.5(@types/node@25.9.3)(jiti@2.7.0)(lightningcss@1.32.0)(tsx@4.22.4)) @@ -1108,6 +1114,9 @@ packages: resolution: {integrity: sha512-oGB+UxlgWcgQkgwo8GcEGwemoTFt3FIO9ababBmaGwXIoBKZ+GTy0pP185beGg7Llih/NSHSV2XAs1lnznocSg==} engines: {node: '>= 8'} + '@opencode-ai/sdk@1.15.7': + resolution: {integrity: sha512-fNwx2coNzA8VAv4hazG9REGdBuUtV1UYjK3hxMo8+/9SZakOgdjihH1xzoTESJA0e0d0JJIKBCJ7FZVF2WVSXg==} + '@opentelemetry/api-logs@0.214.0': resolution: {integrity: sha512-40lSJeqYO8Uz2Yj7u94/SJWE/wONa7rmMKjI1ZcIjgf3MHNHv1OZUCrCETGuaRF62d5pQD1wKIW+L4lmSMTzZA==} engines: {node: '>=8.0.0'} @@ -5199,6 +5208,10 @@ snapshots: '@nodelib/fs.scandir': 2.1.5 fastq: 1.20.1 + '@opencode-ai/sdk@1.15.7': + dependencies: + cross-spawn: 7.0.6 + '@opentelemetry/api-logs@0.214.0': dependencies: '@opentelemetry/api': 1.9.1 diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index bbfd992f..5d560491 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -9,6 +9,10 @@ catalog: '@ai-sdk/google': ^3.0.83 '@ai-sdk/mcp': ^1.0.39 '@ai-sdk/openai': ^3.0.66 + # Type-only: opencode's config/schema types. Pinned exactly to the CLI + # version the opencode runner installs (DEFAULT_OPENCODE_CLI 1.15.7) so the + # types track the binary that actually runs. + '@opencode-ai/sdk': 1.15.7 '@electric-sql/pglite': 0.4.5 '@electric-sql/pglite-socket': 0.1.5 '@supabase/lite': 0.7.1-next.3