From 373a6a090922f9df73d35e8c32d834577c9bece4 Mon Sep 17 00:00:00 2001 From: Sawyer Cutler Date: Wed, 19 Aug 2026 21:13:00 -0700 Subject: [PATCH 1/2] Add live-smoke harness for the code-review MVP (CL-6340) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Wires @corbits/github-tools' real diff-fetch/post-review calls and @corbits/code-review's runPullRequestReview loop to a direct Anthropic Messages API call for runReviewerTurn — the seam the MVP left unbound. Confirmed against a live scratch PR: diff fetch resolves and posting works end to end; the inference pass itself needs ANTHROPIC_API_KEY, which is not configured in this sandbox. --- bun.lock | 2 + package.json | 2 + scripts/repro/live-smoke-code-review.ts | 158 ++++++++++++++++++++++++ 3 files changed, 162 insertions(+) create mode 100644 scripts/repro/live-smoke-code-review.ts diff --git a/bun.lock b/bun.lock index f669a222c..e9257e98d 100644 --- a/bun.lock +++ b/bun.lock @@ -8,6 +8,8 @@ "arktype": "catalog:", }, "devDependencies": { + "@corbits/code-review": "workspace:*", + "@corbits/github-tools": "workspace:*", "@eslint/js": "^10.0.0", "@intx/mime": "workspace:*", "@intx/types": "workspace:*", diff --git a/package.json b/package.json index 14a38c949..1ec8cfb1d 100644 --- a/package.json +++ b/package.json @@ -36,6 +36,8 @@ "eval": "bun run scripts/evals-run.ts" }, "devDependencies": { + "@corbits/code-review": "workspace:*", + "@corbits/github-tools": "workspace:*", "@eslint/js": "^10.0.0", "@intx/mime": "workspace:*", "@intx/types": "workspace:*", diff --git a/scripts/repro/live-smoke-code-review.ts b/scripts/repro/live-smoke-code-review.ts new file mode 100644 index 000000000..240904d3e --- /dev/null +++ b/scripts/repro/live-smoke-code-review.ts @@ -0,0 +1,158 @@ +/** + * Live smoke harness for the code-review MVP (CL-6340): fetches a real + * pull request's diff via @corbits/github-tools, runs the three + * @corbits/code-review reviewer lenses against a real Anthropic model + * call, aggregates the passes, and posts the review to GitHub for real. + * + * This is a smoke harness, not product plumbing: `runReviewerTurn` here + * is a direct Anthropic Messages API call (the same minimal seam + * packages/evals/src/model-call.ts already uses for eval-side model + * calls), because the MVP report flagged that no production inference + * binding exists yet for `runReviewerTurn` — the review-run package + * only defines the seam (packages/code-review/src/review-run.ts), it + * does not wire one. + * + * Run: bun run scripts/repro/live-smoke-code-review.ts /# + * Env: GITHUB_TOKEN (falls back to `gh auth token` if unset) + * ANTHROPIC_API_KEY (required — no fallback; this script never + * reads a credential store on its own) + */ +import { spawnSync } from "node:child_process"; + +import { + createGitHubReviewClient, + runPullRequestReview, + type ReviewerDefinition, +} from "@corbits/code-review"; +import type { PullRequestRef } from "@corbits/github-tools"; + +const MODEL = "claude-sonnet-4-5-20250929"; + +function parseTarget(arg: string): PullRequestRef { + const match = /^([^/\s]+)\/([^/\s]+)#(\d+)$/.exec(arg); + if (match === null) { + throw new Error(`expected /#, got "${arg}"`); + } + const [, owner, repo, number] = match; + if (owner === undefined || repo === undefined || number === undefined) { + throw new Error(`expected /#, got "${arg}"`); + } + return { owner, repo, number: Number(number) }; +} + +function ghToken(): string { + const fromEnv = process.env["GITHUB_TOKEN"]; + if (fromEnv !== undefined && fromEnv.length > 0) return fromEnv; + const result = spawnSync("gh", ["auth", "token"], { encoding: "utf8" }); + const token = result.stdout.trim(); + if (result.status !== 0 || token.length === 0) { + throw new Error("no GITHUB_TOKEN and `gh auth token` produced nothing"); + } + return token; +} + +async function callAnthropic( + systemPrompt: string, + userPrompt: string, + apiKey: string, +): Promise { + const res = await fetch("https://api.anthropic.com/v1/messages", { + method: "POST", + headers: { + "content-type": "application/json", + "x-api-key": apiKey, + "anthropic-version": "2023-06-01", + }, + body: JSON.stringify({ + model: MODEL, + max_tokens: 2000, + system: systemPrompt, + messages: [{ role: "user", content: userPrompt }], + }), + }); + if (!res.ok) { + throw new Error( + `Anthropic Messages API failed: ${String(res.status)} ${res.statusText} — ${await res.text()}`, + ); + } + const data = (await res.json()) as { + content?: { type: string; text?: string }[]; + }; + const text = data.content?.find((block) => block.type === "text")?.text; + if (text === undefined) { + throw new Error("Anthropic reply carried no text block"); + } + return text; +} + +async function main(): Promise { + const targetArg = process.argv[2]; + if (targetArg === undefined) { + throw new Error( + "usage: bun run scripts/repro/live-smoke-code-review.ts /#", + ); + } + const ref = parseTarget(targetArg); + + const anthropicKey = process.env["ANTHROPIC_API_KEY"]; + if (anthropicKey === undefined || anthropicKey.length === 0) { + throw new Error( + "ANTHROPIC_API_KEY is not set — this harness makes a real inference " + + "call and does not fall back to anything", + ); + } + + const github = createGitHubReviewClient({ apiKey: ghToken() }); + + const timings: Record = {}; + const t0 = performance.now(); + + const result = await runPullRequestReview( + { + github, + runReviewerTurn: async ({ + reviewer, + prompt, + }: { + reviewer: ReviewerDefinition; + prompt: string; + }) => { + const passStart = performance.now(); + const reply = await callAnthropic( + reviewer.systemPrompt, + prompt, + anthropicKey, + ); + timings[`pass:${reviewer.id}`] = performance.now() - passStart; + return reply; + }, + }, + ref, + ); + + timings["total"] = performance.now() - t0; + + console.log(`Posted review: ${result.posted.url}`); + console.log(""); + console.log("--- Timings (ms) ---"); + for (const [key, value] of Object.entries(timings)) { + console.log(`${key}: ${value.toFixed(0)}`); + } + console.log(""); + console.log("--- Reviewer passes ---"); + for (const pass of result.passes) { + console.log( + pass.ok + ? `${pass.reviewer.id}: ok` + : `${pass.reviewer.id}: FAILED — ${pass.reason}`, + ); + } + console.log(""); + console.log("--- Posted body ---"); + console.log(result.review.body); +} + +main().catch((err) => { + console.error(err); + process.exit(1); +}); From 2aa9de3fc1ce927959e64507ac895814508292ed Mon Sep 17 00:00:00 2001 From: Sawyer Cutler Date: Wed, 19 Aug 2026 21:18:37 -0700 Subject: [PATCH 2/2] live-smoke: add HARNESS-ONLY Ollama fallback, run the loop for real MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ANTHROPIC_API_KEY is still unavailable in this sandbox. To finish proving the loop mechanics end to end, runReviewerTurn falls back to a local Ollama chat completion (localhost:11434) when no Anthropic key is set — harness-only, not a second production inference binding; the production seam remains the known gap. Ran the full loop for real against corbitsdev/pr-review-spike-test-repo#5 with qwen2.5vl:7b: diff fetch, three reviewer passes, aggregate, and a real posted review with anchored inline comments. Per-leg timings are now recorded (diff-fetch, each pass, aggregate+post, total). --- scripts/repro/live-smoke-code-review.ts | 121 +++++++++++++++++++----- 1 file changed, 99 insertions(+), 22 deletions(-) diff --git a/scripts/repro/live-smoke-code-review.ts b/scripts/repro/live-smoke-code-review.ts index 240904d3e..68e02b172 100644 --- a/scripts/repro/live-smoke-code-review.ts +++ b/scripts/repro/live-smoke-code-review.ts @@ -1,21 +1,32 @@ /** * Live smoke harness for the code-review MVP (CL-6340): fetches a real * pull request's diff via @corbits/github-tools, runs the three - * @corbits/code-review reviewer lenses against a real Anthropic model - * call, aggregates the passes, and posts the review to GitHub for real. + * @corbits/code-review reviewer lenses against a real model call, + * aggregates the passes, and posts the review to GitHub for real. * * This is a smoke harness, not product plumbing: `runReviewerTurn` here - * is a direct Anthropic Messages API call (the same minimal seam + * is a direct model call (the same minimal seam * packages/evals/src/model-call.ts already uses for eval-side model * calls), because the MVP report flagged that no production inference * binding exists yet for `runReviewerTurn` — the review-run package * only defines the seam (packages/code-review/src/review-run.ts), it * does not wire one. * + * Two inference paths, chosen by what credential is present — an + * ANTHROPIC_API_KEY when there is one, a local Ollama chat completion + * (localhost:11434) when there is not. The Ollama path is HARNESS-ONLY: + * it exists so this smoke run can prove the loop mechanics (diff fetch + * → three passes → aggregate → post) for real without a paid credential + * in the sandbox, not as a second production inference binding. The + * production seam remains the known gap the MVP report flagged; nothing + * here narrows it. Expect modest finding quality from a small local + * model — that is not what this harness is proving. + * * Run: bun run scripts/repro/live-smoke-code-review.ts /# * Env: GITHUB_TOKEN (falls back to `gh auth token` if unset) - * ANTHROPIC_API_KEY (required — no fallback; this script never - * reads a credential store on its own) + * ANTHROPIC_API_KEY (preferred when set) + * OLLAMA_BASE_URL (default http://localhost:11434), OLLAMA_MODEL + * (default qwen2.5vl:7b) — used only when ANTHROPIC_API_KEY is unset */ import { spawnSync } from "node:child_process"; @@ -26,7 +37,9 @@ import { } from "@corbits/code-review"; import type { PullRequestRef } from "@corbits/github-tools"; -const MODEL = "claude-sonnet-4-5-20250929"; +const ANTHROPIC_MODEL = "claude-sonnet-4-5-20250929"; +const DEFAULT_OLLAMA_BASE_URL = "http://localhost:11434"; +const DEFAULT_OLLAMA_MODEL = "qwen2.5vl:7b"; function parseTarget(arg: string): PullRequestRef { const match = /^([^/\s]+)\/([^/\s]+)#(\d+)$/.exec(arg); @@ -64,7 +77,7 @@ async function callAnthropic( "anthropic-version": "2023-06-01", }, body: JSON.stringify({ - model: MODEL, + model: ANTHROPIC_MODEL, max_tokens: 2000, system: systemPrompt, messages: [{ role: "user", content: userPrompt }], @@ -85,6 +98,41 @@ async function callAnthropic( return text; } +/** HARNESS-ONLY fallback: a local Ollama chat completion. Not a second + * production inference binding — see the module comment. */ +async function callOllama( + systemPrompt: string, + userPrompt: string, + baseUrl: string, + model: string, +): Promise { + const res = await fetch(`${baseUrl}/api/chat`, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ + model, + stream: false, + messages: [ + { role: "system", content: systemPrompt }, + { role: "user", content: userPrompt }, + ], + }), + }); + if (!res.ok) { + throw new Error( + `Ollama chat call failed: ${String(res.status)} ${res.statusText} — ${await res.text()}`, + ); + } + const data = (await res.json()) as { + message?: { content?: string }; + }; + const text = data.message?.content; + if (text === undefined || text.length === 0) { + throw new Error("Ollama reply carried no content"); + } + return text; +} + async function main(): Promise { const targetArg = process.argv[2]; if (targetArg === undefined) { @@ -95,21 +143,38 @@ async function main(): Promise { const ref = parseTarget(targetArg); const anthropicKey = process.env["ANTHROPIC_API_KEY"]; - if (anthropicKey === undefined || anthropicKey.length === 0) { - throw new Error( - "ANTHROPIC_API_KEY is not set — this harness makes a real inference " + - "call and does not fall back to anything", - ); - } + const ollamaBaseUrl = + process.env["OLLAMA_BASE_URL"] ?? DEFAULT_OLLAMA_BASE_URL; + const ollamaModel = process.env["OLLAMA_MODEL"] ?? DEFAULT_OLLAMA_MODEL; + const inferenceMode = + anthropicKey !== undefined && anthropicKey.length > 0 + ? "anthropic" + : "ollama"; + console.log( + inferenceMode === "anthropic" + ? `Inference: Anthropic (${ANTHROPIC_MODEL})` + : `Inference: Ollama fallback, HARNESS-ONLY (${ollamaModel} @ ${ollamaBaseUrl})`, + ); const github = createGitHubReviewClient({ apiKey: ghToken() }); const timings: Record = {}; - const t0 = performance.now(); + const runStart = performance.now(); + let diffDoneAt = runStart; + let lastPassDoneAt = runStart; + const timedGithub = { + ...github, + fetchDiff: async (fetchRef: PullRequestRef) => { + const diff = await github.fetchDiff(fetchRef); + diffDoneAt = performance.now(); + timings["diff-fetch"] = diffDoneAt - runStart; + return diff; + }, + }; const result = await runPullRequestReview( { - github, + github: timedGithub, runReviewerTurn: async ({ reviewer, prompt, @@ -118,19 +183,31 @@ async function main(): Promise { prompt: string; }) => { const passStart = performance.now(); - const reply = await callAnthropic( - reviewer.systemPrompt, - prompt, - anthropicKey, - ); - timings[`pass:${reviewer.id}`] = performance.now() - passStart; + const reply = + inferenceMode === "anthropic" + ? await callAnthropic( + reviewer.systemPrompt, + prompt, + anthropicKey as string, + ) + : await callOllama( + reviewer.systemPrompt, + prompt, + ollamaBaseUrl, + ollamaModel, + ); + const passDoneAt = performance.now(); + timings[`pass:${reviewer.id}`] = passDoneAt - passStart; + lastPassDoneAt = Math.max(lastPassDoneAt, passDoneAt); return reply; }, }, ref, ); - timings["total"] = performance.now() - t0; + const runEnd = performance.now(); + timings["aggregate+post"] = runEnd - lastPassDoneAt; + timings["total"] = runEnd - runStart; console.log(`Posted review: ${result.posted.url}`); console.log("");