From 4edc3b12f5facbaf31433e5739c0539dda895bc9 Mon Sep 17 00:00:00 2001 From: James Sesler Date: Sun, 23 Aug 2026 01:29:35 -0400 Subject: [PATCH 1/7] feat: Broad-Side batch reconnaissance over the OpenRouter Batch API (#103) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit New codecarto_broadside MCP tool (submit/collect/status) fires six single-turn analysis lenses — architecture, API surface, security, mechanical defect scan, convention extraction, porting — at any git repository as asynchronous OpenRouter batch jobs. Large modules are sliced by top-level directory and oversized slices split rather than truncate. Results land in .codecarto/broadside// as JSON plus rendered markdown, with an optional cross-lens synthesis report. Deliberately not the interactive pipeline: batch is text-in/text-out, so findings are framed as unverified scouting leads for the real analysis to confirm. Works without an initialized workspace; init now tolerates a scout-only .codecarto/, and scaffold refresh never touches broadside state, config, or results. --- .codecarto/.gitignore | 7 + .codecarto/GUIDE.md | 7 +- .codecarto/broadside/SKILL.md | 68 ++ .codecarto/broadside/config.yaml | 27 + CHANGELOG.md | 4 + README.md | 1 + core/broadside.ts | 1603 ++++++++++++++++++++++++++++++ core/index.ts | 1 + core/workspace.ts | 4 +- mcp-server/server.ts | 153 ++- scripts/smoke-broadside.mjs | 52 + scripts/smoke-mcp.mjs | 1 + tests/broadside.test.mjs | 390 ++++++++ 13 files changed, 2315 insertions(+), 3 deletions(-) create mode 100644 .codecarto/broadside/SKILL.md create mode 100644 .codecarto/broadside/config.yaml create mode 100644 core/broadside.ts create mode 100644 scripts/smoke-broadside.mjs create mode 100644 tests/broadside.test.mjs diff --git a/.codecarto/.gitignore b/.codecarto/.gitignore index b66c5f7..39f6ab6 100644 --- a/.codecarto/.gitignore +++ b/.codecarto/.gitignore @@ -17,6 +17,13 @@ findings/config-model/config-model.md scratch/* !scratch/.gitkeep +# Broad-Side machine-local state and generated results (batch ids, run +# directories, costs). The SKILL.md guidance and the config template are +# tracked; the API key inside config.yaml is your own risk to commit. +broadside/* +!broadside/SKILL.md +!broadside/config.yaml + # Orchestrator session pointer (machine-local, written by /codecarto-init # when run from the Pi extension; the MCP path doesn't write it). Contains # absolute paths into the user's Pi session storage, so it must never be diff --git a/.codecarto/GUIDE.md b/.codecarto/GUIDE.md index 92d8301..9194c78 100644 --- a/.codecarto/GUIDE.md +++ b/.codecarto/GUIDE.md @@ -57,6 +57,7 @@ Read these files in order before doing any analysis: 4. `scratch/checkpoints/.md`, if present, to resume durable in-phase progress after compaction or interruption. 5. The current phase's `SKILL.md` for detailed instructions on what to analyze and produce. 6. The output template from `templates/` for the current phase (if starting a new output). +7. `broadside/synthesis.md`, if a Broad-Side batch reconnaissance run has completed — it carries unverified scouting leads (see `broadside/SKILL.md`) that tell you where the interactive phases should spend attention. All paths in this guide are relative to `.codecarto/` unless stated otherwise. @@ -71,7 +72,7 @@ Some files in this workspace are **read-only instructions** and must not be modi | Category | Files | Access | |---|---|---| | Orchestration (read-only) | `GUIDE.md`, `CONTRIBUTING.md`, `LICENSE` | Read only. Never modify. | -| Skills (read-only) | `findings/*/SKILL.md`, `findings/defect-scan/passes/*.md`, `skills/*/SKILL.md` | Read only. Never modify. | +| Skills (read-only) | `findings/*/SKILL.md`, `findings/defect-scan/passes/*.md`, `skills/*/SKILL.md`, `broadside/SKILL.md` | Read only. Never modify. | | Templates (read-only) | `templates/*.md` | Read only. Never modify. | | Pipeline definitions (read-only) | `workflow/pipeline*.yaml`, `workflow/VALIDATE.md` | Read only. Never modify. | | Source code (read-only) | `../` (everything outside `.codecarto/`) | Read only. Analyze but never modify. | @@ -313,6 +314,10 @@ your-repo/ VALIDATE.md # Validation protocol. Run after every phase. closeouts/ # Per-session closeout files (replaces monolithic THREAD_LOG body). -.md + broadside/ # Batch reconnaissance state and results (see broadside/SKILL.md). + SKILL.md # How to read Broad-Side scouting leads (unverified, not evidence). + config.yaml # Broad-Side model/key/lens configuration. + / # Per-run JSON + markdown findings, run-meta.json, synthesis report. CONVENTIONS.md # (Optional, project-grown) Cross-cutting invariants. Orchestrator-maintained. DECISIONS.md # (Optional, project-grown) Numbered decisions log. Orchestrator-maintained. BACKLOG.md # (Optional) Deferred items with rationale. diff --git a/.codecarto/broadside/SKILL.md b/.codecarto/broadside/SKILL.md new file mode 100644 index 0000000..751c700 --- /dev/null +++ b/.codecarto/broadside/SKILL.md @@ -0,0 +1,68 @@ +--- +name: broadside +description: Interpret a Broad-Side batch reconnaissance run. Use after codecarto_broadside collect has produced .codecarto/broadside// results, to triage scouting signals before or during an interactive CodeCartographer pipeline run. +--- + +# Broad-Side + +Broad-Side is CodeCartographer's batch reconnaissance pass. It fires six +analysis lenses — architecture, API surface, security, mechanical defect scan, +convention extraction, and porting — at the repository as single-turn prompts +over the OpenRouter Batch API (~50% of sync pricing, asynchronous, unattended), +then synthesizes one cross-lens report. Results live under +`.codecarto/broadside//` alongside this file. + +## What Broad-Side findings are — and are not + +Broad-Side findings are **unverified scouting signals**, not validated claims. +Every lens is one shot: no cross-file traversal, no runtime verification, no +builds, no tests, no follow-up questions. The batch model is cheap, not strong. +Treat every finding as a lead with a file:line pointer that the interactive +pipeline — or you — must confirm before it is a fact. + +This is the division of labor: Broad-Side is cheap enough to run on any repo to +decide where the expensive interactive run should spend its attention. It does +not replace any phase; it tells phases where to look. + +## Reading a Broad-Side run + +1. Read `synthesis.md` first. It carries the executive summary, severity counts, + the top cross-lens findings, and per-module risk levels. +2. Read the per-lens files behind anything that matters to your current phase: + - `architecture-*.json` → the architecture phase's seed of prior knowledge + - `api-*.json` → endpoints and data types (contracts/protocols phases) + - `security-*.json` → auth, trust boundaries (defect-scan-semantic pass 5) + - `defect-*.json` → mechanical defect leads (defect-scan-mechanical) + - `conventions-*.json` → naming/idiom candidates for CONVENTIONS.md + - `porting-*.json` → platform coupling (porting phase) +3. `run-meta.json` records scope: which lenses ran, at what cost, with what + coverage caps. + +## How to use the leads + +- **A finding that matches your phase's scope is a starting point, not an answer.** + Re-derive it from the source yourself; cite the source, not the Broad-Side + report. Broad-Side output is not evidence. +- **Route, don't believe.** A Broad-Side "high" that your phase can neither + confirm nor dismiss becomes an open question with `needs-runtime-test` or + `needs-maintainer-decision` — never a finding. +- **Promotable conventions are candidates only.** CONVENTIONS.md promotions + still require the orchestrator's review against the code, per the usual + promotion rules. +- **Coverage caps are real.** Directory-sliced lenses cap each slice's input; + `run-meta.json` and the synthesis `coverage` field say what was scanned. + Everything outside that is unscouted, not clean. + +## Running Broad-Side + +Broad-Side is an executable-surface feature (MCP today): + +``` +codecarto_broadside {cwd, action: "submit", lenses: [...]} # fire the batches +codecarto_broadside {cwd, action: "collect"} # poll, save, synthesize +codecarto_broadside {cwd, action: "status"} # show recorded runs +``` + +It works on any git repository — no initialized workspace required — and needs +an OpenRouter API key via the `api_key` parameter, the `OPENROUTER_API_KEY` +environment variable, or `api_key` in this directory's `config.yaml`. diff --git a/.codecarto/broadside/config.yaml b/.codecarto/broadside/config.yaml new file mode 100644 index 0000000..7e28fff --- /dev/null +++ b/.codecarto/broadside/config.yaml @@ -0,0 +1,27 @@ +# Broad-Side batch reconnaissance configuration. Optional. +# Missing keys fall back to the defaults listed below. + +# OpenRouter model to use for batch requests. The default is Google Gemini +# 3.7 Flash (batch) — the cheapest batch model with tool-calling support and +# a 1M-token context window. Change this to another OpenRouter batch model +# if you need a different cost/capability trade-off. +# +# model: google/gemini-3.7-flash:batch + +# OpenRouter API key. Prefer the OPENROUTER_API_KEY environment variable — +# keys in this file are committed if you track .codecarto/ after init. +# The codecarto_broadside tool also accepts api_key as a parameter. +# +# api_key: "" + +# Default lens set for codecarto_broadside submit when no lenses are +# specified. All six lenses are on by default. Remove a lens id to skip it +# globally, or pass an explicit lenses array on the submit call to override. +# +# default_lenses: +# - architecture +# - api +# - security +# - defect +# - conventions +# - porting \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md index 4979ac1..26011bd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,10 @@ All notable changes to this project are documented here. The format is based on ## [Unreleased] +### Added + +- **Broad-Side: batch reconnaissance over the OpenRouter Batch API** (#103). New `codecarto_broadside` MCP tool (actions: `submit`, `collect`, `status`) fires six single-turn analysis lenses — architecture, API surface, security, mechanical defect scan, convention extraction, porting — at any git repository as asynchronous batch jobs on a cheap batch model (~50% of sync pricing, unattended, 24h window), slices large modules by top-level directory, saves JSON plus rendered markdown to `.codecarto/broadside//`, and optionally synthesizes a cross-lens executive report. Works without an initialized workspace; needs an OpenRouter key via the `api_key` parameter, `OPENROUTER_API_KEY`, or `.codecarto/broadside/config.yaml`. Broad-Side findings are explicitly unverified scouting signals — file:line leads for the interactive pipeline to confirm, never evidence themselves. `codecarto_init` tolerates a `.codecarto/` that holds only `broadside/` (no force/backup needed), and scaffold refresh never touches broadside state, config, or results. + ## [0.16.0] — 2026-08-17 The field-test round. Immediately after 0.15.0 shipped, the same 7-phase deepseek-harness analysis was re-run on a fresh worktree through the published binary — this time with the driving chat as orchestrator — and the run's own gaps became this release (#111–#114): the very first completion appended decision rows without their promised heading, both full runs ended with no dashboard ever rendered, the analysis→publish→synthesis library loop was unreachable from any served text, and the terminal completion message named nothing actionable while skills, amendments, a publishable spec, and the usage log all sat unused. diff --git a/README.md b/README.md index 74c4d68..02ce3c3 100644 --- a/README.md +++ b/README.md @@ -357,6 +357,7 @@ Implements MCP spec revision [`2025-11-25`](https://modelcontextprotocol.io/spec | `codecarto_publish` | MCP-only library publish | | `codecarto_library_list` | MCP-only library listing | | `codecarto_library_reindex` | MCP-only library reindex | +| `codecarto_broadside` | MCP-only batch reconnaissance (Broad-Side) | Each workflow tool accepts an absolute `cwd` for the target repository. `codecarto_init` requires `force: true` to overwrite an existing `.codecarto/` (instead of Pi's interactive confirmation). The library tools accept an explicit absolute `library_path` or resolve `library.path` from `.codecarto/workflow/config.yaml` / `~/.codecarto/config.yaml`. The library schema is experimental and may break before v2. diff --git a/core/broadside.ts b/core/broadside.ts new file mode 100644 index 0000000..0b69d5d --- /dev/null +++ b/core/broadside.ts @@ -0,0 +1,1603 @@ +// Broad-Side: cheap batch reconnaissance over the OpenRouter Batch API. +// +// Broad-Side fires every analysis lens at a repository at once. Each lens is a +// single-turn prompt with a structured-output JSON schema, submitted as an +// asynchronous batch job (Google Gemini's batch endpoint, ~50% of sync pricing) +// and polled to completion. Results land in `.codecarto/broadside//` as +// JSON plus rendered markdown, and an optional synthesis pass cross-references +// every lens into one executive report. +// +// This is deliberately NOT the interactive CodeCartographer pipeline. The batch +// API is text-in/text-out: no filesystem access, no multi-turn exploration, no +// runtime verification. Broad-Side findings are unverified scouting signals — +// file:line leads that a real analysis (or a human) must confirm. That division +// of labor is the point: a ~$0.50 unattended sweep that tells the expensive +// interactive run where to look. +// +// Deliberately not in .codecarto/ template prose: Broad-Side requires runtime +// code, so it lives on the executable surfaces (MCP today, Pi on the roadmap). + +import { mkdir, readFile, readdir, writeFile } from "node:fs/promises"; +import { execFile } from "node:child_process"; +import { promisify } from "node:util"; +import { dirname, join } from "node:path"; +import { pathExists, sleep } from "./utils.ts"; +import { loadYamlFile } from "./yaml.ts"; + +const execFileAsync = promisify(execFile); + +// ---------- constants ---------- + +export const BROADSIDE_MODEL = "google/gemini-3.7-flash:batch"; +export const BROADSIDE_BATCH_URL = "https://openrouter.ai/api/beta/batches"; +export const BROADSIDE_DIR = "broadside"; // relative to .codecarto/ +export const BROADSIDE_STATE_FILE = "state.json"; +export const BROADSIDE_CONFIG_FILE = "config.yaml"; +export const BROADSIDE_STATE_SCHEMA_VERSION = 1; + +// Per-token pricing in USD (OpenRouter, google/gemini-3.7-flash:batch). +export const BROADSIDE_INPUT_PRICE_PER_M = 0.1875; +export const BROADSIDE_OUTPUT_PRICE_PER_M = 0.9375; + +export const BROADSIDE_LENS_IDS = [ + "architecture", + "api", + "security", + "defect", + "conventions", + "porting", +] as const; +export type BroadsideLensId = (typeof BROADSIDE_LENS_IDS)[number]; + +export const BROADSIDE_POLL_INTERVAL_MS = 15_000; +export const BROADSIDE_DEFAULT_POLL_BUDGET_MS = 25 * 60 * 1000; + +// ---------- types ---------- + +export type JsonSchemaDef = { + name: string; + strict: boolean; + schema: Record; +}; + +export type RepoInfo = { + name: string; + path: string; + language: string; + manifest: { path: string; content: string } | null; + mainFile: string; + readmeFirst: string; + fileTree: string; + fileCounts: Record; + sourceGlob: string; + sourceExts: string[]; +}; + +export type FileSlice = { + moduleName: string; + content: string; + fileCount: number; + chars: number; +}; + +export type BatchRequest = { + custom_id: string; + body: { + model: string; + messages: { role: "system" | "user"; content: string }[]; + response_format: { type: "json_schema"; json_schema: JsonSchemaDef }; + max_tokens: number; + }; +}; + +export type BatchTerminalStatus = "completed" | "failed" | "expired" | "cancelled"; + +export type BroadsideBatchEntry = { + batchId: string; + requests: number; + status: string; + submittedAt: string; + completedAt?: string; + estimatedCost: number; + cost?: number; + resultCount?: number; + error?: unknown; +}; + +export type BroadsideSynthesisEntry = { + batchId?: string; + status: "pending" | "submitted" | "completed" | "failed"; + cost?: number; +}; + +export type BroadsideRun = { + id: string; + createdAt: string; + model: string; + lenses: BroadsideLensId[]; + status: "in-flight" | "completed" | "partial" | "failed"; + outputDir: string; // relative to .codecarto/broadside/ + batches: Partial>; + synthesis: BroadsideSynthesisEntry; + totalCost?: number; +}; + +export type BroadsideStateFile = { + schema_version: number; + runs: BroadsideRun[]; +}; + +export type BroadsideConfig = { + model: string; + apiKey: string; + defaultLenses: BroadsideLensId[]; +}; + +export type BroadsideSubmitResult = { + runId: string; + outputDir: string; + batches: Partial>; + estimatedTotalCost: number; + estimatedInputTokens: number; + estimatedOutputTokens: number; +}; + +export type BroadsideCollectResult = { + runId: string; + status: string; + totalCost: number; + resultCount: number; + lensOutcomes: Partial>; + synthesis: BroadsideSynthesisEntry; + topFindings: { title: string; severity: string; sourceLens: string; summary: string }[]; +}; + +// ---------- JSON schemas (one per lens, plus synthesis) ---------- + +const SCHEMAS: Record = { + architecture: { + name: "architecture_report", + strict: true, + schema: { + type: "object", + properties: { + tech_stack: { + type: "object", + properties: { + language: { type: "string" }, + version: { type: "string" }, + build_system: { type: "string" }, + key_dependencies: { type: "array", items: { type: "string" } }, + }, + required: ["language", "build_system"], + additionalProperties: false, + }, + module_architecture: { + type: "array", + items: { + type: "object", + properties: { + name: { type: "string" }, + role: { type: "string" }, + file_count: { type: "integer" }, + depends_on: { type: "array", items: { type: "string" } }, + }, + required: ["name", "role"], + additionalProperties: false, + }, + }, + data_flow: { type: "string" }, + entry_points: { type: "array", items: { type: "string" } }, + notable_patterns: { type: "array", items: { type: "string" } }, + }, + required: ["tech_stack", "module_architecture", "data_flow", "entry_points"], + additionalProperties: false, + }, + }, + api_surface: { + name: "api_surface_report", + strict: true, + schema: { + type: "object", + properties: { + endpoints: { + type: "array", + items: { + type: "object", + properties: { + method: { type: "string" }, + path: { type: "string" }, + handler: { type: "string" }, + auth_required: { type: "boolean" }, + description: { type: "string" }, + }, + required: ["method", "path", "handler", "auth_required"], + additionalProperties: false, + }, + }, + data_types: { + type: "array", + items: { + type: "object", + properties: { + name: { type: "string" }, + kind: { type: "string" }, + fields_summary: { type: "string" }, + }, + required: ["name", "kind"], + additionalProperties: false, + }, + }, + authentication_flow: { type: "string" }, + error_handling: { type: "string" }, + }, + required: ["endpoints"], + additionalProperties: false, + }, + }, + security: { + name: "security_review_report", + strict: true, + schema: { + type: "object", + properties: { + findings: { + type: "array", + items: { + type: "object", + properties: { + severity: { type: "string", enum: ["critical", "high", "medium", "low"] }, + category: { type: "string" }, + title: { type: "string" }, + location: { type: "string" }, + description: { type: "string" }, + }, + required: ["severity", "title", "description"], + additionalProperties: false, + }, + }, + overall_assessment: { type: "string" }, + coverage_note: { type: "string" }, + }, + required: ["findings", "overall_assessment"], + additionalProperties: false, + }, + }, + defect_mechanical: { + name: "defect_scan_report", + strict: true, + schema: { + type: "object", + properties: { + module: { type: "string" }, + findings: { + type: "array", + items: { + type: "object", + properties: { + severity: { type: "string", enum: ["high", "medium", "low"] }, + pattern: { type: "string" }, + title: { type: "string" }, + location: { type: "string" }, + description: { type: "string" }, + suggestion: { type: "string" }, + }, + required: ["severity", "pattern", "title", "description"], + additionalProperties: false, + }, + }, + patterns_checked: { type: "array", items: { type: "string" } }, + files_scanned: { type: "integer" }, + overall_notes: { type: "string" }, + }, + required: ["module", "findings", "patterns_checked", "files_scanned"], + additionalProperties: false, + }, + }, + conventions: { + name: "conventions_report", + strict: true, + schema: { + type: "object", + properties: { + module: { type: "string" }, + naming_conventions: { + type: "object", + properties: { + packages: { type: "string" }, + types: { type: "string" }, + functions: { type: "string" }, + variables: { type: "string" }, + files: { type: "string" }, + tests: { type: "string" }, + }, + additionalProperties: false, + }, + error_handling_pattern: { type: "string" }, + logging_approach: { type: "string" }, + test_patterns: { type: "string" }, + code_organization: { type: "string" }, + idioms: { type: "array", items: { type: "string" } }, + inconsistencies: { + type: "array", + items: { + type: "object", + properties: { + description: { type: "string" }, + locations: { type: "array", items: { type: "string" } }, + }, + required: ["description"], + additionalProperties: false, + }, + }, + promotable_conventions: { + type: "array", + items: { + type: "object", + properties: { + title: { type: "string" }, + rule: { type: "string" }, + evidence: { type: "string" }, + }, + required: ["title", "rule"], + additionalProperties: false, + }, + }, + files_scanned: { type: "integer" }, + }, + required: ["module", "naming_conventions", "files_scanned"], + additionalProperties: false, + }, + }, + porting: { + name: "porting_surface_report", + strict: true, + schema: { + type: "object", + properties: { + module: { type: "string" }, + platform_coupling: { + type: "array", + items: { + type: "object", + properties: { + platform: { type: "string" }, + mechanisms: { type: "array", items: { type: "string" } }, + files: { type: "array", items: { type: "string" } }, + }, + required: ["platform", "mechanisms"], + additionalProperties: false, + }, + }, + external_dependencies: { + type: "array", + items: { + type: "object", + properties: { + name: { type: "string" }, + role: { type: "string" }, + replaceability: { type: "string" }, + }, + required: ["name"], + additionalProperties: false, + }, + }, + build_system_complexity: { type: "string" }, + porting_risk_areas: { + type: "array", + items: { + type: "object", + properties: { + area: { type: "string" }, + risk: { type: "string", enum: ["low", "medium", "high"] }, + notes: { type: "string" }, + }, + required: ["area", "risk"], + additionalProperties: false, + }, + }, + files_scanned: { type: "integer" }, + }, + required: ["module", "platform_coupling", "files_scanned"], + additionalProperties: false, + }, + }, + synthesis: { + name: "synthesis_report", + strict: true, + schema: { + type: "object", + properties: { + executive_summary: { type: "string" }, + severity_summary: { + type: "object", + properties: { + critical: { type: "integer" }, + high: { type: "integer" }, + medium: { type: "integer" }, + low: { type: "integer" }, + }, + required: ["critical", "high", "medium", "low"], + additionalProperties: false, + }, + top_findings: { + type: "array", + items: { + type: "object", + properties: { + title: { type: "string" }, + severity: { type: "string" }, + source_lens: { type: "string" }, + summary: { type: "string" }, + }, + required: ["title", "severity", "source_lens", "summary"], + additionalProperties: false, + }, + }, + module_assessments: { + type: "array", + items: { + type: "object", + properties: { + module: { type: "string" }, + quality_notes: { type: "string" }, + risk_level: { type: "string", enum: ["low", "medium", "high"] }, + }, + required: ["module", "risk_level"], + additionalProperties: false, + }, + }, + porting_readiness: { type: "string" }, + gaps_and_unknowns: { type: "array", items: { type: "string" } }, + coverage: { type: "string" }, + }, + required: ["executive_summary", "severity_summary", "top_findings"], + additionalProperties: false, + }, + }, +}; + +// ---------- lens definitions ---------- + +type LensDefinition = { + id: BroadsideLensId; + name: string; + description: string; + schemaName: string; + sliceBy: "none" | "directory"; + maxChars: number; + maxTokens: number; + // Test files rarely carry the surface a lens audits — they bulk up the + // batch and the bill. Convention extraction is the exception: it exists + // partly to catalog test patterns. + skipTestFiles?: boolean; + // Globs are matched against repo-relative forward-slash paths. + globsFor: (info: RepoInfo) => string[]; + systemPrompt: (info: RepoInfo) => string; + userPrompt: (info: RepoInfo, source: string, moduleName: string) => string; +}; + +const LENSES: Record = { + architecture: { + id: "architecture", + name: "Architecture, tech stack & module map", + description: "Repo-wide structural analysis from the manifest, entry point, README, and file tree.", + schemaName: "architecture", + sliceBy: "none", + maxChars: 0, // repo-info lens; no file slurping + maxTokens: 8000, + globsFor: () => [], + systemPrompt: () => + "You are a senior software architect performing a structural analysis of a " + + "codebase. You receive the project manifest, entry point, README excerpt, and " + + "file tree. Return a JSON object following the architecture_report schema " + + "exactly. All findings must be traceable to the provided files — cite file " + + "paths. If you can't determine something, say so rather than guessing.", + userPrompt: (info) => { + const manifest = info.manifest + ? `## ${info.manifest.path}\n\`\`\`\n${info.manifest.content}\n\`\`\`\n\n` + : "## Manifest\n[no manifest found]\n\n"; + return ( + "Analyze the architecture of this project.\n\n" + + manifest + + `## Entry point\n\`\`\`\n${info.mainFile || "[missing]"}\n\`\`\`\n\n` + + `## README (first 4000 chars)\n${info.readmeFirst || "[missing]"}\n\n` + + `## File tree (depth 3, capped)\n${info.fileTree || "[missing]"}\n\n` + + "## File counts by extension\n```json\n" + + JSON.stringify(info.fileCounts) + + "\n```\n\n" + + "Return the architecture_report JSON schema." + ); + }, + }, + api: { + id: "api", + name: "API surface audit", + description: "Endpoint catalog, request/response types, auth flow, error handling.", + schemaName: "api_surface", + sliceBy: "none", + maxChars: 70_000, + maxTokens: 8000, + skipTestFiles: true, + globsFor: (info) => + info.language === "go" + ? ["server/**/*.go", "server/*.go", "api/**/*.go", "api/*.go"] + : ["server/**", "api/**", "src/server/**", "src/api/**"], + systemPrompt: () => + "You are a senior API auditor. Given source files from an HTTP server, " + + "extract every HTTP endpoint (method, path, handler function, auth requirement) " + + "and every key request/response data type. Return a JSON object following the " + + "api_surface_report schema exactly. Cite specific file:line locations.", + userPrompt: (info, source, moduleName) => + "Extract the full API surface from these server source files:\n\n" + + source + + "\n\nReturn the api_surface_report JSON schema.", + }, + security: { + id: "security", + name: "Security review", + description: "Auth, authorization, input validation, TLS, secrets, trust boundaries.", + schemaName: "security", + sliceBy: "none", + maxChars: 70_000, + maxTokens: 8000, + skipTestFiles: true, + globsFor: (info) => + info.language === "go" + ? ["server/**/*.go", "server/*.go", "**/auth*.go", "**/middleware/**/*.go", "SECURITY.md"] + : ["server/**", "**/auth*", "**/middleware/**", "SECURITY.md"], + systemPrompt: () => + "You are a security engineer performing a first-pass review of a codebase. " + + "Given source files, identify potential security issues — focusing on " + + "authentication, authorization, input validation, TLS, secrets handling, " + + "and trust boundaries. Return a JSON object following the security_review_report " + + "schema. Rate severity as critical/high/medium/low. Be specific: cite file:line. " + + "If the provided files don't cover an area, state the gap in coverage_note.", + userPrompt: (info, source, moduleName) => + "Review these server source files for security issues:\n\n" + + source + + "\n\nReturn the security_review_report JSON schema.", + }, + defect: { + id: "defect", + name: "Mechanical defect scan", + description: "Nil derefs, error gaps, leaks, races, panics — pattern-based, sliced per module.", + schemaName: "defect_mechanical", + sliceBy: "directory", + maxChars: 60_000, + maxTokens: 6000, + globsFor: (info) => [info.sourceGlob], + systemPrompt: (info) => + `You are a senior code reviewer performing an automated defect scan on ${info.language} ` + + "source files. Look for these specific patterns:\n" + + " 1. Nil/null pointer dereference risks (unchecked returns, missing guards)\n" + + " 2. Error handling gaps (ignored errors, deferred errors unchecked)\n" + + " 3. Resource leaks (unclosed files, connections, goroutines without ctx)\n" + + " 4. Race conditions (shared state without sync, channel misuse)\n" + + " 5. Integer overflow/underflow in arithmetic or bounds\n" + + " 6. Unsafe type assertions without ok check\n" + + " 7. Panic-prone code (slice out of bounds, map access without ok)\n" + + " 8. Timezone/locale assumptions\n\n" + + "Return a JSON object following the defect_scan_report schema. " + + "Cite file:line for every finding. List which patterns you checked. " + + "If the code looks clean for a pattern, say so rather than staying silent. " + + "Prefer precision over volume — 3 solid findings beat 15 vague ones.", + userPrompt: (info, source, moduleName) => + `Scan this ${info.language} module for mechanical defects.\n\n` + + `Module: ${moduleName}\n\n` + + "## Source files\n\n" + + source + + "\n\nReturn the defect_scan_report JSON schema.", + }, + conventions: { + id: "conventions", + name: "Convention extraction", + description: "Naming, error handling, idioms, inconsistencies, promotable conventions.", + schemaName: "conventions", + sliceBy: "directory", + maxChars: 60_000, + maxTokens: 6000, + globsFor: (info) => [info.sourceGlob], + systemPrompt: () => + "You are a code style analyst extracting conventions from source files. " + + "Catalog: naming conventions per category (packages, types, functions, variables, " + + "source files, test files), the dominant error-handling pattern, logging approach, " + + "test organization patterns, file/package organization rules, and recurring idioms. " + + "Also flag inconsistencies — places where the same convention is violated. " + + "If you find well-established conventions worth formalizing, list them as " + + "promotable_conventions with a title, rule, and evidence from the code. " + + "Return a JSON object following the conventions_report schema.", + userPrompt: (info, source, moduleName) => + "Extract coding conventions from this module.\n\n" + + `Module: ${moduleName}\n\n` + + "## Source files\n\n" + + source + + "\n\nReturn the conventions_report JSON schema.", + }, + porting: { + id: "porting", + name: "Porting surface assessment", + description: "Platform coupling, external deps, build complexity, porting risk areas.", + schemaName: "porting", + sliceBy: "directory", + maxChars: 60_000, + maxTokens: 6000, + skipTestFiles: true, + globsFor: (info) => [ + info.sourceGlob, + "**/*.c", + "**/*.h", + "**/*.cpp", + "**/*.cc", + "**/*.m", + "**/*.mm", + "**/CMakeLists.txt", + "**/*.cmake", + "go.mod", + ], + systemPrompt: () => + "You are a software portability analyst. Examine source files and " + + "identify everything that ties this codebase to a specific platform, OS, " + + "architecture, or external dependency. Catalog: platform-specific build tags, " + + "FFI usage, OS-specific syscalls, external library bindings, and " + + "compile-time constants that encode platform assumptions. " + + "For each external dependency, note whether it could be replaced by a " + + "cross-platform alternative. Assess the build system complexity. " + + "Return a JSON object following the porting_surface_report schema.", + userPrompt: (info, source, moduleName) => + "Assess porting surface for this module.\n\n" + + `Module: ${moduleName}\n\n` + + "## Source files\n\n" + + source + + "\n\nReturn the porting_surface_report JSON schema.", + }, +}; + +export function getLens(lensId: BroadsideLensId): LensDefinition { + return LENSES[lensId]; +} + +export function listLenses(): LensDefinition[] { + return BROADSIDE_LENS_IDS.map((id) => LENSES[id]); +} + +// ---------- repo info ---------- + +const SKIP_DIR_NAMES = new Set([ + ".git", + ".github", + ".claude", + ".opencode", + ".codecarto", + "node_modules", + "vendor", + "dist", + "build", + "target", + "testdata", + "__pycache__", +]); + +const SKIP_FILE_EXTENSIONS = new Set([ + ".png", + ".jpg", + ".jpeg", + ".gif", + ".svg", + ".ico", + ".icns", + ".bmp", + ".webp", + ".mp3", + ".mp4", + ".mov", + ".avi", + ".wav", + ".ogg", + ".zip", + ".gz", + ".tar", + ".bz2", + ".xz", + ".7z", + ".pdf", + ".woff", + ".woff2", + ".ttf", + ".eot", + ".otf", + ".bin", + ".exe", + ".dll", + ".so", + ".dylib", + ".a", + ".o", + ".obj", + ".class", + ".jar", + ".war", + ".pyc", + ".wasm", + ".model", + ".bpe", +]); + +const MANIFEST_CANDIDATES = [ + ["go.mod", "go"], + ["package.json", "typescript"], + ["Cargo.toml", "rust"], + ["pyproject.toml", "python"], + ["setup.py", "python"], + ["requirements.txt", "python"], +]; + +const SOURCE_SPECS: Record = { + go: { glob: "**/*.go", exts: [".go"] }, + python: { glob: "**/*.py", exts: [".py"] }, + rust: { glob: "**/*.rs", exts: [".rs"] }, + typescript: { glob: "**/*.ts", exts: [".ts", ".tsx"] }, + javascript: { glob: "**/*.js", exts: [".js", ".jsx"] }, +}; + +async function listRepoFiles(targetDir: string): Promise { + // git ls-tree is the fast path; fall back to a bounded walk for non-git trees. + try { + const { stdout } = await execFileAsync("git", ["-C", targetDir, "ls-tree", "-r", "--name-only", "HEAD"], { + maxBuffer: 64 * 1024 * 1024, + }); + return stdout.split("\n").filter(Boolean); + } catch { + return walkFiles(targetDir, targetDir, 0, 30_000); + } +} + +async function walkFiles( + rootDir: string, + dir: string, + depth: number, + remaining: number, +): Promise { + if (remaining <= 0) return []; + let out: string[] = []; + let entries: import("node:fs").Dirent[] = []; + try { + entries = await readdir(dir, { withFileTypes: true }); + } catch { + return out; + } + for (const entry of entries) { + if (entry.name.startsWith(".") && entry.name !== ".github") continue; + if (entry.isDirectory()) { + if (SKIP_DIR_NAMES.has(entry.name)) continue; + if (depth > 8) continue; + const children = await walkFiles(rootDir, join(dir, entry.name), depth + 1, remaining - out.length); + out = out.concat(children); + } else if (entry.isFile()) { + const rel = join(dir, entry.name).slice(rootDir.length + 1).split("\\").join("/"); + out.push(rel); + } + } + return out; +} + +function detectLanguage(fileCounts: Record, manifestPath: string | null): string { + if (manifestPath) { + for (const [candidate, lang] of MANIFEST_CANDIDATES) { + if (manifestPath === candidate) return lang; + } + } + const counts: Record = { go: fileCounts[".go"] ?? 0, python: fileCounts[".py"] ?? 0, rust: fileCounts[".rs"] ?? 0, typescript: (fileCounts[".ts"] ?? 0) + (fileCounts[".tsx"] ?? 0) }; + const best = Object.entries(counts).sort((a, b) => b[1] - a[1])[0]; + return best && best[1] > 0 ? best[0] : "unknown"; +} + +export async function collectRepoInfo(targetDir: string): Promise { + const allFiles = await listRepoFiles(targetDir); + + const fileCounts: Record = {}; + for (const f of allFiles) { + const slash = f.lastIndexOf("/"); + const base = slash >= 0 ? f.slice(slash + 1) : f; + const dot = base.lastIndexOf("."); + const ext = dot > 0 ? base.slice(dot).toLowerCase() : "(no ext)"; + fileCounts[ext] = (fileCounts[ext] ?? 0) + 1; + } + const sortedCounts: Record = {}; + for (const [ext, n] of Object.entries(fileCounts).sort((a, b) => b[1] - a[1])) { + sortedCounts[ext] = n; + } + + let manifest: { path: string; content: string } | null = null; + for (const [candidate] of MANIFEST_CANDIDATES) { + const p = join(targetDir, candidate); + if (await pathExists(p)) { + try { + manifest = { path: candidate, content: await readFile(p, "utf8") }; + } catch { + manifest = null; + } + break; + } + } + + let mainFile = ""; + for (const candidate of ["main.go", "main.py", "src/main.rs", "src/index.ts", "index.ts"]) { + const p = join(targetDir, candidate); + if (await pathExists(p)) { + try { + mainFile = await readFile(p, "utf8"); + } catch { + mainFile = ""; + } + break; + } + } + + let readmeFirst = ""; + const readmePath = join(targetDir, "README.md"); + if (await pathExists(readmePath)) { + try { + readmeFirst = (await readFile(readmePath, "utf8")).slice(0, 4000); + } catch { + readmeFirst = ""; + } + } + + const fileTree = buildFileTree(allFiles); + + const language = detectLanguage(sortedCounts, manifest?.path ?? null); + const sourceSpec = SOURCE_SPECS[language] ?? SOURCE_SPECS.go; + const name = targetDir.split(/[\\/]/).filter(Boolean).pop() ?? "repo"; + + return { + name, + path: targetDir, + language, + manifest, + mainFile, + readmeFirst, + fileTree, + fileCounts: sortedCounts, + sourceGlob: sourceSpec.glob, + sourceExts: sourceSpec.exts, + }; +} + +function buildFileTree(allFiles: string[], maxDepth = 3, maxLines = 200): string { + const lines: string[] = []; + let count = 0; + for (const f of allFiles) { + if (f.split("/").length - 1 > maxDepth) continue; + if (f.startsWith(".git/") || f.startsWith(".github/")) continue; + if (f.endsWith(".sum") || f.endsWith(".lock")) continue; + lines.push(f); + count += 1; + if (count >= maxLines) { + lines.push(`... (${allFiles.length} total files, showing first ${maxLines})`); + break; + } + } + return lines.join("\n"); +} + +// ---------- glob matching & file slurping ---------- + +function globToRegExp(glob: string): RegExp { + let re = ""; + for (let i = 0; i < glob.length; i++) { + const c = glob[i]; + if (c === "*") { + if (glob[i + 1] === "*") { + // `**/` matches zero or more directories; a trailing `**` + // matches anything including slashes. + if (glob[i + 2] === "/") { + re += "(?:.*/)?"; + i += 2; + } else { + re += ".*"; + i += 1; + } + } else { + re += "[^/]*"; + } + } else if (c === "?") { + re += "[^/]"; + } else { + re += c.replace(/[.+^${}()|[\]\\]/g, "\\$&"); + } + } + return new RegExp(`^${re}$`); +} + +function matchesAnyGlob(path: string, globs: string[]): boolean { + for (const glob of globs) { + if (globToRegExp(glob).test(path)) return true; + } + return false; +} + +function isSlurpable(relPath: string): boolean { + const segments = relPath.split("/"); + for (const seg of segments) { + if (SKIP_DIR_NAMES.has(seg)) return false; + } + const slash = relPath.lastIndexOf("/"); + const base = slash >= 0 ? relPath.slice(slash + 1) : relPath; + const dot = base.lastIndexOf("."); + if (dot > 0 && SKIP_FILE_EXTENSIONS.has(base.slice(dot).toLowerCase())) return false; + return true; +} + +function sanitizeId(segment: string): string { + return segment.replace(/[^a-zA-Z0-9_-]+/g, "-").replace(/^-+|-+$/g, "") || "root"; +} + +function topLevelModule(relPath: string): string { + const slash = relPath.indexOf("/"); + return slash >= 0 ? relPath.slice(0, slash) : "root"; +} + +type CollectedFile = { relPath: string; moduleName: string }; + +function isTestFile(relPath: string): boolean { + const base = relPath.slice(relPath.lastIndexOf("/") + 1); + return /[._](test|spec)\.[a-z]+$/i.test(base) || base.includes("_test."); +} + +function collectLensFiles(allFiles: string[], lens: LensDefinition, info: RepoInfo): CollectedFile[] { + const globs = lens.globsFor(info); + if (globs.length === 0) return []; + const out: CollectedFile[] = []; + for (const f of allFiles) { + if (!isSlurpable(f)) continue; + if (lens.skipTestFiles && isTestFile(f)) continue; + if (!matchesAnyGlob(f, globs)) continue; + out.push({ relPath: f, moduleName: lens.sliceBy === "directory" ? topLevelModule(f) : info.name }); + } + return out; +} + +async function slurpFileList( + targetDir: string, + files: CollectedFile[], + maxChars: number, +): Promise { + const slices: FileSlice[] = []; + let currentModule = ""; + let parts: string[] = []; + let running = 0; + let fileCount = 0; + + const flush = () => { + if (parts.length === 0) return; + slices.push({ + moduleName: currentModule, + content: parts.join("\n"), + fileCount, + chars: running, + }); + parts = []; + running = 0; + fileCount = 0; + }; + + for (const file of files) { + let content = ""; + try { + content = await readFile(join(targetDir, file.relPath), "utf8"); + } catch { + content = "[BINARY or UNREADABLE]"; + } + const block = `=== ${file.relPath} ===\n${content}\n`; + + if (file.moduleName !== currentModule && parts.length > 0) { + flush(); + } + currentModule = file.moduleName; + + if (running + block.length > maxChars && parts.length > 0) { + // Slice is full: flush it and start another slice for the same module + // rather than truncating, so big modules get full coverage. + flush(); + currentModule = file.moduleName; + } + parts.push(block); + running += block.length; + fileCount += 1; + } + flush(); + return slices; +} + +export async function gatherSlices(targetDir: string, lens: LensDefinition, info: RepoInfo): Promise { + if (lens.sliceBy === "none" && lens.globsFor(info).length === 0) { + // Repo-info lens (architecture): the prompt is built from info alone. + return [{ moduleName: "root", content: "", fileCount: 0, chars: 0 }]; + } + const allFiles = await listRepoFiles(targetDir); + const files = collectLensFiles(allFiles, lens, info); + return slurpFileList(targetDir, files, lens.maxChars); +} + +// ---------- request building ---------- + +export function buildBatchRequest( + lens: LensDefinition, + info: RepoInfo, + slice: FileSlice, + index: number, + sliceCount: number, +): BatchRequest { + const moduleTag = sanitizeId(slice.moduleName); + const customId = sliceCount > 1 ? `${lens.id}-${moduleTag}-${index + 1}` : `${lens.id}-${moduleTag}`; + return { + custom_id: customId, + body: { + model: BROADSIDE_MODEL, + messages: [ + { role: "system", content: lens.systemPrompt(info) }, + { role: "user", content: lens.userPrompt(info, slice.content, slice.moduleName) }, + ], + response_format: { type: "json_schema", json_schema: SCHEMAS[lens.schemaName] }, + max_tokens: lens.maxTokens, + }, + }; +} + +export function estimateCost(lens: LensDefinition, slices: FileSlice[]): { + inputTokens: number; + outputTokens: number; + cost: number; +} { + const inputTokens = Math.ceil(slices.reduce((sum, s) => sum + (lens.maxChars === 0 ? 6000 : s.chars), 0) / 4); + const outputTokens = Math.ceil(lens.maxTokens * 0.75); + const cost = + (inputTokens / 1_000_000) * BROADSIDE_INPUT_PRICE_PER_M + + (outputTokens / 1_000_000) * BROADSIDE_OUTPUT_PRICE_PER_M; + return { inputTokens, outputTokens, cost }; +} + +// ---------- state & config ---------- + +export function broadsideDirFor(cwd: string): string { + return join(cwd, ".codecarto", BROADSIDE_DIR); +} + +export function defaultBroadsideState(): BroadsideStateFile { + return { schema_version: BROADSIDE_STATE_SCHEMA_VERSION, runs: [] }; +} + +export async function loadBroadsideState(broadsideDir: string): Promise { + const statePath = join(broadsideDir, BROADSIDE_STATE_FILE); + if (!(await pathExists(statePath))) return defaultBroadsideState(); + try { + const raw = JSON.parse(await readFile(statePath, "utf8")); + if (!raw || typeof raw !== "object" || !Array.isArray(raw.runs)) return defaultBroadsideState(); + return raw as BroadsideStateFile; + } catch { + return defaultBroadsideState(); + } +} + +export async function saveBroadsideState(broadsideDir: string, state: BroadsideStateFile): Promise { + await mkdir(broadsideDir, { recursive: true }); + await writeFile(join(broadsideDir, BROADSIDE_STATE_FILE), `${JSON.stringify(state, null, "\t")}\n`, "utf8"); +} + +export async function loadBroadsideConfig(broadsideDir: string): Promise { + const configPath = join(broadsideDir, BROADSIDE_CONFIG_FILE); + let raw: Record = {}; + if (await pathExists(configPath)) { + try { + raw = (await loadYamlFile>(configPath)) ?? {}; + } catch { + raw = {}; + } + } + const lenses = Array.isArray(raw.default_lenses) + ? (raw.default_lenses.filter((l): l is BroadsideLensId => BROADSIDE_LENS_IDS.includes(l as BroadsideLensId))) + : []; + return { + model: typeof raw.model === "string" && raw.model.trim() ? raw.model.trim() : BROADSIDE_MODEL, + apiKey: typeof raw.api_key === "string" ? raw.api_key.trim() : "", + defaultLenses: lenses.length > 0 ? lenses : [...BROADSIDE_LENS_IDS], + }; +} + +// ---------- batch client ---------- + +export type FetchLike = (url: string, init: Record) => Promise; + +export async function submitBatch( + batchRequests: BatchRequest[], + apiKey: string, + fetcher: FetchLike = fetch as FetchLike, +): Promise<{ batchId: string; status: string; error?: unknown }> { + // The OpenRouter batch endpoint stream-parses the body and requires + // `endpoint` and `model` to serialize before `requests` — key order matters. + const payload = { + endpoint: "/v1/chat/completions", + model: BROADSIDE_MODEL, + requests: batchRequests, + }; + const resp = await fetcher(BROADSIDE_BATCH_URL, { + method: "POST", + headers: { + Authorization: `Bearer ${apiKey}`, + "Content-Type": "application/json", + }, + body: JSON.stringify(payload), + signal: AbortSignal.timeout(30_000), + }); + const data = (await resp.json()) as Record; + if (resp.status !== 202) { + return { batchId: "", status: "rejected", error: data }; + } + return { batchId: String(data.id), status: String(data.status) }; +} + +export async function fetchBatch( + batchId: string, + apiKey: string, + fetcher: FetchLike = fetch as FetchLike, +): Promise> { + const resp = await fetcher(`${BROADSIDE_BATCH_URL}/${batchId}`, { + method: "GET", + headers: { Authorization: `Bearer ${apiKey}` }, + signal: AbortSignal.timeout(30_000), + }); + return (await resp.json()) as Record; +} + +export async function pollBatchUntilTerminal( + batchId: string, + apiKey: string, + opts: { + deadlineMs?: number; + onStatus?: (status: string, counts: Record) => void; + fetcher?: FetchLike; + } = {}, +): Promise> { + const deadline = Date.now() + (opts.deadlineMs ?? BROADSIDE_DEFAULT_POLL_BUDGET_MS); + const fetcher = opts.fetcher ?? (fetch as FetchLike); + for (;;) { + let batch: Record; + try { + batch = await fetchBatch(batchId, apiKey, fetcher); + } catch { + if (Date.now() >= deadline) return { id: batchId, status: "timeout" }; + await sleep(BROADSIDE_POLL_INTERVAL_MS); + continue; + } + const status = String(batch.status ?? "unknown"); + const counts = (batch.request_counts ?? {}) as Record; + opts.onStatus?.(status, counts); + if (["completed", "failed", "expired", "cancelled"].includes(status)) return batch; + if (Date.now() >= deadline) return { id: batchId, status: "timeout" }; + await sleep(BROADSIDE_POLL_INTERVAL_MS); + } +} + +// ---------- run orchestration ---------- + +export async function runBroadsideSubmit( + cwd: string, + apiKey: string, + opts: { lenses?: BroadsideLensId[]; fetcher?: FetchLike } = {}, +): Promise { + const info = await collectRepoInfo(cwd); + const lensIds = opts.lenses ?? BROADSIDE_LENS_IDS; + const broadsideDir = broadsideDirFor(cwd); + const state = await loadBroadsideState(broadsideDir); + const runId = new Date().toISOString().replace(/[:.]/g, "-"); + const run: BroadsideRun = { + id: runId, + createdAt: new Date().toISOString(), + model: BROADSIDE_MODEL, + lenses: [...lensIds], + status: "in-flight", + outputDir: runId, + batches: {}, + synthesis: { status: "pending" }, + }; + state.runs.push(run); + await saveBroadsideState(broadsideDir, state); + + let estimatedInputTokens = 0; + let estimatedOutputTokens = 0; + let estimatedTotalCost = 0; + + const submissions: Promise[] = []; + for (const lensId of lensIds) { + const lens = getLens(lensId); + const slices = await gatherSlices(cwd, lens, info); + const requests = slices.map((s, i) => buildBatchRequest(lens, info, s, i, slices.length)); + const estimate = estimateCost(lens, slices); + estimatedInputTokens += estimate.inputTokens; + estimatedOutputTokens += estimate.outputTokens; + estimatedTotalCost += estimate.cost; + + const entry: BroadsideBatchEntry = { + batchId: "", + requests: requests.length, + status: "submitting", + submittedAt: new Date().toISOString(), + estimatedCost: estimate.cost, + }; + run.batches[lensId] = entry; + + submissions.push( + (async () => { + const { batchId, status, error } = await submitBatch(requests, apiKey, opts.fetcher); + entry.batchId = batchId; + entry.status = status; + if (error) entry.error = error; + })(), + ); + } + await Promise.allSettled(submissions); + await saveBroadsideState(broadsideDir, state); + + return { + runId, + outputDir: join(".codecarto", BROADSIDE_DIR, runId), + batches: run.batches, + estimatedTotalCost, + estimatedInputTokens, + estimatedOutputTokens, + }; +} + +export type StoredLensResult = { + lensId: BroadsideLensId; + customId: string; + moduleName: string; + content: string; + raw: Record; +}; + +function extractContent(result: Record): string | null { + const response = result.response as Record | undefined; + if (!response?.body) return null; + const body = response.body as Record; + const choices = body.choices as Array> | undefined; + const message = choices?.[0]?.message as Record | undefined; + return typeof message?.content === "string" ? message.content : null; +} + +export async function saveLensResults( + runDir: string, + lensId: BroadsideLensId, + batch: Record, +): Promise { + const results = Array.isArray(batch.results) ? (batch.results as Array>) : []; + const out: StoredLensResult[] = []; + for (const result of results) { + const customId = String(result.custom_id ?? "unknown"); + const content = extractContent(result); + if (content === null) { + if (result.error) { + await writeFile(join(runDir, `${sanitizeId(customId)}.error.json`), `${JSON.stringify(result.error, null, "\t")}\n`, "utf8"); + } + continue; + } + await writeFile(join(runDir, `${sanitizeId(customId)}.json`), `${content}\n`, "utf8"); + await writeFile(join(runDir, `${sanitizeId(customId)}.md`), renderFindingsMarkdown(content), "utf8"); + out.push({ lensId, customId, moduleName: String(customId).replace(/^[a-z]+-/, ""), content, raw: result }); + } + return out; +} + +export async function runBroadsideCollect( + cwd: string, + apiKey: string, + opts: { + waitMs?: number; + includeSynthesis?: boolean; + onStatus?: (lensId: string, status: string, counts: Record) => void; + fetcher?: FetchLike; + } = {}, +): Promise { + const broadsideDir = broadsideDirFor(cwd); + const state = await loadBroadsideState(broadsideDir); + const run = state.runs[state.runs.length - 1]; + if (!run) { + throw new Error("No Broad-Side run recorded. Call codecarto_broadside with action 'submit' first."); + } + + const runDir = join(broadsideDir, run.outputDir); + await mkdir(runDir, { recursive: true }); + + const deadline = Date.now() + (opts.waitMs ?? BROADSIDE_DEFAULT_POLL_BUDGET_MS); + let totalCost = 0; + let resultCount = 0; + const lensOutcomes: BroadsideCollectResult["lensOutcomes"] = {}; + + const allLensResults: StoredLensResult[] = []; + + for (const lensId of run.lenses) { + const entry = run.batches[lensId]; + if (!entry || !entry.batchId) { + lensOutcomes[lensId] = { status: entry?.status ?? "failed", resultCount: 0 }; + continue; + } + if (["completed", "failed", "expired", "cancelled"].includes(entry.status)) { + totalCost += entry.cost ?? 0; + resultCount += entry.resultCount ?? 0; + lensOutcomes[lensId] = { status: entry.status, cost: entry.cost, resultCount: entry.resultCount }; + continue; + } + + const batch = await pollBatchUntilTerminal(entry.batchId, apiKey, { + deadlineMs: Math.max(0, deadline - Date.now()), + onStatus: (status, counts) => opts.onStatus?.(lensId, status, counts), + fetcher: opts.fetcher, + }); + + const status = String(batch.status ?? "unknown"); + entry.status = status; + if (status === "completed") { + const usage = (batch.usage ?? {}) as Record; + const cost = typeof usage.cost === "number" ? usage.cost : undefined; + entry.cost = cost; + entry.completedAt = new Date().toISOString(); + const stored = await saveLensResults(runDir, lensId, batch); + entry.resultCount = stored.length; + allLensResults.push(...stored); + resultCount += stored.length; + totalCost += cost ?? 0; + await writeFile( + join(runDir, `raw-${lensId}.json`), + `${JSON.stringify(batch, null, "\t")}\n`, + "utf8", + ); + } else if (batch.error) { + entry.error = batch.error; + } + lensOutcomes[lensId] = { status, cost: entry.cost, resultCount: entry.resultCount }; + await saveBroadsideState(broadsideDir, state); + } + + // Synthesis: one cross-lens report, only after every lens batch is terminal. + let topFindings: BroadsideCollectResult["topFindings"] = []; + if (opts.includeSynthesis !== false && allLensResults.length > 0) { + const allTerminal = run.lenses.every((lensId) => { + const entry = run.batches[lensId]; + return entry && ["completed", "failed", "expired", "cancelled"].includes(entry.status); + }); + if (allTerminal && run.synthesis.status === "pending") { + const findingsText = allLensResults + .map((r) => `## ${r.lensId} — ${r.customId}\n\n${r.content}\n`) + .join("\n"); + const request: BatchRequest = { + custom_id: "synthesis", + body: { + model: BROADSIDE_MODEL, + messages: [ + { + role: "system", + content: + "You are a technical editor synthesizing multiple analysis reports about a single " + + "codebase into one coherent summary. The reports come from different lenses — " + + "architecture, API surface, security review, defect scanning, convention extraction, " + + "and porting assessment. Cross-reference findings across lenses: if a security issue " + + "also appears as a defect, merge them. Produce a JSON object following the " + + "synthesis_report schema. Prioritize the most actionable findings. " + + "Be honest about gaps — if a lens found nothing, say 'no issues found' rather than " + + "inventing problems. These are scouting signals from a batch model, not verified " + + "claims; note that in the summary.", + }, + { + role: "user", + content: + "Synthesize these analysis reports into a single summary.\n\n" + + findingsText + + "\n\nReturn the synthesis_report JSON schema.", + }, + ], + response_format: { type: "json_schema", json_schema: SCHEMAS.synthesis }, + max_tokens: 12_000, + }, + }; + run.synthesis.status = "submitted"; + await saveBroadsideState(broadsideDir, state); + const { batchId, error } = await submitBatch([request], apiKey, opts.fetcher); + if (error) { + run.synthesis.status = "failed"; + } else { + run.synthesis.batchId = batchId; + const batch = await pollBatchUntilTerminal(batchId, apiKey, { + deadlineMs: BROADSIDE_DEFAULT_POLL_BUDGET_MS, + onStatus: (status, counts) => opts.onStatus?.("synthesis", status, counts), + fetcher: opts.fetcher, + }); + if (batch.status === "completed") { + const usage = (batch.usage ?? {}) as Record; + const cost = typeof usage.cost === "number" ? usage.cost : undefined; + run.synthesis.status = "completed"; + run.synthesis.cost = cost; + totalCost += cost ?? 0; + const results = Array.isArray(batch.results) ? (batch.results as Array>) : []; + const content = results.length > 0 ? extractContent(results[0]) : null; + if (content !== null) { + await writeFile(join(runDir, "synthesis.json"), `${content}\n`, "utf8"); + await writeFile(join(runDir, "synthesis.md"), renderFindingsMarkdown(content), "utf8"); + topFindings = parseSynthesisTopFindings(content); + } + } else if (batch.error) { + run.synthesis.status = "failed"; + } + } + } + } + + const terminal = run.lenses.every((lensId) => { + const entry = run.batches[lensId]; + return entry && ["completed", "failed", "expired", "cancelled"].includes(entry.status); + }); + run.status = terminal ? (resultCount > 0 ? "completed" : "failed") : "partial"; + run.totalCost = totalCost; + await saveBroadsideState(broadsideDir, state); + + await writeFile( + join(runDir, "run-meta.json"), + `${JSON.stringify( + { + experimental: true, + method: "Broad-Side (OpenRouter Batch API)", + model: BROADSIDE_MODEL, + run_id: run.id, + created_at: run.createdAt, + status: run.status, + total_cost: totalCost, + result_count: resultCount, + lenses: run.lenses, + disclaimer: + "Findings are unverified scouting signals from a batch model, not validated claims. " + + "Re-verify every file:line lead with the interactive pipeline or by hand.", + }, + null, + "\t", + )}\n`, + "utf8", + ); + + return { runId: run.id, status: run.status, totalCost, resultCount, lensOutcomes, synthesis: run.synthesis, topFindings }; +} + +export async function runBroadsideStatus(cwd: string): Promise<{ state: BroadsideStateFile }> { + const broadsideDir = broadsideDirFor(cwd); + const state = await loadBroadsideState(broadsideDir); + return { state }; +} + +// ---------- rendering ---------- + +export function renderFindingsMarkdown(content: string): string { + let parsed: unknown; + try { + parsed = JSON.parse(content); + } catch { + return content; + } + return formatAsMarkdown(parsed); +} + +function formatAsMarkdown(value: unknown, depth = 0): string { + const indent = "\t".repeat(depth); + if (Array.isArray(value)) { + const lines: string[] = []; + for (let i = 0; i < value.length; i++) { + const item = value[i] as Record; + if (item && typeof item === "object") { + const title = (item.title ?? item.name ?? item.module ?? item.area ?? item.platform ?? "") as string; + lines.push(`${indent}${i + 1}. ${title}`); + lines.push(formatAsMarkdown(item, depth + 1)); + } else { + lines.push(`${indent}- ${String(item)}`); + } + } + return lines.join("\n"); + } + if (value && typeof value === "object") { + const lines: string[] = []; + for (const [key, entryValue] of Object.entries(value as Record)) { + if (entryValue && typeof entryValue === "object") { + lines.push(`${indent}**${key}**:`); + lines.push(formatAsMarkdown(entryValue, depth + 1)); + } else { + lines.push(`${indent}- **${key}**: ${String(entryValue)}`); + } + } + return lines.join("\n"); + } + return `${indent}${String(value)}`; +} + +function parseSynthesisTopFindings( + content: string, +): BroadsideCollectResult["topFindings"] { + try { + const parsed = JSON.parse(content) as Record; + const findings = Array.isArray(parsed.top_findings) + ? (parsed.top_findings as Array>) + : []; + return findings + .filter((f) => typeof f.title === "string") + .map((f) => ({ + title: String(f.title), + severity: String(f.severity ?? "unknown"), + sourceLens: String(f.source_lens ?? "unknown"), + summary: String(f.summary ?? ""), + })); + } catch { + return []; + } +} + +// ---------- formatting helpers for tool output ---------- + +export function estimateSubmitText(result: BroadsideSubmitResult, lenses: LensDefinition[]): string { + const lines = [ + `Broad-Side submitted ${result.batches ? Object.keys(result.batches).length : 0} batch(es).`, + ]; + for (const lens of lenses) { + const entry = result.batches[lens.id]; + if (!entry) continue; + const status = entry.batchId ? `batch ${entry.batchId}` : entry.status; + lines.push(` ${lens.name}: ${status} (${entry.requests} request(s), ~$${entry.estimatedCost.toFixed(4)})`); + } + lines.push( + `Estimated total: ~$${result.estimatedTotalCost.toFixed(4)}`, + `Results will land in ${result.outputDir}/`, + "Call codecarto_broadside with action 'collect' once batches finish, or pass wait_seconds on submit to block.", + "Disclaimer: Broad-Side findings are unverified scouting signals from a batch model, not validated claims.", + ); + return lines.join("\n"); +} + +export function collectResultText(result: BroadsideCollectResult): string { + const lines = [ + `Broad-Side run ${result.runId}: ${result.status}`, + ` Results: ${result.resultCount} | Total cost: $${result.totalCost.toFixed(6)}`, + ]; + for (const lensId of BROADSIDE_LENS_IDS) { + const outcome = result.lensOutcomes[lensId]; + if (!outcome) continue; + lines.push( + ` ${lensId}: ${outcome.status}` + + (outcome.cost !== undefined ? `, $${outcome.cost.toFixed(6)}` : "") + + (outcome.resultCount !== undefined ? `, ${outcome.resultCount} result(s)` : ""), + ); + } + if (result.synthesis.status === "completed") { + lines.push(` synthesis: completed, $${(result.synthesis.cost ?? 0).toFixed(6)}`); + if (result.topFindings.length > 0) { + lines.push("", "Top findings (unverified leads):"); + for (const f of result.topFindings.slice(0, 10)) { + lines.push(` [${f.severity}] ${f.title}`); + } + } + } + lines.push("", "Disclaimer: Broad-Side findings are unverified scouting signals from a batch model, not validated claims."); + return lines.join("\n"); +} + +export function statusText(state: BroadsideStateFile): string { + if (state.runs.length === 0) { + return "No Broad-Side runs recorded. Call codecarto_broadside with action 'submit' first."; + } + const lines: string[] = []; + for (const run of [...state.runs].reverse().slice(0, 3)) { + lines.push(`Run ${run.id} — ${run.status}`); + for (const lensId of BROADSIDE_LENS_IDS) { + const entry = run.batches[lensId]; + if (!entry) continue; + lines.push(` ${lensId}: ${entry.status}${entry.batchId ? ` (${entry.batchId})` : ""}${entry.cost !== undefined ? `, $${entry.cost.toFixed(6)}` : ""}`); + } + lines.push(` synthesis: ${run.synthesis.status}`); + if (run.totalCost !== undefined) lines.push(` total cost: $${run.totalCost.toFixed(6)}`); + } + return lines.join("\n"); +} diff --git a/core/index.ts b/core/index.ts index 9ee3dcc..b5f4ed3 100644 --- a/core/index.ts +++ b/core/index.ts @@ -17,3 +17,4 @@ export * from "./guide.ts"; export * from "./dashboard.ts"; export * from "./library.ts"; export * from "./synthesis.ts"; +export * from "./broadside.ts"; diff --git a/core/workspace.ts b/core/workspace.ts index 39a80bd..9f31fe1 100644 --- a/core/workspace.ts +++ b/core/workspace.ts @@ -141,7 +141,9 @@ export async function seedOrchestratorFiles(workspaceDir: string): Promise entry !== BROADSIDE_DIR); + broadsideOnly = entries.length === 0 && (await pathExists(join(targetWorkspaceDir, BROADSIDE_DIR))); + } + + if (targetExists && !sameWorkspace && !broadsideOnly) { if (!args.force) { throw new McpError( ErrorCode.InvalidRequest, @@ -174,6 +196,10 @@ export async function handleInit(args: { cwd: string; pipeline?: string; force?: if (!(await pathExists(targetWorkspaceDir))) { await mkdir(cwd, { recursive: true }); await cp(packagedWorkspaceDir, targetWorkspaceDir, { recursive: true }); + } else if (broadsideOnly) { + // Merge the template into the scout-only .codecarto/, preserving the + // broadside state and results already on disk. + await cp(packagedWorkspaceDir, targetWorkspaceDir, { recursive: true }); } const statusPath = join(targetWorkspaceDir, "workflow", "status.yaml"); @@ -950,6 +976,96 @@ export async function handleAmend(args: { cwd: string; name: string }) { }); } +// ---------- broadside (batch reconnaissance) ---------- + +function resolveBroadsideApiKey(explicit: string | undefined, config: { apiKey: string }): string { + if (explicit && explicit.trim()) return explicit.trim(); + const fromEnv = process.env.OPENROUTER_API_KEY?.trim(); + if (fromEnv) return fromEnv; + if (config.apiKey) return config.apiKey; + throw new McpError( + ErrorCode.InvalidParams, + "No OpenRouter API key found. Pass api_key, set the OPENROUTER_API_KEY environment variable, or add api_key to .codecarto/broadside/config.yaml.", + ); +} + +export async function handleBroadside(args: { + cwd: string; + action: "submit" | "collect" | "status"; + lenses?: string[]; + api_key?: string; + wait_seconds?: number; + include_synthesis?: boolean; +}) { + const cwd = await validateCwd(args.cwd); + const action = args.action ?? "submit"; + if (!["submit", "collect", "status"].includes(action)) { + throw new McpError(ErrorCode.InvalidParams, `Unknown action: ${action}. Valid actions: submit, collect, status.`); + } + + const config = await loadBroadsideConfig(broadsideDirFor(cwd)); + + if (action === "status") { + const { state } = await runBroadsideStatus(cwd); + return textResult(statusText(state), { state }); + } + + const apiKey = resolveBroadsideApiKey(args.api_key, config); + const waitMs = typeof args.wait_seconds === "number" && args.wait_seconds > 0 ? args.wait_seconds * 1000 : undefined; + + if (action === "submit") { + let lenses: BroadsideLensId[]; + if (args.lenses && args.lenses.length > 0) { + const unknown = args.lenses.filter((l) => !BROADSIDE_LENS_IDS.includes(l as BroadsideLensId)); + if (unknown.length > 0) { + throw new McpError(ErrorCode.InvalidParams, `Unknown lens(es): ${unknown.join(", ")}. Valid: ${BROADSIDE_LENS_IDS.join(", ")}`); + } + lenses = args.lenses as BroadsideLensId[]; + } else { + lenses = config.defaultLenses; + } + + const result = await runBroadsideSubmit(cwd, apiKey, { lenses }).catch((error) => { + throw new McpError(ErrorCode.InvalidRequest, error instanceof Error ? error.message : String(error)); + }); + + const lines = [estimateSubmitText(result, lenses.map(getLens))]; + if (waitMs) { + lines.push("", "Waiting for batches to complete..."); + const collect = await runBroadsideCollect(cwd, apiKey, { + waitMs, + includeSynthesis: args.include_synthesis !== false, + onStatus: (lensId, status, counts) => + lines.push(` ${lensId}: ${status} (${counts.completed ?? 0}/${counts.total ?? "?"})`), + }); + lines.push("", collectResultText(collect)); + } + return textResult(lines.join("\n"), { + runId: result.runId, + outputDir: result.outputDir, + batches: result.batches, + estimatedTotalCost: result.estimatedTotalCost, + }); + } + + // action === "collect" + const collect = await runBroadsideCollect(cwd, apiKey, { + waitMs, + includeSynthesis: args.include_synthesis !== false, + }).catch((error) => { + throw new McpError(ErrorCode.InvalidRequest, error instanceof Error ? error.message : String(error)); + }); + return textResult(collectResultText(collect), { + runId: collect.runId, + status: collect.status, + totalCost: collect.totalCost, + resultCount: collect.resultCount, + lensOutcomes: collect.lensOutcomes, + synthesis: collect.synthesis, + topFindings: collect.topFindings, + }); +} + // ---------- tool registry ---------- const TOOLS = [ @@ -1238,6 +1354,40 @@ const TOOLS = [ required: ["cwd"], }, }, + { + name: "codecarto_broadside", + description: + "Broad-Side: fire a cheap batch reconnaissance scan at a repository via the OpenRouter Batch API. Six lenses (architecture, api, security, defect, conventions, porting) run as asynchronous single-turn prompts with structured JSON schemas; results land in .codecarto/broadside// as JSON plus markdown, with an optional cross-lens synthesis report. Works on any git repository — no CodeCartographer workspace required. Requires an OpenRouter API key (api_key param, OPENROUTER_API_KEY env var, or .codecarto/broadside/config.yaml). Findings are unverified scouting signals from a batch model, not validated claims — they tell the interactive pipeline where to look. Actions: submit (fire batches, returns batch ids and cost estimate), collect (poll to completion, save results, optionally synthesize), status (show recorded runs).", + inputSchema: { + type: "object", + properties: { + cwd: { type: "string", description: "Absolute path to the target repository." }, + action: { + type: "string", + enum: ["submit", "collect", "status"], + description: "submit fires all lens batches and returns batch ids; collect polls submitted batches, saves results, and optionally runs the synthesis pass; status shows recorded runs.", + }, + lenses: { + type: "array", + items: { type: "string", enum: [...BROADSIDE_LENS_IDS] }, + description: "Lenses to run (submit only). Defaults to all six.", + }, + api_key: { + type: "string", + description: "OpenRouter API key. Prefer the OPENROUTER_API_KEY environment variable or .codecarto/broadside/config.yaml.", + }, + wait_seconds: { + type: "number", + description: "For submit: after submitting, poll up to this many seconds before returning. For collect: poll up to this many seconds before returning with partial state.", + }, + include_synthesis: { + type: "boolean", + description: "Run the cross-lens synthesis pass once all lens batches complete (default true).", + }, + }, + required: ["cwd", "action"], + }, + }, ] as const; const HANDLERS: Record Promise> = { @@ -1262,6 +1412,7 @@ const HANDLERS: Record Promise> = { codecarto_dashboard: handleDashboard, codecarto_list_skills: handleListSkills, codecarto_guide: handleGuide, + codecarto_broadside: handleBroadside, }; export async function handleGuide(args: { topic?: string }) { diff --git a/scripts/smoke-broadside.mjs b/scripts/smoke-broadside.mjs new file mode 100644 index 0000000..c2495ea --- /dev/null +++ b/scripts/smoke-broadside.mjs @@ -0,0 +1,52 @@ +// Broad-Side smoke test: drives the MCP handler through a real submit → +// collect → synthesis cycle against a live target repository using the +// OpenRouter Batch API. +// +// This spends real money (~$0.03 for the default two lenses). It is therefore +// opt-in: it skips cleanly unless OPENROUTER_API_KEY is set and a target is +// passed on the command line. +// +// OPENROUTER_API_KEY=sk-or-... node scripts/smoke-broadside.mjs /path/to/repo + +import assert from "node:assert/strict"; +import { dirname, join, resolve } from "node:path"; +import { fileURLToPath, pathToFileURL } from "node:url"; + +const REPO_ROOT = resolve(dirname(fileURLToPath(import.meta.url)), ".."); +const target = process.argv[2]; + +if (!process.env.OPENROUTER_API_KEY) { + console.log("SKIP: OPENROUTER_API_KEY not set — broadside smoke spends real money and is opt-in."); + process.exit(0); +} +if (!target) { + console.error("Usage: OPENROUTER_API_KEY=sk-or-... node scripts/smoke-broadside.mjs /path/to/repo"); + process.exit(1); +} + +const { handleBroadside } = await import(pathToFileURL(join(REPO_ROOT, "mcp-server/server.ts")).href); + +async function step(label, fn) { + console.log(`\n== ${label} ==`); + const result = await fn(); + console.log(result.content[0].text); + return result; +} + +const submit = await step("submit (architecture + api)", () => + handleBroadside({ cwd: target, action: "submit", lenses: ["architecture", "api"], api_key: process.env.OPENROUTER_API_KEY }), +); +assert.equal(submit.structuredContent.runId.length > 0, true, "submit must return a run id"); + +const status = await step("status", () => handleBroadside({ cwd: target, action: "status" })); +assert.match(status.content[0].text, /architecture/); + +const collect = await step("collect (polls to completion, synthesizes)", () => + handleBroadside({ cwd: target, action: "collect", api_key: process.env.OPENROUTER_API_KEY }), +); +assert.equal(collect.structuredContent.status, "completed", "collect must reach completed"); +assert.ok(collect.structuredContent.resultCount > 0, "collect must save at least one result"); +assert.equal(collect.structuredContent.synthesis.status, "completed", "synthesis must complete"); +assert.ok(collect.structuredContent.topFindings.length > 0, "synthesis must produce top findings"); + +console.log("\nSMOKE PASSED"); diff --git a/scripts/smoke-mcp.mjs b/scripts/smoke-mcp.mjs index 391a996..9b38c14 100644 --- a/scripts/smoke-mcp.mjs +++ b/scripts/smoke-mcp.mjs @@ -87,6 +87,7 @@ async function setupFixture() { const EXPECTED_TOOLS = [ "codecarto_amend", + "codecarto_broadside", "codecarto_complete", "codecarto_config", "codecarto_dashboard", diff --git a/tests/broadside.test.mjs b/tests/broadside.test.mjs new file mode 100644 index 0000000..93c306f --- /dev/null +++ b/tests/broadside.test.mjs @@ -0,0 +1,390 @@ +// Broad-Side unit tests. No network: the batch client is exercised through an +// injected fake fetcher, and file collection runs against temp fixtures built +// in-memory. The real OpenRouter API is covered by a manual smoke path, not +// this suite. + +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { mkdtemp, mkdir, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { dirname, join, resolve } from "node:path"; +import { fileURLToPath, pathToFileURL } from "node:url"; + +const REPO_ROOT = resolve(dirname(fileURLToPath(import.meta.url)), ".."); +const core = await import(pathToFileURL(join(REPO_ROOT, "core/index.ts")).href); + +const { + BROADSIDE_LENS_IDS, + BROADSIDE_MODEL, + buildBatchRequest, + collectRepoInfo, + defaultBroadsideState, + estimateCost, + gatherSlices, + getLens, + loadBroadsideConfig, + loadBroadsideState, + listLenses, + renderFindingsMarkdown, + runBroadsideCollect, + runBroadsideStatus, + runBroadsideSubmit, + saveBroadsideState, + submitBatch, +} = core; + +// ---------- lens registry ---------- + +test("the registry carries six lenses with unique schema names", () => { + const lenses = listLenses(); + assert.equal(lenses.length, 6); + assert.deepEqual( + lenses.map((l) => l.id).sort(), + [...BROADSIDE_LENS_IDS].sort(), + ); + const schemaNames = new Set(); + for (const lens of lenses) { + schemaNames.add(lens.schemaName); + assert.ok(lens.name.length > 5, "every lens needs a human-readable name"); + assert.ok(lens.systemPrompt({ language: "go" }).length > 100); + } + assert.equal(schemaNames.size, 6, "each lens must declare its own schema"); +}); + +// ---------- file collection ---------- + +async function makeFixture() { + const dir = await mkdtemp(join(tmpdir(), "broadside-fixture-")); + await mkdir(join(dir, "server"), { recursive: true }); + await mkdir(join(dir, "model", "deep"), { recursive: true }); + await writeFile(join(dir, "go.mod"), "module example.com/fixture\n\ngo 1.26.0\n"); + await writeFile(join(dir, "main.go"), "package main\n\nfunc main() {}\n"); + await writeFile(join(dir, "server", "routes.go"), "package server\n\n// GET /api/version\nfunc routes() {}\n"); + await writeFile(join(dir, "server", "auth.go"), "package server\n\nfunc auth() {}\n"); + await writeFile(join(dir, "model", "core.go"), "package model\n\nfunc core() {}\n"); + await writeFile(join(dir, "model", "deep", "nested.go"), "package deep\n\nfunc nested() {}\n"); + await writeFile(join(dir, "README.md"), "# Fixture\n"); + return dir; +} + +test("collectRepoInfo detects Go and gathers manifest, tree, and counts", async () => { + const dir = await makeFixture(); + try { + const info = await collectRepoInfo(dir); + assert.equal(info.language, "go"); + assert.ok(info.manifest, "go.mod must be found"); + assert.equal(info.manifest.path, "go.mod"); + assert.match(info.fileTree, /server\/routes\.go/); + assert.ok(info.fileCounts[".go"] >= 5, "five go files expected"); + assert.equal(info.sourceGlob, "**/*.go"); + } finally { + await rm(dir, { recursive: true, force: true }); + } +}); + +test("directory slicing puts nested files under their top-level module", async () => { + const dir = await makeFixture(); + try { + const info = await collectRepoInfo(dir); + const lens = getLens("defect"); + const slices = await gatherSlices(dir, lens, info); + const byModule = Object.fromEntries(slices.map((s) => [s.moduleName, s])); + assert.ok(byModule.server, "server/ must be its own slice"); + assert.ok(byModule.model, "model/ must be its own slice"); + assert.ok(byModule.root, "top-level main.go must land in the root slice"); + assert.match(byModule.server.content, /server\/routes\.go/); + assert.match(byModule.model.content, /model\/deep\/nested\.go/, "nested files belong to the top-level module"); + assert.match(byModule.root.content, /main\.go/); + for (const slice of slices) { + assert.ok(slice.chars <= lens.maxChars); + } + } finally { + await rm(dir, { recursive: true, force: true }); + } +}); + +test("oversized modules split into multiple slices instead of truncating", async () => { + const dir = await mkdtemp(join(tmpdir(), "broadside-split-")); + try { + await mkdir(join(dir, "big")); + await writeFile(join(dir, "go.mod"), "module x\n"); + for (let i = 0; i < 80; i++) { + await writeFile(join(dir, "big", `file${i}.go`), "package big\n" + `// ${"x".repeat(2000)}\n`); + } + const info = await collectRepoInfo(dir); + const lens = getLens("defect"); + const slices = await gatherSlices(dir, lens, info); + const big = slices.filter((s) => s.moduleName === "big"); + assert.ok(big.length >= 2, `expected split slices, got ${big.length}`); + const totalChars = big.reduce((sum, s) => sum + s.chars, 0); + assert.ok(totalChars > 20 * 2000, "split slices must carry all content, not drop it"); + } finally { + await rm(dir, { recursive: true, force: true }); + } +}); + +test("architecture lens needs no file slurping and builds from repo info", async () => { + const dir = await makeFixture(); + try { + const info = await collectRepoInfo(dir); + const lens = getLens("architecture"); + const slices = await gatherSlices(dir, lens, info); + assert.equal(slices.length, 1); + const request = buildBatchRequest(lens, info, slices[0], 0, 1); + assert.equal(request.custom_id, "architecture-root"); + assert.match(request.body.messages[1].content, /module example\.com\/fixture/); + assert.equal(request.body.model, BROADSIDE_MODEL); + assert.equal(request.body.response_format.json_schema.name, "architecture_report"); + } finally { + await rm(dir, { recursive: true, force: true }); + } +}); + +test("api and security lenses skip test files; conventions keeps them", async () => { + const dir = await makeFixture(); + try { + await writeFile(join(dir, "server", "routes_test.go"), "package server\n"); + const info = await collectRepoInfo(dir); + const apiSlices = await gatherSlices(dir, getLens("api"), info); + const apiText = apiSlices.map((s) => s.content).join("\n"); + assert.ok(!apiText.includes("routes_test.go"), "api lens must skip test files"); + const securitySlices = await gatherSlices(dir, getLens("security"), info); + const securityText = securitySlices.map((s) => s.content).join("\n"); + assert.ok(!securityText.includes("routes_test.go"), "security lens must skip test files"); + const conventionsSlices = await gatherSlices(dir, getLens("conventions"), info); + const convText = conventionsSlices.map((s) => s.content).join("\n"); + assert.ok(convText.includes("routes_test.go"), "conventions lens must catalog test files"); + } finally { + await rm(dir, { recursive: true, force: true }); + } +}); + +// ---------- cost estimation ---------- + +test("estimateCost matches the documented per-token pricing", () => { + const lens = getLens("defect"); + const slices = [ + { moduleName: "a", content: "x".repeat(4000), fileCount: 1, chars: 4000 }, + { moduleName: "b", content: "y".repeat(4000), fileCount: 1, chars: 4000 }, + ]; + const { inputTokens, outputTokens, cost } = estimateCost(lens, slices); + assert.equal(inputTokens, 2000); // 8000 chars / 4 + assert.equal(outputTokens, 4500); // maxTokens 6000 * 0.75 + const expected = (2000 / 1e6) * 0.1875 + (4500 / 1e6) * 0.9375; + assert.ok(Math.abs(cost - expected) < 1e-12); +}); + +// ---------- state & config ---------- + +test("state round-trips and defaults to an empty run list", async () => { + const dir = await mkdtemp(join(tmpdir(), "broadside-state-")); + try { + const state = await loadBroadsideState(dir); + assert.deepEqual(state, defaultBroadsideState()); + state.runs.push({ + id: "run-1", + createdAt: "2026-08-23T00:00:00Z", + model: BROADSIDE_MODEL, + lenses: ["architecture"], + status: "in-flight", + outputDir: "run-1", + batches: {}, + synthesis: { status: "pending" }, + }); + await saveBroadsideState(dir, state); + const reloaded = await loadBroadsideState(dir); + assert.equal(reloaded.runs.length, 1); + assert.equal(reloaded.runs[0].id, "run-1"); + } finally { + await rm(dir, { recursive: true, force: true }); + } +}); + +test("corrupt state files degrade to defaults, not crashes", async () => { + const dir = await mkdtemp(join(tmpdir(), "broadside-corrupt-")); + try { + await writeFile(join(dir, "state.json"), "{ this is not json"); + const state = await loadBroadsideState(dir); + assert.equal(state.runs.length, 0); + } finally { + await rm(dir, { recursive: true, force: true }); + } +}); + +test("config falls back to defaults and honors overrides", async () => { + const dir = await mkdtemp(join(tmpdir(), "broadside-config-")); + try { + const defaults = await loadBroadsideConfig(dir); + assert.equal(defaults.model, BROADSIDE_MODEL); + assert.equal(defaults.apiKey, ""); + assert.equal(defaults.defaultLenses.length, 6); + + await writeFile( + join(dir, "config.yaml"), + "model: custom/model\napi_key: sk-test\ndefault_lenses:\n - architecture\n - security\n", + ); + const overridden = await loadBroadsideConfig(dir); + assert.equal(overridden.model, "custom/model"); + assert.equal(overridden.apiKey, "sk-test"); + assert.deepEqual(overridden.defaultLenses, ["architecture", "security"]); + + await writeFile(join(dir, "config.yaml"), "default_lenses:\n - bogus\n - architecture\n"); + const filtered = await loadBroadsideConfig(dir); + assert.deepEqual(filtered.defaultLenses, ["architecture"], "unknown lens ids must be dropped"); + } finally { + await rm(dir, { recursive: true, force: true }); + } +}); + +// ---------- batch client with a fake fetcher ---------- + +function fakeResponse(status, body) { + return { + status, + ok: status >= 200 && status < 300, + json: async () => body, + }; +} + +test("submitBatch orders endpoint and model before requests in the payload", async () => { + let captured; + const fetcher = async (url, init) => { + captured = { url, payload: JSON.parse(init.body) }; + return fakeResponse(202, { id: "batch-test-1", status: "validating" }); + }; + const result = await submitBatch( + [ + { + custom_id: "req-1", + body: { + model: BROADSIDE_MODEL, + messages: [{ role: "user", content: "hi" }], + response_format: { type: "json_schema", json_schema: { name: "x", strict: true, schema: {} } }, + max_tokens: 100, + }, + }, + ], + "sk-fake", + fetcher, + ); + assert.equal(result.batchId, "batch-test-1"); + const keys = Object.keys(captured.payload); + assert.deepEqual(keys, ["endpoint", "model", "requests"], "the API stream-parses and rejects requests-first bodies"); + assert.equal(captured.payload.endpoint, "/v1/chat/completions"); + assert.equal(captured.payload.model, BROADSIDE_MODEL); + assert.equal(captured.payload.requests.length, 1); +}); + +test("submitBatch surfaces non-202 rejection bodies", async () => { + const fetcher = async () => fakeResponse(400, { error: { message: "no" } }); + const result = await submitBatch([], "sk-fake", fetcher); + assert.equal(result.status, "rejected"); + assert.ok(result.error); +}); + +test("runBroadsideSubmit records a run and fires one batch per lens", async () => { + const dir = await makeFixture(); + try { + const seenBatches = []; + const fetcher = async (url, init) => { + if (init.method === "POST") { + const payload = JSON.parse(init.body); + seenBatches.push(payload); + return fakeResponse(202, { id: `batch-${seenBatches.length}`, status: "validating" }); + } + return fakeResponse(200, { id: "x", status: "in_progress" }); + }; + const result = await runBroadsideSubmit(dir, "sk-fake", { lenses: ["architecture", "security"], fetcher }); + assert.equal(Object.keys(result.batches).length, 2); + assert.ok(result.estimatedTotalCost > 0); + assert.equal(seenBatches.length, 2); + for (const payload of seenBatches) { + assert.ok(payload.requests.length >= 1); + } + const { state } = await runBroadsideStatus(dir); + assert.equal(state.runs.length, 1); + assert.equal(state.runs[0].status, "in-flight"); + assert.equal(state.runs[0].batches.architecture.status, "validating"); + } finally { + await rm(dir, { recursive: true, force: true }); + } +}); + +// ---------- collect with fake fetcher ---------- + +test("runBroadsideCollect polls, saves results, and runs synthesis", async () => { + const dir = await makeFixture(); + try { + let submits = 0; + let gets = 0; + const lensPayload = { + results: [ + { + custom_id: "architecture-root", + response: { + status_code: 200, + body: { + choices: [{ message: { role: "assistant", content: JSON.stringify({ tech_stack: { language: "Go", build_system: "go modules" }, module_architecture: [], data_flow: "x", entry_points: ["main.go"] }) } }], + }, + }, + error: null, + }, + ], + usage: { cost: 0.001 }, + request_counts: { total: 1, completed: 1, failed: 0 }, + }; + const synthPayload = { + results: [ + { + custom_id: "synthesis", + response: { + status_code: 200, + body: { + choices: [{ message: { role: "assistant", content: JSON.stringify({ executive_summary: "ok", severity_summary: { critical: 0, high: 1, medium: 2, low: 3 }, top_findings: [{ title: "lead", severity: "high", source_lens: "architecture", summary: "a lead" }] }) } }], + }, + }, + error: null, + }, + ], + usage: { cost: 0.002 }, + request_counts: { total: 1, completed: 1, failed: 0 }, + }; + const fetcher = async (url, init) => { + if (init.method === "POST") { + submits += 1; + return fakeResponse(202, { id: submits === 1 ? "batch-lens" : "batch-synth", status: "validating" }); + } + gets += 1; + // First GET per batch returns completed immediately. + if (String(url).includes("batch-lens")) { + return fakeResponse(200, { id: "batch-lens", status: "completed", ...lensPayload }); + } + return fakeResponse(200, { id: "batch-synth", status: "completed", ...synthPayload }); + }; + + await runBroadsideSubmit(dir, "sk-fake", { lenses: ["architecture"], fetcher }); + const collect = await runBroadsideCollect(dir, "sk-fake", { fetcher }); + assert.equal(collect.status, "completed"); + assert.equal(collect.resultCount, 1); + assert.ok(collect.totalCost > 0); + assert.equal(collect.synthesis.status, "completed"); + assert.equal(collect.topFindings.length, 1); + assert.equal(collect.topFindings[0].title, "lead"); + } finally { + await rm(dir, { recursive: true, force: true }); + } +}); + +// ---------- markdown rendering ---------- + +test("renderFindingsMarkdown turns parsed JSON into readable text", () => { + const md = renderFindingsMarkdown(JSON.stringify({ title: "T", severity: "high", nested: { a: "b" }, list: [{ title: "x" }] })); + assert.match(md, /\*\*title\*\*: T/); + assert.match(md, /\*\*severity\*\*: high/); + assert.match(md, /\*\*nested\*\*:/); + assert.match(md, /1\. x/); +}); + +test("renderFindingsMarkdown passes invalid JSON through untouched", () => { + assert.equal(renderFindingsMarkdown("not json"), "not json"); +}); From f5a7703a8b3864a64d85b3a8a59272836411b621 Mon Sep 17 00:00:00 2001 From: James Sesler Date: Sun, 23 Aug 2026 02:03:19 -0400 Subject: [PATCH 2/7] =?UTF-8?q?fix:=20broadside=20resilience=20=E2=80=94?= =?UTF-8?q?=20auth=20bail,=20empty-lens=20skip,=20submission=20throw=20gua?= =?UTF-8?q?rd=20(#103)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three failures surfaced by running Broad-Side on its own repo: - The poll loop retried a dead key for the whole budget. fetchBatch now surfaces the HTTP status and the poller returns auth-failed fast on 401/403, which collect treats as terminal. - A lens whose globs match no files submitted an empty batch (rejected by the API) and blocked the synthesis terminal check forever. Empty lens slices are now marked skipped before submission, and skipped/rejected are terminal statuses for collect and synthesis gating. - A network-level throw during submission stranded the batch entry in 'submitting' with no batch id. The submission guard now records the failure as rejected with the error message. Also broadens the non-Go api-lens globs (mcp-server/**, routes/router/ handler/endpoint name patterns) so repos whose server code does not live under server/ or api/ still get scanned. --- core/broadside.ts | 53 ++++++++++++++++++++++++++++++++-------- tests/broadside.test.mjs | 39 +++++++++++++++++++++++++++++ 2 files changed, 82 insertions(+), 10 deletions(-) diff --git a/core/broadside.ts b/core/broadside.ts index 0b69d5d..38b9576 100644 --- a/core/broadside.ts +++ b/core/broadside.ts @@ -522,7 +522,17 @@ const LENSES: Record = { globsFor: (info) => info.language === "go" ? ["server/**/*.go", "server/*.go", "api/**/*.go", "api/*.go"] - : ["server/**", "api/**", "src/server/**", "src/api/**"], + : [ + "server/**", + "api/**", + "src/server/**", + "src/api/**", + "mcp-server/**", + "**/*routes*", + "**/*router*", + "**/*handler*", + "**/*endpoint*", + ], systemPrompt: () => "You are a senior API auditor. Given source files from an HTTP server, " + "extract every HTTP endpoint (method, path, handler function, auth requirement) " + @@ -1147,7 +1157,11 @@ export async function fetchBatch( headers: { Authorization: `Bearer ${apiKey}` }, signal: AbortSignal.timeout(30_000), }); - return (await resp.json()) as Record; + const data = (await resp.json()) as Record; + // Surface the HTTP status so the poller can bail fast on auth expiry + // instead of retrying a dead key for the whole budget. + data.http_status = resp.status; + return data; } export async function pollBatchUntilTerminal( @@ -1170,10 +1184,14 @@ export async function pollBatchUntilTerminal( await sleep(BROADSIDE_POLL_INTERVAL_MS); continue; } + const httpStatus = Number(batch.http_status ?? 200); + if (httpStatus === 401 || httpStatus === 403) { + return { id: batchId, status: "auth-failed", error: batch.error ?? batch }; + } const status = String(batch.status ?? "unknown"); const counts = (batch.request_counts ?? {}) as Record; opts.onStatus?.(status, counts); - if (["completed", "failed", "expired", "cancelled"].includes(status)) return batch; + if (["completed", "failed", "expired", "cancelled", "auth-failed"].includes(status)) return batch; if (Date.now() >= deadline) return { id: batchId, status: "timeout" }; await sleep(BROADSIDE_POLL_INTERVAL_MS); } @@ -1227,12 +1245,27 @@ export async function runBroadsideSubmit( }; run.batches[lensId] = entry; + if (requests.length === 0) { + // No files matched the lens's globs. That is a coverage gap to + // report, not a batch to submit — the API rejects empty batches. + entry.status = "skipped"; + continue; + } + submissions.push( (async () => { - const { batchId, status, error } = await submitBatch(requests, apiKey, opts.fetcher); - entry.batchId = batchId; - entry.status = status; - if (error) entry.error = error; + // A network-level throw (DNS, abort, TLS) must not strand the + // entry in "submitting" forever — allSettled would swallow the + // rejection and collect would never see a terminal status. + try { + const { batchId, status, error } = await submitBatch(requests, apiKey, opts.fetcher); + entry.batchId = batchId; + entry.status = status; + if (error) entry.error = error; + } catch (error) { + entry.status = "rejected"; + entry.error = error instanceof Error ? error.message : String(error); + } })(), ); } @@ -1322,7 +1355,7 @@ export async function runBroadsideCollect( lensOutcomes[lensId] = { status: entry?.status ?? "failed", resultCount: 0 }; continue; } - if (["completed", "failed", "expired", "cancelled"].includes(entry.status)) { + if (["completed", "failed", "expired", "cancelled", "auth-failed", "skipped", "rejected"].includes(entry.status)) { totalCost += entry.cost ?? 0; resultCount += entry.resultCount ?? 0; lensOutcomes[lensId] = { status: entry.status, cost: entry.cost, resultCount: entry.resultCount }; @@ -1364,7 +1397,7 @@ export async function runBroadsideCollect( if (opts.includeSynthesis !== false && allLensResults.length > 0) { const allTerminal = run.lenses.every((lensId) => { const entry = run.batches[lensId]; - return entry && ["completed", "failed", "expired", "cancelled"].includes(entry.status); + return entry && ["completed", "failed", "expired", "cancelled", "auth-failed", "skipped", "rejected"].includes(entry.status); }); if (allTerminal && run.synthesis.status === "pending") { const findingsText = allLensResults @@ -1434,7 +1467,7 @@ export async function runBroadsideCollect( const terminal = run.lenses.every((lensId) => { const entry = run.batches[lensId]; - return entry && ["completed", "failed", "expired", "cancelled"].includes(entry.status); + return entry && ["completed", "failed", "expired", "cancelled", "auth-failed", "skipped", "rejected"].includes(entry.status); }); run.status = terminal ? (resultCount > 0 ? "completed" : "failed") : "partial"; run.totalCost = totalCost; diff --git a/tests/broadside.test.mjs b/tests/broadside.test.mjs index 93c306f..0e5c7de 100644 --- a/tests/broadside.test.mjs +++ b/tests/broadside.test.mjs @@ -159,6 +159,45 @@ test("api and security lenses skip test files; conventions keeps them", async () } }); +test("submit marks a lens with no matching files as skipped, never submitting an empty batch", async () => { + const dir = await mkdtemp(join(tmpdir(), "broadside-empty-")); + try { + await writeFile(join(dir, "go.mod"), "module x\n"); + await writeFile(join(dir, "main.go"), "package main\n"); + const posted = []; + const fetcher = async (url, init) => { + if (init.method === "POST") { + posted.push(JSON.parse(init.body)); + return fakeResponse(202, { id: `batch-${posted.length}`, status: "validating" }); + } + return fakeResponse(200, { id: "x", status: "completed" }); + }; + // api lens globs target server/ and api/ — neither exists here. + const result = await runBroadsideSubmit(dir, "sk-fake", { lenses: ["architecture", "api"], fetcher }); + assert.equal(result.batches.api.status, "skipped"); + assert.equal(posted.length, 1, "only the architecture batch may be submitted"); + assert.ok(posted[0].requests.length > 0, "submitted batches must be non-empty"); + } finally { + await rm(dir, { recursive: true, force: true }); + } +}); + +test("a network throw during submission marks the entry rejected, not stuck submitting", async () => { + const dir = await mkdtemp(join(tmpdir(), "broadside-throw-")); + try { + await writeFile(join(dir, "go.mod"), "module x\n"); + await writeFile(join(dir, "main.go"), "package main\n"); + const fetcher = async () => { + throw new Error("ECONNREFUSED"); + }; + const result = await runBroadsideSubmit(dir, "sk-fake", { lenses: ["architecture"], fetcher }); + assert.equal(result.batches.architecture.status, "rejected"); + assert.ok(result.batches.architecture.error, "the failure reason must be recorded"); + } finally { + await rm(dir, { recursive: true, force: true }); + } +}); + // ---------- cost estimation ---------- test("estimateCost matches the documented per-token pricing", () => { From e2c1984e9b2717ffe1d35119d1a09c970c5d14aa Mon Sep 17 00:00:00 2001 From: James Sesler Date: Sun, 23 Aug 2026 02:25:28 -0400 Subject: [PATCH 3/7] docs: Broad-Side roadmap with linked feat-tagged issues (#103) --- ROADMAP.md | 58 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 58 insertions(+) create mode 100644 ROADMAP.md diff --git a/ROADMAP.md b/ROADMAP.md new file mode 100644 index 0000000..e13316e --- /dev/null +++ b/ROADMAP.md @@ -0,0 +1,58 @@ +# Roadmap — Broad-Side + +Broad-Side is CodeCartographer's batch reconnaissance feature: a cheap, +unattended multi-lens scan over the OpenRouter Batch API that produces +unverified scouting leads for the interactive pipeline to confirm. Shipped +behind the `feat/103-broadside` branch (MCP surface first). + +This roadmap is the working agreement on what comes next. Items are tracked +as GitHub issues labeled `feat`; status changes happen in the issues, this +file only moves when a tier completes. + +## Shipped + +- `codecarto_broadside` MCP tool — `submit` / `collect` / `status` actions. +- Six lenses: architecture, api, security, defect, conventions, porting. +- Directory slicing with overflow splitting (no truncation of coverage). +- Cross-lens synthesis report. +- Repo-local state file (`broadside/state.json`) with resumable collect. +- Works without an initialized workspace; `codecarto_init` tolerates a + scout-only `.codecarto/`; scaffold refresh never touches broadside state. +- Auth-expiry fast bail, empty-lens skip, submission-throw guard. +- Tests: 18 unit tests (fake-fetcher based), opt-in live smoke script. + +## Tier 1 — make Broad-Side better at what it does + +| Item | Issue | Notes | +|---|---|---| +| **Triage lens** — prioritized fix queue (impact × difficulty, grouped by module) | [#135](https://github.com/HuginnIndustries/CodeCartographer/issues/135) | Highest-value next lens: turns leads into a work order | +| **Truncation repair** — detect max_tokens-cutoff JSON, resubmit slices, report truncation in summaries | [#133](https://github.com/HuginnIndustries/CodeCartographer/issues/133) | Found in the self-scan: 3/10 defect slices truncated | +| **Concurrent polling** — poll all in-flight batches round-robin against one deadline | [#136](https://github.com/HuginnIndustries/CodeCartographer/issues/136) | Submissions already parallel; polling is sequential today | +| **Per-language prompts** — Go/Python/Rust/TS lens prompts; globs already adapt | [#137](https://github.com/HuginnIndustries/CodeCartographer/issues/137) | Schemas stay shared so synthesis is unaffected | + +## Tier 2 — integration depth + +| Item | Issue | Notes | +|---|---|---| +| **Pi extension** — `/codecarto-broadside` command with lens picker and live progress | [#138](https://github.com/HuginnIndustries/CodeCartographer/issues/138) | Agreed order: MCP first (shipped), Pi second | +| **Pipeline phase** — `broadside-scout` phase feeding later phases via `required_reads` | [#139](https://github.com/HuginnIndustries/CodeCartographer/issues/139) | SKILL.md contract stays: leads, never evidence | +| **Zero-config executive** — meta-pass picks lenses and slicing resolution from repo shape | [#140](https://github.com/HuginnIndustries/CodeCartographer/issues/140) | Decision recorded in `run-meta.json` for reproducibility | + +## Tier 3 — cost and coverage economics + +| Item | Issue | Notes | +|---|---|---| +| **Multi-model** — DeepSeek/Anthropic batch endpoints behind the lens registry | [#141](https://github.com/HuginnIndustries/CodeCartographer/issues/141) | Stronger batch models for semantic lenses at ~2-3× cost | +| **Incremental re-scouting** — diff against previous run's HEAD, rescan changed modules only | [#142](https://github.com/HuginnIndustries/CodeCartographer/issues/142) | Makes recurring scans O(delta) | +| **CodeCartoShow pipeline stage** — BATCH-SCOUT between SELECT and the interactive run | [CodeCartoShow#1](https://github.com/HuginnIndustries/CodeCartoShow/issues/1) | `scripts/batch-analyze.py` proved it; evidence rules apply unchanged | + +## Principles + +1. **Leads, never evidence.** Every Broad-Side artifact carries the + disclaimer; nothing downstream may cite a Broad-Side report as fact. +2. **Coverage is spoken, not implied.** Truncations, skipped lenses, and + unscouted scope appear in `run-meta.json` and the collect summary. +3. **Cost before submission.** Estimates are shown on submit; no silent + spend. The live smoke script stays opt-in. +4. **The cheap model is a feature.** Gemini batch is weak but ~50% price; + its job is to tell the expensive run where to look, not to be right. From 30fe5405e5df98be9c7d2c122d9fff26f3ef4f46 Mon Sep 17 00:00:00 2001 From: James Sesler Date: Sun, 23 Aug 2026 02:56:46 -0400 Subject: [PATCH 4/7] =?UTF-8?q?feat:=20broadside=20expense=20guardrails=20?= =?UTF-8?q?=E2=80=94=20live=20pricing=20lookup=20and=20max=5Fcost=20limit?= =?UTF-8?q?=20(#103)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Before submitting, Broad-Side now estimates the run cost from the collected file sizes (approx 4 chars/token) against the configured model's real per-token pricing: - Live lookup from OpenRouter's model catalog, cached 24h in broadside/pricing-cache.json. Unknown models that cannot be priced refuse to submit rather than silently estimating at default rates. - config.yaml gains model (now actually wired through submissions), max_cost, and pricing.input_per_m/output_per_m overrides. - The MCP tool gains max_cost and force parameters; a submit whose estimate exceeds the limit refuses with a per-lens breakdown and creates no run entry unless force: true. - The submit response reports the pricing used and its source (built-in/config/live/cache), and run-meta.json records both pricing and the limit for audit. Motivated by the expensive end of the batch catalog (e.g. openai/gpt-5.2-pro:batch at ~$84/M output tokens): an estimate against the wrong model's rates is a wrong guardrail. --- .codecarto/broadside/SKILL.md | 6 + .codecarto/broadside/config.yaml | 25 +++- CHANGELOG.md | 1 + ROADMAP.md | 5 +- core/broadside.ts | 214 ++++++++++++++++++++++++++++--- mcp-server/server.ts | 22 +++- tests/broadside.test.mjs | 161 ++++++++++++++++++++++- 7 files changed, 408 insertions(+), 26 deletions(-) diff --git a/.codecarto/broadside/SKILL.md b/.codecarto/broadside/SKILL.md index 751c700..8fb40ee 100644 --- a/.codecarto/broadside/SKILL.md +++ b/.codecarto/broadside/SKILL.md @@ -66,3 +66,9 @@ codecarto_broadside {cwd, action: "status"} # show recorded ru It works on any git repository — no initialized workspace required — and needs an OpenRouter API key via the `api_key` parameter, the `OPENROUTER_API_KEY` environment variable, or `api_key` in this directory's `config.yaml`. + +Submits are priced before they fire: Broad-Side estimates the run from the +collected file sizes against the model's live per-token pricing and refuses +when the estimate exceeds `max_cost` (`config.yaml` or the tool parameter) +unless `force` is passed. See `config.yaml` for the model, limit, and manual +pricing-override keys. diff --git a/.codecarto/broadside/config.yaml b/.codecarto/broadside/config.yaml index 7e28fff..5e68a27 100644 --- a/.codecarto/broadside/config.yaml +++ b/.codecarto/broadside/config.yaml @@ -4,7 +4,11 @@ # OpenRouter model to use for batch requests. The default is Google Gemini # 3.7 Flash (batch) — the cheapest batch model with tool-calling support and # a 1M-token context window. Change this to another OpenRouter batch model -# if you need a different cost/capability trade-off. +# (https://openrouter.ai/models?variant=batch) if you need a different +# cost/capability trade-off; Broad-Side looks up its per-token pricing +# automatically and uses it for cost estimates and the max_cost guardrail. +# Beware the expensive end of that list — some batch models exceed +# $80 per million output tokens. # # model: google/gemini-3.7-flash:batch @@ -24,4 +28,21 @@ # - security # - defect # - conventions -# - porting \ No newline at end of file +# - porting + +# Approximate run expense limit in USD (0 = no limit). Before submitting, +# Broad-Side estimates the run cost from the collected file sizes and the +# model's per-token pricing — fetched live from OpenRouter's model catalog +# and cached for 24h. If the estimate exceeds max_cost, submit refuses and +# prints the per-lens breakdown; pass force: true to override, or set a +# value here so every run is guarded by default. +# +# max_cost: 1.00 + +# Manual pricing overrides in USD per MILLION tokens. Normally Broad-Side +# looks the model's pricing up automatically; set both fields only when the +# lookup fails (offline, private model) or you want to assert a ceiling. +# +# pricing: +# input_per_m: 0.1875 +# output_per_m: 0.9375 \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md index 26011bd..039bab8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ All notable changes to this project are documented here. The format is based on ### Added - **Broad-Side: batch reconnaissance over the OpenRouter Batch API** (#103). New `codecarto_broadside` MCP tool (actions: `submit`, `collect`, `status`) fires six single-turn analysis lenses — architecture, API surface, security, mechanical defect scan, convention extraction, porting — at any git repository as asynchronous batch jobs on a cheap batch model (~50% of sync pricing, unattended, 24h window), slices large modules by top-level directory, saves JSON plus rendered markdown to `.codecarto/broadside//`, and optionally synthesizes a cross-lens executive report. Works without an initialized workspace; needs an OpenRouter key via the `api_key` parameter, `OPENROUTER_API_KEY`, or `.codecarto/broadside/config.yaml`. Broad-Side findings are explicitly unverified scouting signals — file:line leads for the interactive pipeline to confirm, never evidence themselves. `codecarto_init` tolerates a `.codecarto/` that holds only `broadside/` (no force/backup needed), and scaffold refresh never touches broadside state, config, or results. +- **Broad-Side: expense guardrails and live per-model pricing** (#103). `config.yaml` now accepts `model`, `max_cost`, and `pricing.input_per_m`/`output_per_m` overrides, and the MCP tool accepts `max_cost` and `force` parameters. Before submitting, Broad-Side estimates the run cost from collected file sizes (≈4 chars/token) against the configured model's per-token pricing — looked up live from OpenRouter's model catalog (cached 24h), so models like `openai/gpt-5.2-pro:batch` at ~$84/M output are priced correctly, not at the default model's rates. A submit whose estimate exceeds `max_cost` refuses with a per-lens breakdown and creates no run entry unless `force: true`. The submit response now reports the pricing used and its source (built-in/config/live/cache). ## [0.16.0] — 2026-08-17 diff --git a/ROADMAP.md b/ROADMAP.md index e13316e..4026e46 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -19,7 +19,10 @@ file only moves when a tier completes. - Works without an initialized workspace; `codecarto_init` tolerates a scout-only `.codecarto/`; scaffold refresh never touches broadside state. - Auth-expiry fast bail, empty-lens skip, submission-throw guard. -- Tests: 18 unit tests (fake-fetcher based), opt-in live smoke script. +- Live per-model pricing lookup (OpenRouter catalog, 24h cache) with + `max_cost` expense guardrail and `force` override; configurable model + (`config.yaml` `model` key, now wired through submissions). +- Tests: 26 unit tests (fake-fetcher based), opt-in live smoke script. ## Tier 1 — make Broad-Side better at what it does diff --git a/core/broadside.ts b/core/broadside.ts index 38b9576..68401c1 100644 --- a/core/broadside.ts +++ b/core/broadside.ts @@ -39,6 +39,12 @@ export const BROADSIDE_STATE_SCHEMA_VERSION = 1; export const BROADSIDE_INPUT_PRICE_PER_M = 0.1875; export const BROADSIDE_OUTPUT_PRICE_PER_M = 0.9375; +// OpenRouter's public model catalog; pricing lives per model id. +export const BROADSIDE_MODELS_URL = "https://openrouter.ai/api/v1/models"; + +export const BROADSIDE_PRICING_CACHE_FILE = "pricing-cache.json"; +export const BROADSIDE_PRICING_CACHE_TTL_MS = 24 * 60 * 60 * 1000; + export const BROADSIDE_LENS_IDS = [ "architecture", "api", @@ -54,6 +60,15 @@ export const BROADSIDE_DEFAULT_POLL_BUDGET_MS = 25 * 60 * 1000; // ---------- types ---------- +export type ModelPricing = { + /** USD per million input tokens. */ + inputPerM: number; + /** USD per million output tokens. */ + outputPerM: number; + /** Where the numbers came from — affects what the submit text claims. */ + source: "built-in" | "config" | "live" | "cache"; +}; + export type JsonSchemaDef = { name: string; strict: boolean; @@ -120,6 +135,8 @@ export type BroadsideRun = { batches: Partial>; synthesis: BroadsideSynthesisEntry; totalCost?: number; + pricing?: ModelPricing; + maxCost?: number; }; export type BroadsideStateFile = { @@ -131,6 +148,10 @@ export type BroadsideConfig = { model: string; apiKey: string; defaultLenses: BroadsideLensId[]; + /** Approximate run expense limit in USD; 0 means no limit. */ + maxCost: number; + /** Manual pricing overrides (USD per million). Live lookup is preferred. */ + pricing: { inputPerM: number; outputPerM: number } | null; }; export type BroadsideSubmitResult = { @@ -140,6 +161,8 @@ export type BroadsideSubmitResult = { estimatedTotalCost: number; estimatedInputTokens: number; estimatedOutputTokens: number; + pricing: ModelPricing; + maxCost?: number; }; export type BroadsideCollectResult = { @@ -1038,13 +1061,14 @@ export function buildBatchRequest( slice: FileSlice, index: number, sliceCount: number, + model: string = BROADSIDE_MODEL, ): BatchRequest { const moduleTag = sanitizeId(slice.moduleName); const customId = sliceCount > 1 ? `${lens.id}-${moduleTag}-${index + 1}` : `${lens.id}-${moduleTag}`; return { custom_id: customId, body: { - model: BROADSIDE_MODEL, + model, messages: [ { role: "system", content: lens.systemPrompt(info) }, { role: "user", content: lens.userPrompt(info, slice.content, slice.moduleName) }, @@ -1055,7 +1079,11 @@ export function buildBatchRequest( }; } -export function estimateCost(lens: LensDefinition, slices: FileSlice[]): { +export function estimateCost( + lens: LensDefinition, + slices: FileSlice[], + pricing: ModelPricing, +): { inputTokens: number; outputTokens: number; cost: number; @@ -1063,8 +1091,8 @@ export function estimateCost(lens: LensDefinition, slices: FileSlice[]): { const inputTokens = Math.ceil(slices.reduce((sum, s) => sum + (lens.maxChars === 0 ? 6000 : s.chars), 0) / 4); const outputTokens = Math.ceil(lens.maxTokens * 0.75); const cost = - (inputTokens / 1_000_000) * BROADSIDE_INPUT_PRICE_PER_M + - (outputTokens / 1_000_000) * BROADSIDE_OUTPUT_PRICE_PER_M; + (inputTokens / 1_000_000) * pricing.inputPerM + + (outputTokens / 1_000_000) * pricing.outputPerM; return { inputTokens, outputTokens, cost }; } @@ -1108,13 +1136,107 @@ export async function loadBroadsideConfig(broadsideDir: string): Promise BROADSIDE_LENS_IDS.includes(l as BroadsideLensId))) : []; + const rawPricing = (raw.pricing ?? {}) as Record; + const inputOverride = typeof rawPricing.input_per_m === "number" ? rawPricing.input_per_m : undefined; + const outputOverride = typeof rawPricing.output_per_m === "number" ? rawPricing.output_per_m : undefined; return { model: typeof raw.model === "string" && raw.model.trim() ? raw.model.trim() : BROADSIDE_MODEL, apiKey: typeof raw.api_key === "string" ? raw.api_key.trim() : "", defaultLenses: lenses.length > 0 ? lenses : [...BROADSIDE_LENS_IDS], + maxCost: typeof raw.max_cost === "number" && raw.max_cost > 0 ? raw.max_cost : 0, + pricing: + inputOverride !== undefined && outputOverride !== undefined + ? { inputPerM: inputOverride, outputPerM: outputOverride } + : null, }; } +// ---------- pricing resolution ---------- + +type PricingCacheFile = { + schema_version: number; + models: Record; +}; + +async function readPricingCache(broadsideDir: string): Promise { + const cachePath = join(broadsideDir, BROADSIDE_PRICING_CACHE_FILE); + if (!(await pathExists(cachePath))) return null; + try { + const parsed = JSON.parse(await readFile(cachePath, "utf8")) as PricingCacheFile; + if (!parsed || typeof parsed !== "object" || typeof parsed.models !== "object") return null; + return parsed; + } catch { + return null; + } +} + +async function writePricingCache(broadsideDir: string, cache: PricingCacheFile): Promise { + await mkdir(broadsideDir, { recursive: true }); + await writeFile(join(broadsideDir, BROADSIDE_PRICING_CACHE_FILE), `${JSON.stringify(cache, null, "\t")}\n`, "utf8"); +} + +export function builtInPricing(model: string): ModelPricing | null { + if (model !== BROADSIDE_MODEL) return null; + return { inputPerM: BROADSIDE_INPUT_PRICE_PER_M, outputPerM: BROADSIDE_OUTPUT_PRICE_PER_M, source: "built-in" }; +} + +export async function resolveModelPricing( + broadsideDir: string, + config: BroadsideConfig, + model: string, + apiKey: string, + fetcher: FetchLike = fetch as FetchLike, +): Promise { + // Manual overrides always win — the user is asserting a price, and a + // config assertion is cheaper to respect than to second-guess. + if (config.pricing) { + return { ...config.pricing, source: "config" }; + } + const builtIn = builtInPricing(model); + if (builtIn) return builtIn; + + // Unknown model: check the on-disk cache first, then the live catalog. + const cache = await readPricingCache(broadsideDir); + const cached = cache?.models[model]; + if (cached && Date.now() - new Date(cached.fetchedAt).getTime() < BROADSIDE_PRICING_CACHE_TTL_MS) { + return { inputPerM: cached.inputPerM, outputPerM: cached.outputPerM, source: "cache" }; + } + + let live: { inputPerM: number; outputPerM: number } | null = null; + try { + const resp = await fetcher(BROADSIDE_MODELS_URL, { + method: "GET", + headers: { Authorization: `Bearer ${apiKey}` }, + signal: AbortSignal.timeout(30_000), + }); + const data = (await resp.json()) as { data?: Array> }; + const hit = (data.data ?? []).find((m) => String(m.id) === model); + if (hit && typeof hit.pricing === "object") { + const p = hit.pricing as { prompt?: string; completion?: string }; + const input = typeof p.prompt === "string" ? Number(p.prompt) : NaN; + const output = typeof p.completion === "string" ? Number(p.completion) : NaN; + if (Number.isFinite(input) && Number.isFinite(output)) { + live = { inputPerM: input * 1_000_000, outputPerM: output * 1_000_000 }; + } + } + } catch { + live = null; + } + + if (live) { + const updated: PricingCacheFile = { schema_version: 1, models: { ...(cache?.models ?? {}) } }; + updated.models[model] = { ...live, fetchedAt: new Date().toISOString() }; + await writePricingCache(broadsideDir, updated); + return { ...live, source: "live" }; + } + + throw new Error( + `Could not resolve per-token pricing for batch model "${model}". ` + + "Set pricing.input_per_m and pricing.output_per_m in .codecarto/broadside/config.yaml " + + "(USD per million tokens), or check the model id against https://openrouter.ai/models?variant=batch.", + ); +} + // ---------- batch client ---------- export type FetchLike = (url: string, init: Record) => Promise; @@ -1123,12 +1245,13 @@ export async function submitBatch( batchRequests: BatchRequest[], apiKey: string, fetcher: FetchLike = fetch as FetchLike, + model: string = BROADSIDE_MODEL, ): Promise<{ batchId: string; status: string; error?: unknown }> { // The OpenRouter batch endpoint stream-parses the body and requires // `endpoint` and `model` to serialize before `requests` — key order matters. const payload = { endpoint: "/v1/chat/completions", - model: BROADSIDE_MODEL, + model, requests: batchRequests, }; const resp = await fetcher(BROADSIDE_BATCH_URL, { @@ -1202,39 +1325,78 @@ export async function pollBatchUntilTerminal( export async function runBroadsideSubmit( cwd: string, apiKey: string, - opts: { lenses?: BroadsideLensId[]; fetcher?: FetchLike } = {}, + opts: { + lenses?: BroadsideLensId[]; + fetcher?: FetchLike; + model?: string; + /** Approximate run expense limit in USD; 0 means no limit. */ + maxCost?: number; + /** Submit even when the estimate exceeds maxCost. */ + force?: boolean; + } = {}, ): Promise { const info = await collectRepoInfo(cwd); const lensIds = opts.lenses ?? BROADSIDE_LENS_IDS; const broadsideDir = broadsideDirFor(cwd); + const model = opts.model ?? BROADSIDE_MODEL; + + // Resolve pricing before anything is submitted: the guardrail must know + // the model's real per-token rates, not the default model's. + const config = await loadBroadsideConfig(broadsideDir); + const pricing = await resolveModelPricing(broadsideDir, config, model, apiKey, opts.fetcher); + const limit = opts.maxCost ?? config.maxCost; + + // Slice offline first so the estimate covers every request we would send. + const slicesByLens = new Map(); + let estimatedInputTokens = 0; + let estimatedOutputTokens = 0; + let estimatedTotalCost = 0; + const perLensEstimate: Array<{ lens: LensDefinition; cost: number }> = []; + for (const lensId of lensIds) { + const lens = getLens(lensId); + const slices = await gatherSlices(cwd, lens, info); + slicesByLens.set(lensId, slices); + const estimate = estimateCost(lens, slices, pricing); + estimatedInputTokens += estimate.inputTokens; + estimatedOutputTokens += estimate.outputTokens; + estimatedTotalCost += estimate.cost; + perLensEstimate.push({ lens, cost: estimate.cost }); + } + + if (limit > 0 && !opts.force && estimatedTotalCost > limit) { + const breakdown = perLensEstimate + .map(({ lens, cost }) => ` ${lens.name}: ~$${cost.toFixed(4)}`) + .join("\n"); + throw new Error( + `Estimated Broad-Side cost ~$${estimatedTotalCost.toFixed(4)} exceeds the run limit ` + + `$${limit.toFixed(2)}. Nothing was submitted.\nBreakdown:\n${breakdown}\n` + + `Pass force: true to submit anyway, or raise max_cost in .codecarto/broadside/config.yaml.`, + ); + } + const state = await loadBroadsideState(broadsideDir); const runId = new Date().toISOString().replace(/[:.]/g, "-"); const run: BroadsideRun = { id: runId, createdAt: new Date().toISOString(), - model: BROADSIDE_MODEL, + model, lenses: [...lensIds], status: "in-flight", outputDir: runId, batches: {}, synthesis: { status: "pending" }, + pricing, + maxCost: limit > 0 ? limit : undefined, }; state.runs.push(run); await saveBroadsideState(broadsideDir, state); - let estimatedInputTokens = 0; - let estimatedOutputTokens = 0; - let estimatedTotalCost = 0; - const submissions: Promise[] = []; for (const lensId of lensIds) { const lens = getLens(lensId); - const slices = await gatherSlices(cwd, lens, info); - const requests = slices.map((s, i) => buildBatchRequest(lens, info, s, i, slices.length)); - const estimate = estimateCost(lens, slices); - estimatedInputTokens += estimate.inputTokens; - estimatedOutputTokens += estimate.outputTokens; - estimatedTotalCost += estimate.cost; + const slices = slicesByLens.get(lensId) ?? []; + const requests = slices.map((s, i) => buildBatchRequest(lens, info, s, i, slices.length, model)); + const estimate = estimateCost(lens, slices, pricing); const entry: BroadsideBatchEntry = { batchId: "", @@ -1258,7 +1420,7 @@ export async function runBroadsideSubmit( // entry in "submitting" forever — allSettled would swallow the // rejection and collect would never see a terminal status. try { - const { batchId, status, error } = await submitBatch(requests, apiKey, opts.fetcher); + const { batchId, status, error } = await submitBatch(requests, apiKey, opts.fetcher, model); entry.batchId = batchId; entry.status = status; if (error) entry.error = error; @@ -1279,6 +1441,8 @@ export async function runBroadsideSubmit( estimatedTotalCost, estimatedInputTokens, estimatedOutputTokens, + pricing, + maxCost: limit > 0 ? limit : undefined, }; } @@ -1406,7 +1570,7 @@ export async function runBroadsideCollect( const request: BatchRequest = { custom_id: "synthesis", body: { - model: BROADSIDE_MODEL, + model: run.model, messages: [ { role: "system", @@ -1435,7 +1599,7 @@ export async function runBroadsideCollect( }; run.synthesis.status = "submitted"; await saveBroadsideState(broadsideDir, state); - const { batchId, error } = await submitBatch([request], apiKey, opts.fetcher); + const { batchId, error } = await submitBatch([request], apiKey, opts.fetcher, run.model); if (error) { run.synthesis.status = "failed"; } else { @@ -1479,7 +1643,9 @@ export async function runBroadsideCollect( { experimental: true, method: "Broad-Side (OpenRouter Batch API)", - model: BROADSIDE_MODEL, + model: run.model, + pricing: run.pricing, + max_cost: run.maxCost, run_id: run.id, created_at: run.createdAt, status: run.status, @@ -1583,6 +1749,12 @@ export function estimateSubmitText(result: BroadsideSubmitResult, lenses: LensDe } lines.push( `Estimated total: ~$${result.estimatedTotalCost.toFixed(4)}`, + `Pricing: $${result.pricing.inputPerM.toFixed(4)}/M in, $${result.pricing.outputPerM.toFixed(4)}/M out (${result.pricing.source})`, + ); + if (result.maxCost) { + lines.push(`Run limit: $${result.maxCost.toFixed(2)} (enforced on estimate; pass force to override)`); + } + lines.push( `Results will land in ${result.outputDir}/`, "Call codecarto_broadside with action 'collect' once batches finish, or pass wait_seconds on submit to block.", "Disclaimer: Broad-Side findings are unverified scouting signals from a batch model, not validated claims.", diff --git a/mcp-server/server.ts b/mcp-server/server.ts index 7f1dc26..7544872 100644 --- a/mcp-server/server.ts +++ b/mcp-server/server.ts @@ -996,6 +996,8 @@ export async function handleBroadside(args: { api_key?: string; wait_seconds?: number; include_synthesis?: boolean; + max_cost?: number; + force?: boolean; }) { const cwd = await validateCwd(args.cwd); const action = args.action ?? "submit"; @@ -1025,7 +1027,14 @@ export async function handleBroadside(args: { lenses = config.defaultLenses; } - const result = await runBroadsideSubmit(cwd, apiKey, { lenses }).catch((error) => { + const maxCost = typeof args.max_cost === "number" && args.max_cost > 0 ? args.max_cost : config.maxCost; + + const result = await runBroadsideSubmit(cwd, apiKey, { + lenses, + model: config.model, + maxCost, + force: args.force === true, + }).catch((error) => { throw new McpError(ErrorCode.InvalidRequest, error instanceof Error ? error.message : String(error)); }); @@ -1045,6 +1054,8 @@ export async function handleBroadside(args: { outputDir: result.outputDir, batches: result.batches, estimatedTotalCost: result.estimatedTotalCost, + pricing: result.pricing, + maxCost: result.maxCost, }); } @@ -1384,6 +1395,15 @@ const TOOLS = [ type: "boolean", description: "Run the cross-lens synthesis pass once all lens batches complete (default true).", }, + max_cost: { + type: "number", + description: + "Approximate run expense limit in USD. The submit action estimates the run cost from slice sizes and the configured model's per-token pricing (live OpenRouter lookup, cached 24h) and refuses to submit when the estimate exceeds the limit unless force is true. Falls back to max_cost in .codecarto/broadside/config.yaml.", + }, + force: { + type: "boolean", + description: "Submit even when the cost estimate exceeds max_cost (default false).", + }, }, required: ["cwd", "action"], }, diff --git a/tests/broadside.test.mjs b/tests/broadside.test.mjs index 0e5c7de..cbf37b6 100644 --- a/tests/broadside.test.mjs +++ b/tests/broadside.test.mjs @@ -17,6 +17,7 @@ const { BROADSIDE_LENS_IDS, BROADSIDE_MODEL, buildBatchRequest, + builtInPricing, collectRepoInfo, defaultBroadsideState, estimateCost, @@ -26,6 +27,7 @@ const { loadBroadsideState, listLenses, renderFindingsMarkdown, + resolveModelPricing, runBroadsideCollect, runBroadsideStatus, runBroadsideSubmit, @@ -206,13 +208,22 @@ test("estimateCost matches the documented per-token pricing", () => { { moduleName: "a", content: "x".repeat(4000), fileCount: 1, chars: 4000 }, { moduleName: "b", content: "y".repeat(4000), fileCount: 1, chars: 4000 }, ]; - const { inputTokens, outputTokens, cost } = estimateCost(lens, slices); + const pricing = { inputPerM: 0.1875, outputPerM: 0.9375, source: "built-in" }; + const { inputTokens, outputTokens, cost } = estimateCost(lens, slices, pricing); assert.equal(inputTokens, 2000); // 8000 chars / 4 assert.equal(outputTokens, 4500); // maxTokens 6000 * 0.75 const expected = (2000 / 1e6) * 0.1875 + (4500 / 1e6) * 0.9375; assert.ok(Math.abs(cost - expected) < 1e-12); }); +test("estimateCost scales with the pricing table, not the default model", () => { + const lens = getLens("defect"); + const slices = [{ moduleName: "a", content: "x".repeat(40000), fileCount: 1, chars: 40000 }]; + const cheap = estimateCost(lens, slices, { inputPerM: 0.1875, outputPerM: 0.9375, source: "built-in" }); + const expensive = estimateCost(lens, slices, { inputPerM: 3.75, outputPerM: 84, source: "live" }); + assert.ok(expensive.cost > cheap.cost * 20, "an $84/M output model must estimate far higher"); +}); + // ---------- state & config ---------- test("state round-trips and defaults to an empty run list", async () => { @@ -275,6 +286,154 @@ test("config falls back to defaults and honors overrides", async () => { } }); +// ---------- pricing resolution & expense limits ---------- + +function modelsCatalog(body) { + return { data: body }; +} + +test("builtInPricing covers only the default model", () => { + assert.ok(core.builtInPricing(BROADSIDE_MODEL)); + assert.equal(core.builtInPricing("openai/gpt-5.2-pro:batch"), null); +}); + +test("resolveModelPricing prefers config overrides over everything", async () => { + const dir = await mkdtemp(join(tmpdir(), "broadside-pricing-")); + try { + const config = { + model: "custom/model", + apiKey: "", + defaultLenses: ["architecture"], + maxCost: 0, + pricing: { inputPerM: 1.5, outputPerM: 42 }, + }; + const fetcher = async () => { + throw new Error("the network must not be touched when config pricing exists"); + }; + const pricing = await resolveModelPricing(dir, config, "custom/model", "sk-fake", fetcher); + assert.deepEqual(pricing, { inputPerM: 1.5, outputPerM: 42, source: "config" }); + } finally { + await rm(dir, { recursive: true, force: true }); + } +}); + +test("resolveModelPricing falls back to built-in for the default model without network", async () => { + const dir = await mkdtemp(join(tmpdir(), "broadside-pricing-")); + try { + const config = { model: BROADSIDE_MODEL, apiKey: "", defaultLenses: ["architecture"], maxCost: 0, pricing: null }; + const fetcher = async () => { + throw new Error("the default model needs no lookup"); + }; + const pricing = await resolveModelPricing(dir, config, BROADSIDE_MODEL, "sk-fake", fetcher); + assert.equal(pricing.source, "built-in"); + assert.equal(pricing.inputPerM, 0.1875); + } finally { + await rm(dir, { recursive: true, force: true }); + } +}); + +test("resolveModelPricing looks up unknown models live and caches the result", async () => { + const dir = await mkdtemp(join(tmpdir(), "broadside-pricing-")); + try { + const config = { model: "openai/gpt-5.2-pro:batch", apiKey: "", defaultLenses: ["architecture"], maxCost: 0, pricing: null }; + let fetches = 0; + const fetcher = async () => { + fetches += 1; + return fakeResponse( + 200, + modelsCatalog([{ id: "openai/gpt-5.2-pro:batch", pricing: { prompt: "0.00000375", completion: "0.000084" } }]), + ); + }; + const pricing = await resolveModelPricing(dir, config, "openai/gpt-5.2-pro:batch", "sk-fake", fetcher); + assert.equal(pricing.source, "live"); + assert.equal(pricing.inputPerM, 3.75); + assert.equal(pricing.outputPerM, 84); + assert.equal(fetches, 1); + + const cached = await resolveModelPricing(dir, config, "openai/gpt-5.2-pro:batch", "sk-fake", fetcher); + assert.equal(cached.source, "cache"); + assert.equal(fetches, 1, "second resolution must come from the 24h cache"); + } finally { + await rm(dir, { recursive: true, force: true }); + } +}); + +test("resolveModelPricing refuses unknown models it cannot price", async () => { + const dir = await mkdtemp(join(tmpdir(), "broadside-pricing-")); + try { + const config = { model: "vendor/mystery", apiKey: "", defaultLenses: ["architecture"], maxCost: 0, pricing: null }; + const fetcher = async () => fakeResponse(200, modelsCatalog([{ id: "other/model", pricing: { prompt: "0.000001", completion: "0.000002" } }])); + await assert.rejects( + () => resolveModelPricing(dir, config, "vendor/mystery", "sk-fake", fetcher), + /Could not resolve per-token pricing/, + ); + } finally { + await rm(dir, { recursive: true, force: true }); + } +}); + +test("submit refuses over-budget runs and creates no run entry; force bypasses", async () => { + const dir = await mkdtemp(join(tmpdir(), "broadside-limit-")); + try { + await writeFile(join(dir, "go.mod"), "module x\n"); + await mkdir(join(dir, "big"), { recursive: true }); + for (let i = 0; i < 40; i++) { + await writeFile(join(dir, "big", `file${i}.go`), "package big\n" + `// ${"y".repeat(2000)}\n`); + } + const fetcher = async (url, init) => { + if (init.method === "POST") { + return fakeResponse(202, { id: "batch-ok", status: "validating" }); + } + return fakeResponse(200, { id: "x", status: "in_progress" }); + }; + + await assert.rejects( + () => runBroadsideSubmit(dir, "sk-fake", { lenses: ["defect"], fetcher, maxCost: 0.0001 }), + /estimated.*exceeds the run limit|exceeds the run limit/i, + ); + let state = await loadBroadsideState(join(dir, ".codecarto", "broadside")); + assert.equal(state.runs.length, 0, "a refused submit must not create a run entry"); + + const forced = await runBroadsideSubmit(dir, "sk-fake", { lenses: ["defect"], fetcher, maxCost: 0.0001, force: true }); + assert.equal(forced.batches.defect.status, "validating"); + assert.equal(forced.maxCost, 0.0001); + + state = await loadBroadsideState(join(dir, ".codecarto", "broadside")); + assert.equal(state.runs.length, 1); + assert.equal(state.runs[0].pricing.source, "built-in"); + } finally { + await rm(dir, { recursive: true, force: true }); + } +}); + +test("submit passes the configured model into batch payloads", async () => { + const dir = await mkdtemp(join(tmpdir(), "broadside-model-")); + try { + await writeFile(join(dir, "go.mod"), "module x\n"); + await writeFile(join(dir, "main.go"), "package main\n"); + let payload; + const fetcher = async (url, init) => { + if (init.method === "POST") { + payload = JSON.parse(init.body); + return fakeResponse(202, { id: "batch-m", status: "validating" }); + } + return fakeResponse(200, { id: "x", status: "in_progress" }); + }; + // config pricing override: submit must not hit the network for pricing. + await mkdir(join(dir, ".codecarto", "broadside"), { recursive: true }); + await writeFile( + join(dir, ".codecarto", "broadside", "config.yaml"), + "model: openai/gpt-5.2-pro:batch\npricing:\n input_per_m: 3.75\n output_per_m: 84\n", + ); + const result = await runBroadsideSubmit(dir, "sk-fake", { lenses: ["architecture"], fetcher, model: "openai/gpt-5.2-pro:batch" }); + assert.equal(payload.model, "openai/gpt-5.2-pro:batch"); + assert.equal(payload.requests[0].body.model, "openai/gpt-5.2-pro:batch"); + assert.equal(result.pricing.source, "config"); + } finally { + await rm(dir, { recursive: true, force: true }); + } +}); + // ---------- batch client with a fake fetcher ---------- function fakeResponse(status, body) { From d5c9522d606776d9e179e5739f76c049e1114cb9 Mon Sep 17 00:00:00 2001 From: James Sesler Date: Sun, 23 Aug 2026 03:43:18 -0400 Subject: [PATCH 5/7] =?UTF-8?q?feat:=20broadside=20models=20action=20?= =?UTF-8?q?=E2=80=94=20batch-model=20catalog,=20benchmarks,=20capability?= =?UTF-8?q?=20pre-flight=20(#103)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Wired OpenRouter's model catalog and benchmarks endpoints into Broad-Side (field shapes per the official OpenRouterTeam/skills references): - New action 'models' lists every :batch variant — pricing per million tokens, context window, completion ceiling, structured-output support — cheapest first with the configured model marked, and optionally annotates each with its Artificial Analysis coding index (GET /api/v1/benchmarks, attribution preserved). - The pricing cache becomes a full model-catalog cache (shared by the models action and submit-time resolution). - Submit pre-flight: lens max_tokens clamps to the provider's completion ceiling (a request above it fails the whole batch), deprecated models are flagged in the submit text, and models that do not advertise structured-output support are refused outright — every lens depends on json_schema response_format. - The default model's capabilities are asserted from its shipped configuration (1M ctx, 64K out, structured outputs) so the built-in path needs no network. 5 new tests (catalog filtering/sort, structured-output refusal, max_tokens clamping, benchmark slug mapping, models table rendering); 373 total, all passing. --- .codecarto/broadside/SKILL.md | 10 + .codecarto/broadside/config.yaml | 12 +- CHANGELOG.md | 1 + ROADMAP.md | 8 +- core/broadside.ts | 343 ++++++++++++++++++++++++++----- mcp-server/server.ts | 32 ++- tests/broadside.test.mjs | 128 ++++++++++++ 7 files changed, 474 insertions(+), 60 deletions(-) diff --git a/.codecarto/broadside/SKILL.md b/.codecarto/broadside/SKILL.md index 8fb40ee..869a822 100644 --- a/.codecarto/broadside/SKILL.md +++ b/.codecarto/broadside/SKILL.md @@ -61,8 +61,18 @@ Broad-Side is an executable-surface feature (MCP today): codecarto_broadside {cwd, action: "submit", lenses: [...]} # fire the batches codecarto_broadside {cwd, action: "collect"} # poll, save, synthesize codecarto_broadside {cwd, action: "status"} # show recorded runs +codecarto_broadside {cwd, action: "models"} # compare batch models ``` +The `models` action lists every `:batch` variant on OpenRouter — pricing per +million tokens, context window, output ceiling, structured-output support, and +(optionally) Artificial Analysis coding indices — cheapest first, with the +configured model marked. Use it before switching models in `config.yaml`. +Submits pre-flight the chosen model: pricing comes from the live catalog +(cached 24h), requests clamp to the provider's completion ceiling, and a +model that does not advertise structured-output support is refused outright, +because every lens depends on `json_schema` response_format. + It works on any git repository — no initialized workspace required — and needs an OpenRouter API key via the `api_key` parameter, the `OPENROUTER_API_KEY` environment variable, or `api_key` in this directory's `config.yaml`. diff --git a/.codecarto/broadside/config.yaml b/.codecarto/broadside/config.yaml index 5e68a27..8d82315 100644 --- a/.codecarto/broadside/config.yaml +++ b/.codecarto/broadside/config.yaml @@ -4,11 +4,13 @@ # OpenRouter model to use for batch requests. The default is Google Gemini # 3.7 Flash (batch) — the cheapest batch model with tool-calling support and # a 1M-token context window. Change this to another OpenRouter batch model -# (https://openrouter.ai/models?variant=batch) if you need a different -# cost/capability trade-off; Broad-Side looks up its per-token pricing -# automatically and uses it for cost estimates and the max_cost guardrail. -# Beware the expensive end of that list — some batch models exceed -# $80 per million output tokens. +# if you need a different cost/capability trade-off. To compare what's +# available, run codecarto_broadside with action "models" — it lists every +# :batch variant with pricing, context, output caps, structured-output +# support, and optional coding benchmarks. Beware the expensive end of that +# list — some batch models exceed $80 per million output tokens — and note +# that every lens requires structured-output (json_schema) support, which +# submit refuses to proceed without. # # model: google/gemini-3.7-flash:batch diff --git a/CHANGELOG.md b/CHANGELOG.md index 039bab8..ecc9fa3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ All notable changes to this project are documented here. The format is based on - **Broad-Side: batch reconnaissance over the OpenRouter Batch API** (#103). New `codecarto_broadside` MCP tool (actions: `submit`, `collect`, `status`) fires six single-turn analysis lenses — architecture, API surface, security, mechanical defect scan, convention extraction, porting — at any git repository as asynchronous batch jobs on a cheap batch model (~50% of sync pricing, unattended, 24h window), slices large modules by top-level directory, saves JSON plus rendered markdown to `.codecarto/broadside//`, and optionally synthesizes a cross-lens executive report. Works without an initialized workspace; needs an OpenRouter key via the `api_key` parameter, `OPENROUTER_API_KEY`, or `.codecarto/broadside/config.yaml`. Broad-Side findings are explicitly unverified scouting signals — file:line leads for the interactive pipeline to confirm, never evidence themselves. `codecarto_init` tolerates a `.codecarto/` that holds only `broadside/` (no force/backup needed), and scaffold refresh never touches broadside state, config, or results. - **Broad-Side: expense guardrails and live per-model pricing** (#103). `config.yaml` now accepts `model`, `max_cost`, and `pricing.input_per_m`/`output_per_m` overrides, and the MCP tool accepts `max_cost` and `force` parameters. Before submitting, Broad-Side estimates the run cost from collected file sizes (≈4 chars/token) against the configured model's per-token pricing — looked up live from OpenRouter's model catalog (cached 24h), so models like `openai/gpt-5.2-pro:batch` at ~$84/M output are priced correctly, not at the default model's rates. A submit whose estimate exceeds `max_cost` refuses with a per-lens breakdown and creates no run entry unless `force: true`. The submit response now reports the pricing used and its source (built-in/config/live/cache). +- **Broad-Side: model catalog action and capability pre-flight** (#103). New `models` action lists every `:batch` model on OpenRouter — pricing per million tokens, context window, completion ceiling, structured-output support, and optional Artificial Analysis coding indices (via `GET /api/v1/benchmarks`, attribution preserved) — cheapest first with the configured model marked. Submits now pre-flight the chosen model against that catalog: lens `max_tokens` clamps to the provider's completion ceiling, deprecated models are flagged, and models that do not advertise structured-output support are refused outright, since every lens depends on `json_schema` response_format. The catalog cache is shared between the `models` action and submit-time pricing resolution. ## [0.16.0] — 2026-08-17 diff --git a/ROADMAP.md b/ROADMAP.md index 4026e46..d267d9b 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -22,7 +22,11 @@ file only moves when a tier completes. - Live per-model pricing lookup (OpenRouter catalog, 24h cache) with `max_cost` expense guardrail and `force` override; configurable model (`config.yaml` `model` key, now wired through submissions). -- Tests: 26 unit tests (fake-fetcher based), opt-in live smoke script. +- `models` action: batch-model catalog with pricing, context, output caps, + structured-output support, and optional Artificial Analysis coding + benchmarks; submit pre-flight refuses models without structured outputs + and clamps lens `max_tokens` to the provider's completion ceiling. +- Tests: 31 unit tests (fake-fetcher based), opt-in live smoke script. ## Tier 1 — make Broad-Side better at what it does @@ -45,7 +49,7 @@ file only moves when a tier completes. | Item | Issue | Notes | |---|---|---| -| **Multi-model** — DeepSeek/Anthropic batch endpoints behind the lens registry | [#141](https://github.com/HuginnIndustries/CodeCartographer/issues/141) | Stronger batch models for semantic lenses at ~2-3× cost | +| **Multi-model** — DeepSeek/Anthropic batch endpoints behind the lens registry | [#141](https://github.com/HuginnIndustries/CodeCartographer/issues/141) | Partially shipped: catalog lookup, `models` action, pricing + capability pre-flight. Remaining: per-model prompt tweaks and a stronger default for semantic lenses | | **Incremental re-scouting** — diff against previous run's HEAD, rescan changed modules only | [#142](https://github.com/HuginnIndustries/CodeCartographer/issues/142) | Makes recurring scans O(delta) | | **CodeCartoShow pipeline stage** — BATCH-SCOUT between SELECT and the interactive run | [CodeCartoShow#1](https://github.com/HuginnIndustries/CodeCartoShow/issues/1) | `scripts/batch-analyze.py` proved it; evidence rules apply unchanged | diff --git a/core/broadside.ts b/core/broadside.ts index 68401c1..e370b3f 100644 --- a/core/broadside.ts +++ b/core/broadside.ts @@ -39,11 +39,13 @@ export const BROADSIDE_STATE_SCHEMA_VERSION = 1; export const BROADSIDE_INPUT_PRICE_PER_M = 0.1875; export const BROADSIDE_OUTPUT_PRICE_PER_M = 0.9375; -// OpenRouter's public model catalog; pricing lives per model id. +// OpenRouter's public model catalog; pricing, context, and capabilities live +// per model id. The benchmarks endpoint adds coding/intelligence indices. export const BROADSIDE_MODELS_URL = "https://openrouter.ai/api/v1/models"; +export const BROADSIDE_BENCHMARKS_URL = "https://openrouter.ai/api/v1/benchmarks"; -export const BROADSIDE_PRICING_CACHE_FILE = "pricing-cache.json"; -export const BROADSIDE_PRICING_CACHE_TTL_MS = 24 * 60 * 60 * 1000; +export const BROADSIDE_CATALOG_CACHE_FILE = "model-catalog.json"; +export const BROADSIDE_CATALOG_CACHE_TTL_MS = 24 * 60 * 60 * 1000; export const BROADSIDE_LENS_IDS = [ "architecture", @@ -69,6 +71,34 @@ export type ModelPricing = { source: "built-in" | "config" | "live" | "cache"; }; +/** The subset of the OpenRouter model catalog Broad-Side actually uses. */ +export type CatalogEntry = { + id: string; + name: string; + inputPerM: number; + outputPerM: number; + cachedInputPerM?: number; + contextLength?: number; + maxCompletionTokens?: number; + /** Empty array means unknown, not "supports nothing". */ + supportedParameters: string[]; + expirationDate?: string | null; +}; + +export type CodingBenchmarks = { + /** Base model slug (batch suffix stripped) → indices. */ + byBaseSlug: Record; + /** Citation/attribution metadata from the benchmarks endpoint. */ + meta: Record; +}; + +export type BroadsideCatalogResult = { + model: string; + source: "built-in" | "config" | "live" | "cache"; + entry: CatalogEntry | null; + benchmarks?: CodingBenchmarks; +}; + export type JsonSchemaDef = { name: string; strict: boolean; @@ -163,6 +193,12 @@ export type BroadsideSubmitResult = { estimatedOutputTokens: number; pricing: ModelPricing; maxCost?: number; + modelInfo: { + contextLength?: number; + maxCompletionTokens?: number; + supportsStructuredOutputs?: boolean; + expirationDate?: string | null; + }; }; export type BroadsideCollectResult = { @@ -1062,6 +1098,7 @@ export function buildBatchRequest( index: number, sliceCount: number, model: string = BROADSIDE_MODEL, + maxTokensOverride?: number, ): BatchRequest { const moduleTag = sanitizeId(slice.moduleName); const customId = sliceCount > 1 ? `${lens.id}-${moduleTag}-${index + 1}` : `${lens.id}-${moduleTag}`; @@ -1074,7 +1111,7 @@ export function buildBatchRequest( { role: "user", content: lens.userPrompt(info, slice.content, slice.moduleName) }, ], response_format: { type: "json_schema", json_schema: SCHEMAS[lens.schemaName] }, - max_tokens: lens.maxTokens, + max_tokens: maxTokensOverride ?? lens.maxTokens, }, }; } @@ -1083,13 +1120,14 @@ export function estimateCost( lens: LensDefinition, slices: FileSlice[], pricing: ModelPricing, + maxTokensOverride?: number, ): { inputTokens: number; outputTokens: number; cost: number; } { const inputTokens = Math.ceil(slices.reduce((sum, s) => sum + (lens.maxChars === 0 ? 6000 : s.chars), 0) / 4); - const outputTokens = Math.ceil(lens.maxTokens * 0.75); + const outputTokens = Math.ceil((maxTokensOverride ?? lens.maxTokens) * 0.75); const cost = (inputTokens / 1_000_000) * pricing.inputPerM + (outputTokens / 1_000_000) * pricing.outputPerM; @@ -1151,18 +1189,19 @@ export async function loadBroadsideConfig(broadsideDir: string): Promise; + fetched_at: string; + models: Record; }; -async function readPricingCache(broadsideDir: string): Promise { - const cachePath = join(broadsideDir, BROADSIDE_PRICING_CACHE_FILE); +async function readCatalogCache(broadsideDir: string): Promise { + const cachePath = join(broadsideDir, BROADSIDE_CATALOG_CACHE_FILE); if (!(await pathExists(cachePath))) return null; try { - const parsed = JSON.parse(await readFile(cachePath, "utf8")) as PricingCacheFile; + const parsed = JSON.parse(await readFile(cachePath, "utf8")) as CatalogCacheFile; if (!parsed || typeof parsed !== "object" || typeof parsed.models !== "object") return null; return parsed; } catch { @@ -1170,39 +1209,97 @@ async function readPricingCache(broadsideDir: string): Promise { +async function writeCatalogCache(broadsideDir: string, cache: CatalogCacheFile): Promise { await mkdir(broadsideDir, { recursive: true }); - await writeFile(join(broadsideDir, BROADSIDE_PRICING_CACHE_FILE), `${JSON.stringify(cache, null, "\t")}\n`, "utf8"); + await writeFile(join(broadsideDir, BROADSIDE_CATALOG_CACHE_FILE), `${JSON.stringify(cache, null, "\t")}\n`, "utf8"); } -export function builtInPricing(model: string): ModelPricing | null { +function parseCatalogEntry(raw: Record): CatalogEntry | null { + const id = String(raw.id ?? ""); + if (!id) return null; + const p = (raw.pricing ?? {}) as { prompt?: unknown; completion?: unknown; cached_input?: unknown }; + const input = typeof p.prompt === "string" ? Number(p.prompt) : NaN; + const output = typeof p.completion === "string" ? Number(p.completion) : NaN; + if (!Number.isFinite(input) || !Number.isFinite(output)) return null; + const cached = typeof p.cached_input === "string" ? Number(p.cached_input) : NaN; + const topProvider = (raw.top_provider ?? {}) as Record; + const contextLength = typeof raw.context_length === "number" ? raw.context_length : undefined; + const maxCompletion = + typeof topProvider.max_completion_tokens === "number" ? topProvider.max_completion_tokens : undefined; + return { + id, + name: String(raw.name ?? id), + inputPerM: input * 1_000_000, + outputPerM: output * 1_000_000, + cachedInputPerM: Number.isFinite(cached) ? cached * 1_000_000 : undefined, + contextLength, + maxCompletionTokens: maxCompletion, + supportedParameters: Array.isArray(raw.supported_parameters) + ? raw.supported_parameters.map((entry) => String(entry)) + : [], + expirationDate: typeof raw.expiration_date === "string" ? raw.expiration_date : null, + }; +} + +export function builtInCatalogEntry(model: string): CatalogEntry | null { + // The default model's rates are compile-time constants; its capabilities + // are asserted from the shipped configuration (1M context, 64K output, + // structured outputs used by every lens). if (model !== BROADSIDE_MODEL) return null; - return { inputPerM: BROADSIDE_INPUT_PRICE_PER_M, outputPerM: BROADSIDE_OUTPUT_PRICE_PER_M, source: "built-in" }; + return { + id: BROADSIDE_MODEL, + name: "Google: Gemini 3.7 Flash (batch)", + inputPerM: BROADSIDE_INPUT_PRICE_PER_M, + outputPerM: BROADSIDE_OUTPUT_PRICE_PER_M, + contextLength: 1_048_576, + maxCompletionTokens: 65_536, + supportedParameters: ["tools", "structured_outputs", "json_schema", "response_format"], + expirationDate: null, + }; } -export async function resolveModelPricing( +export function builtInPricing(model: string): ModelPricing | null { + const entry = builtInCatalogEntry(model); + if (!entry) return null; + return { inputPerM: entry.inputPerM, outputPerM: entry.outputPerM, source: "built-in" }; +} + +export async function resolveCatalogEntry( broadsideDir: string, config: BroadsideConfig, model: string, apiKey: string, fetcher: FetchLike = fetch as FetchLike, -): Promise { - // Manual overrides always win — the user is asserting a price, and a - // config assertion is cheaper to respect than to second-guess. +): Promise { + // Manual overrides always win for pricing — the user is asserting a rate, + // and a config assertion is cheaper to respect than to second-guess. + // Capabilities stay unknown in that case: nothing is refused, nothing + // is clamped, and the submit text says the pricing came from config. if (config.pricing) { - return { ...config.pricing, source: "config" }; + return { + model, + source: "config", + entry: { + id: model, + name: model, + inputPerM: config.pricing.inputPerM, + outputPerM: config.pricing.outputPerM, + supportedParameters: [], + }, + }; } - const builtIn = builtInPricing(model); - if (builtIn) return builtIn; - // Unknown model: check the on-disk cache first, then the live catalog. - const cache = await readPricingCache(broadsideDir); + const builtIn = builtInCatalogEntry(model); + if (builtIn) return { model, source: "built-in", entry: builtIn }; + + // Unknown model: on-disk cache first, then the live catalog. + const cache = await readCatalogCache(broadsideDir); const cached = cache?.models[model]; - if (cached && Date.now() - new Date(cached.fetchedAt).getTime() < BROADSIDE_PRICING_CACHE_TTL_MS) { - return { inputPerM: cached.inputPerM, outputPerM: cached.outputPerM, source: "cache" }; + if (cached && Date.now() - new Date(cache!.fetched_at).getTime() < BROADSIDE_CATALOG_CACHE_TTL_MS) { + return { model, source: "cache", entry: cached }; } - let live: { inputPerM: number; outputPerM: number } | null = null; + let live: CatalogEntry | null = null; try { const resp = await fetcher(BROADSIDE_MODELS_URL, { method: "GET", @@ -1211,23 +1308,20 @@ export async function resolveModelPricing( }); const data = (await resp.json()) as { data?: Array> }; const hit = (data.data ?? []).find((m) => String(m.id) === model); - if (hit && typeof hit.pricing === "object") { - const p = hit.pricing as { prompt?: string; completion?: string }; - const input = typeof p.prompt === "string" ? Number(p.prompt) : NaN; - const output = typeof p.completion === "string" ? Number(p.completion) : NaN; - if (Number.isFinite(input) && Number.isFinite(output)) { - live = { inputPerM: input * 1_000_000, outputPerM: output * 1_000_000 }; - } - } + if (hit) live = parseCatalogEntry(hit); } catch { live = null; } if (live) { - const updated: PricingCacheFile = { schema_version: 1, models: { ...(cache?.models ?? {}) } }; - updated.models[model] = { ...live, fetchedAt: new Date().toISOString() }; - await writePricingCache(broadsideDir, updated); - return { ...live, source: "live" }; + const updated: CatalogCacheFile = { + schema_version: 2, + fetched_at: new Date().toISOString(), + models: { ...(cache?.models ?? {}) }, + }; + updated.models[model] = live; + await writeCatalogCache(broadsideDir, updated); + return { model, source: "live", entry: live }; } throw new Error( @@ -1237,6 +1331,84 @@ export async function resolveModelPricing( ); } +export async function resolveModelPricing( + broadsideDir: string, + config: BroadsideConfig, + model: string, + apiKey: string, + fetcher: FetchLike = fetch as FetchLike, +): Promise { + const { source, entry } = await resolveCatalogEntry(broadsideDir, config, model, apiKey, fetcher); + if (!entry) throw new Error(`No pricing resolved for ${model}.`); + return { inputPerM: entry.inputPerM, outputPerM: entry.outputPerM, source }; +} + +/** Base slug with the OpenRouter variant suffix (e.g. `:batch`) stripped. */ +function baseSlug(modelId: string): string { + const idx = modelId.indexOf(":"); + return idx >= 0 ? modelId.slice(0, idx) : modelId; +} + +export async function fetchCodingBenchmarks( + apiKey: string, + fetcher: FetchLike = fetch as FetchLike, +): Promise { + try { + const resp = await fetcher(`${BROADSIDE_BENCHMARKS_URL}?source=artificial-analysis&task_type=coding`, { + method: "GET", + headers: { Authorization: `Bearer ${apiKey}` }, + signal: AbortSignal.timeout(30_000), + }); + const data = (await resp.json()) as { data?: Array>; meta?: Record }; + const byBaseSlug: CodingBenchmarks["byBaseSlug"] = {}; + for (const row of data.data ?? []) { + const slug = baseSlug(String(row.model_permaslug ?? "")); + if (!slug) continue; + const toIndex = (v: unknown) => (typeof v === "number" && Number.isFinite(v) ? v : undefined); + byBaseSlug[slug] = { + codingIndex: toIndex(row.coding_index), + intelligenceIndex: toIndex(row.intelligence_index), + }; + } + return { byBaseSlug, meta: data.meta ?? {} }; + } catch { + return null; + } +} + +export async function listBatchModels( + broadsideDir: string, + config: BroadsideConfig, + apiKey: string, + opts: { includeBenchmarks?: boolean; fetcher?: FetchLike } = {}, +): Promise<{ entries: CatalogEntry[]; source: string; benchmarks: CodingBenchmarks | null; defaultModel: string }> { + const fetcher = opts.fetcher ?? (fetch as FetchLike); + const resp = await fetcher(BROADSIDE_MODELS_URL, { + method: "GET", + headers: { Authorization: `Bearer ${apiKey}` }, + signal: AbortSignal.timeout(30_000), + }); + const data = (await resp.json()) as { data?: Array> }; + const entries: CatalogEntry[] = []; + const seen = new Set(); + for (const raw of data.data ?? []) { + const entry = parseCatalogEntry(raw); + if (!entry || seen.has(entry.id)) continue; + seen.add(entry.id); + if (!entry.id.endsWith(":batch")) continue; + entries.push(entry); + } + entries.sort((a, b) => a.inputPerM + a.outputPerM - (b.inputPerM + b.outputPerM)); + + // Persist the catalog so the next submit's pricing resolution hits cache. + const cache: CatalogCacheFile = { schema_version: 2, fetched_at: new Date().toISOString(), models: {} }; + for (const entry of entries) cache.models[entry.id] = entry; + await writeCatalogCache(broadsideDir, cache); + + const benchmarks = opts.includeBenchmarks ? await fetchCodingBenchmarks(apiKey, fetcher) : null; + return { entries, source: "live", benchmarks, defaultModel: config.model }; +} + // ---------- batch client ---------- export type FetchLike = (url: string, init: Record) => Promise; @@ -1340,27 +1512,52 @@ export async function runBroadsideSubmit( const broadsideDir = broadsideDirFor(cwd); const model = opts.model ?? BROADSIDE_MODEL; - // Resolve pricing before anything is submitted: the guardrail must know - // the model's real per-token rates, not the default model's. + // Resolve the model's catalog entry before anything is submitted: the + // guardrail must know real per-token rates, and every lens requires + // structured-output support that not all batch models offer. const config = await loadBroadsideConfig(broadsideDir); - const pricing = await resolveModelPricing(broadsideDir, config, model, apiKey, opts.fetcher); + const catalog = await resolveCatalogEntry(broadsideDir, config, model, apiKey, opts.fetcher); + const pricing: ModelPricing = { + inputPerM: catalog.entry!.inputPerM, + outputPerM: catalog.entry!.outputPerM, + source: catalog.source, + }; const limit = opts.maxCost ?? config.maxCost; + const entry = catalog.entry!; + const supportsStructuredOutputs = + entry.supportedParameters.length === 0 || + entry.supportedParameters.some((p) => + ["structured_outputs", "json_schema", "response_format", "structuredoutputs"].includes(p.toLowerCase()), + ); + if (!supportsStructuredOutputs) { + throw new Error( + `Batch model "${model}" does not advertise structured-output support ` + + `(supported_parameters: ${entry.supportedParameters.join(", ") || "unknown"}), but every ` + + "Broad-Side lens requires json_schema response_format. Choose another batch model " + + "(codecarto_broadside action 'models') or pass a pricing override only if you know it works.", + ); + } + // Respect the provider's completion ceiling: a request asking for more + // output than the model can produce fails the whole batch. + const outputCap = entry.maxCompletionTokens; + // Slice offline first so the estimate covers every request we would send. const slicesByLens = new Map(); let estimatedInputTokens = 0; let estimatedOutputTokens = 0; let estimatedTotalCost = 0; - const perLensEstimate: Array<{ lens: LensDefinition; cost: number }> = []; + const perLensEstimate: Array<{ lens: LensDefinition; cost: number; maxTokens: number }> = []; for (const lensId of lensIds) { const lens = getLens(lensId); const slices = await gatherSlices(cwd, lens, info); slicesByLens.set(lensId, slices); - const estimate = estimateCost(lens, slices, pricing); + const maxTokens = outputCap ? Math.min(lens.maxTokens, outputCap) : lens.maxTokens; + const estimate = estimateCost(lens, slices, pricing, maxTokens); estimatedInputTokens += estimate.inputTokens; estimatedOutputTokens += estimate.outputTokens; estimatedTotalCost += estimate.cost; - perLensEstimate.push({ lens, cost: estimate.cost }); + perLensEstimate.push({ lens, cost: estimate.cost, maxTokens }); } if (limit > 0 && !opts.force && estimatedTotalCost > limit) { @@ -1395,8 +1592,9 @@ export async function runBroadsideSubmit( for (const lensId of lensIds) { const lens = getLens(lensId); const slices = slicesByLens.get(lensId) ?? []; - const requests = slices.map((s, i) => buildBatchRequest(lens, info, s, i, slices.length, model)); - const estimate = estimateCost(lens, slices, pricing); + const maxTokens = outputCap ? Math.min(lens.maxTokens, outputCap) : lens.maxTokens; + const requests = slices.map((s, i) => buildBatchRequest(lens, info, s, i, slices.length, model, maxTokens)); + const estimate = estimateCost(lens, slices, pricing, maxTokens); const entry: BroadsideBatchEntry = { batchId: "", @@ -1443,6 +1641,12 @@ export async function runBroadsideSubmit( estimatedOutputTokens, pricing, maxCost: limit > 0 ? limit : undefined, + modelInfo: { + contextLength: entry.contextLength, + maxCompletionTokens: entry.maxCompletionTokens, + supportsStructuredOutputs: entry.supportedParameters.length === 0 ? undefined : supportsStructuredOutputs, + expirationDate: entry.expirationDate ?? null, + }, }; } @@ -1751,6 +1955,15 @@ export function estimateSubmitText(result: BroadsideSubmitResult, lenses: LensDe `Estimated total: ~$${result.estimatedTotalCost.toFixed(4)}`, `Pricing: $${result.pricing.inputPerM.toFixed(4)}/M in, $${result.pricing.outputPerM.toFixed(4)}/M out (${result.pricing.source})`, ); + if (result.modelInfo.contextLength) { + lines.push(`Model: ${result.modelInfo.contextLength.toLocaleString()} context, ${result.modelInfo.maxCompletionTokens?.toLocaleString() ?? "?"} max output`); + } + if (result.modelInfo.supportsStructuredOutputs === false) { + lines.push("Warning: model does not advertise structured-output support; lens JSON may be unreliable."); + } + if (result.modelInfo.expirationDate) { + lines.push(`Warning: this model is deprecated (expires ${result.modelInfo.expirationDate}).`); + } if (result.maxCost) { lines.push(`Run limit: $${result.maxCost.toFixed(2)} (enforced on estimate; pass force to override)`); } @@ -1762,6 +1975,42 @@ export function estimateSubmitText(result: BroadsideSubmitResult, lenses: LensDe return lines.join("\n"); } +export function modelsText( + entries: CatalogEntry[], + opts: { benchmarks: CodingBenchmarks | null; defaultModel: string }, +): string { + const lines = [ + `Batch models on OpenRouter (${entries.length}, cheapest first).`, + "", + "id | $/M in | $/M out | ctx | max out | structured | coding idx", + ]; + for (const entry of entries) { + const bench = opts.benchmarks?.byBaseSlug[baseSlug(entry.id)]; + const structured = entry.supportedParameters.length === 0 + ? "?" + : entry.supportedParameters.some((p) => ["structured_outputs", "json_schema", "response_format", "structuredoutputs"].includes(p.toLowerCase())) + ? "yes" + : "no"; + const coding = bench?.codingIndex !== undefined ? bench.codingIndex.toFixed(1) : "-"; + const ctx = entry.contextLength + ? entry.contextLength >= 1_000_000 + ? `${(entry.contextLength / 1_000_000).toFixed(1)}M` + : `${(entry.contextLength / 1024).toFixed(0)}k` + : "?"; + const out = entry.maxCompletionTokens ? `${(entry.maxCompletionTokens / 1024).toFixed(0)}k` : "?"; + const tag = entry.id === opts.defaultModel ? " (default)" : ""; + const exp = entry.expirationDate ? " [deprecated]" : ""; + lines.push( + `${entry.id}${tag}${exp} | ${entry.inputPerM.toFixed(3)} | ${entry.outputPerM.toFixed(3)} | ${ctx} | ${out} | ${structured} | ${coding}`, + ); + } + if (opts.benchmarks?.meta.as_of) { + lines.push("", `Benchmarks: Artificial Analysis coding index (as of ${String(opts.benchmarks.meta.as_of)}).`); + } + lines.push("", "Set the batch model in .codecarto/broadside/config.yaml (model key). Higher coding index ≠ better scout: precision, context, and structured-output support matter most here."); + return lines.join("\n"); +} + export function collectResultText(result: BroadsideCollectResult): string { const lines = [ `Broad-Side run ${result.runId}: ${result.status}`, diff --git a/mcp-server/server.ts b/mcp-server/server.ts index 7544872..84d0ab9 100644 --- a/mcp-server/server.ts +++ b/mcp-server/server.ts @@ -55,9 +55,11 @@ import { isWithinPathResolved, type LibraryIndexEntry, type LibraryVisibility, + listBatchModels, listEntries, listGuideTopics, loadBroadsideConfig, + modelsText, readGuide, listSkillNames, loadCodecartoConfig, @@ -991,18 +993,19 @@ function resolveBroadsideApiKey(explicit: string | undefined, config: { apiKey: export async function handleBroadside(args: { cwd: string; - action: "submit" | "collect" | "status"; + action: "submit" | "collect" | "status" | "models"; lenses?: string[]; api_key?: string; wait_seconds?: number; include_synthesis?: boolean; max_cost?: number; force?: boolean; + include_benchmarks?: boolean; }) { const cwd = await validateCwd(args.cwd); const action = args.action ?? "submit"; - if (!["submit", "collect", "status"].includes(action)) { - throw new McpError(ErrorCode.InvalidParams, `Unknown action: ${action}. Valid actions: submit, collect, status.`); + if (!["submit", "collect", "status", "models"].includes(action)) { + throw new McpError(ErrorCode.InvalidParams, `Unknown action: ${action}. Valid actions: submit, collect, status, models.`); } const config = await loadBroadsideConfig(broadsideDirFor(cwd)); @@ -1015,6 +1018,19 @@ export async function handleBroadside(args: { const apiKey = resolveBroadsideApiKey(args.api_key, config); const waitMs = typeof args.wait_seconds === "number" && args.wait_seconds > 0 ? args.wait_seconds * 1000 : undefined; + if (action === "models") { + const { entries, benchmarks } = await listBatchModels(broadsideDirFor(cwd), config, apiKey, { + includeBenchmarks: args.include_benchmarks === true, + }).catch((error) => { + throw new McpError(ErrorCode.InvalidRequest, error instanceof Error ? error.message : String(error)); + }); + return textResult(modelsText(entries, { benchmarks, defaultModel: config.model }), { + models: entries, + defaultModel: config.model, + benchmarkMeta: benchmarks?.meta ?? null, + }); + } + if (action === "submit") { let lenses: BroadsideLensId[]; if (args.lenses && args.lenses.length > 0) { @@ -1368,15 +1384,15 @@ const TOOLS = [ { name: "codecarto_broadside", description: - "Broad-Side: fire a cheap batch reconnaissance scan at a repository via the OpenRouter Batch API. Six lenses (architecture, api, security, defect, conventions, porting) run as asynchronous single-turn prompts with structured JSON schemas; results land in .codecarto/broadside// as JSON plus markdown, with an optional cross-lens synthesis report. Works on any git repository — no CodeCartographer workspace required. Requires an OpenRouter API key (api_key param, OPENROUTER_API_KEY env var, or .codecarto/broadside/config.yaml). Findings are unverified scouting signals from a batch model, not validated claims — they tell the interactive pipeline where to look. Actions: submit (fire batches, returns batch ids and cost estimate), collect (poll to completion, save results, optionally synthesize), status (show recorded runs).", + "Broad-Side: fire a cheap batch reconnaissance scan at a repository via the OpenRouter Batch API. Six lenses (architecture, api, security, defect, conventions, porting) run as asynchronous single-turn prompts with structured JSON schemas; results land in .codecarto/broadside// as JSON plus markdown, with an optional cross-lens synthesis report. Works on any git repository — no CodeCartographer workspace required. Requires an OpenRouter API key (api_key param, OPENROUTER_API_KEY env var, or .codecarto/broadside/config.yaml). Findings are unverified scouting signals from a batch model, not validated claims — they tell the interactive pipeline where to look. Actions: submit (fire batches, returns batch ids and cost estimate), collect (poll to completion, save results, optionally synthesize), status (show recorded runs), models (list batch-capable models with pricing, context, output caps, structured-output support, and optional coding benchmarks).", inputSchema: { type: "object", properties: { cwd: { type: "string", description: "Absolute path to the target repository." }, action: { type: "string", - enum: ["submit", "collect", "status"], - description: "submit fires all lens batches and returns batch ids; collect polls submitted batches, saves results, and optionally runs the synthesis pass; status shows recorded runs.", + enum: ["submit", "collect", "status", "models"], + description: "submit fires all lens batches and returns batch ids; collect polls submitted batches, saves results, and optionally runs the synthesis pass; status shows recorded runs; models lists batch-capable models with pricing and capabilities.", }, lenses: { type: "array", @@ -1404,6 +1420,10 @@ const TOOLS = [ type: "boolean", description: "Submit even when the cost estimate exceeds max_cost (default false).", }, + include_benchmarks: { + type: "boolean", + description: "For action 'models': annotate each model with its Artificial Analysis coding index (extra API call; default false).", + }, }, required: ["cwd", "action"], }, diff --git a/tests/broadside.test.mjs b/tests/broadside.test.mjs index cbf37b6..779a798 100644 --- a/tests/broadside.test.mjs +++ b/tests/broadside.test.mjs @@ -21,11 +21,14 @@ const { collectRepoInfo, defaultBroadsideState, estimateCost, + fetchCodingBenchmarks, gatherSlices, getLens, + listBatchModels, loadBroadsideConfig, loadBroadsideState, listLenses, + modelsText, renderFindingsMarkdown, resolveModelPricing, runBroadsideCollect, @@ -434,6 +437,131 @@ test("submit passes the configured model into batch payloads", async () => { } }); +// ---------- model catalog & models action ---------- + +function catalogWith(...models) { + return { data: models }; +} + +const BATCH_MODEL_SHAPE = (id, prompt, completion, extra = {}) => ({ + id, + name: id, + pricing: { prompt: String(prompt), completion: String(completion) }, + context_length: 1_000_000, + top_provider: { max_completion_tokens: 65_536 }, + supported_parameters: ["tools", "structured_outputs"], + expiration_date: null, + ...extra, +}); + +test("listBatchModels keeps only :batch variants and sorts cheapest first", async () => { + const dir = await mkdtemp(join(tmpdir(), "broadside-models-")); + try { + const config = { model: BROADSIDE_MODEL, apiKey: "", defaultLenses: ["architecture"], maxCost: 0, pricing: null }; + const fetcher = async () => + fakeResponse( + 200, + catalogWith( + BATCH_MODEL_SHAPE("openai/gpt-5.2-pro:batch", 0.00000375, 0.000084), + BATCH_MODEL_SHAPE("google/gemini-3.7-flash:batch", 0.0000001875, 0.0000009375), + { id: "openai/gpt-5.2-pro", pricing: { prompt: "0.00001", completion: "0.0001" } }, // non-batch, must be excluded + BATCH_MODEL_SHAPE("deepseek/deepseek-v4-pro:batch", 0.000000481, 0.000000963), + ), + ); + const { entries } = await listBatchModels(dir, config, "sk-fake", { fetcher }); + assert.equal(entries.length, 3); + assert.ok(!entries.some((e) => !e.id.endsWith(":batch"))); + assert.equal(entries[0].id, "google/gemini-3.7-flash:batch", "cheapest first"); + assert.equal(entries[2].id, "openai/gpt-5.2-pro:batch", "most expensive last"); + } finally { + await rm(dir, { recursive: true, force: true }); + } +}); + +test("submit refuses models that do not advertise structured outputs", async () => { + const dir = await mkdtemp(join(tmpdir(), "broadside-cap-")); + try { + await writeFile(join(dir, "go.mod"), "module x\n"); + await writeFile(join(dir, "main.go"), "package main\n"); + const fetcher = async (url, init) => { + if (init.method === "POST") { + return fakeResponse(202, { id: "batch-x", status: "validating" }); + } + return fakeResponse( + 200, + catalogWith(BATCH_MODEL_SHAPE("vendor/no-structured:batch", 0.0000001, 0.0000002, { supported_parameters: ["tools"] })), + ); + }; + await assert.rejects( + () => runBroadsideSubmit(dir, "sk-fake", { lenses: ["architecture"], fetcher, model: "vendor/no-structured:batch" }), + /does not advertise structured-output support/, + ); + } finally { + await rm(dir, { recursive: true, force: true }); + } +}); + +test("submit clamps lens max_tokens to the provider completion ceiling", async () => { + const dir = await mkdtemp(join(tmpdir(), "broadside-clamp-")); + try { + await writeFile(join(dir, "go.mod"), "module x\n"); + await writeFile(join(dir, "main.go"), "package main\n"); + let payload; + const fetcher = async (url, init) => { + if (init.method === "POST") { + payload = JSON.parse(init.body); + return fakeResponse(202, { id: "batch-c", status: "validating" }); + } + return fakeResponse( + 200, + catalogWith(BATCH_MODEL_SHAPE("vendor/tiny-out:batch", 0.0000001, 0.0000002, { top_provider: { max_completion_tokens: 1000 } })), + ); + }; + await runBroadsideSubmit(dir, "sk-fake", { lenses: ["architecture"], fetcher, model: "vendor/tiny-out:batch" }); + assert.equal(payload.requests[0].body.max_tokens, 1000, "8000-token lens must clamp to the 1000-token ceiling"); + } finally { + await rm(dir, { recursive: true, force: true }); + } +}); + +test("fetchCodingBenchmarks maps indices by base slug (batch suffix stripped)", async () => { + const fetcher = async () => + fakeResponse( + 200, + { + data: [{ model_permaslug: "google/gemini-3.7-flash", coding_index: 62.4, intelligence_index: 58.1 }], + meta: { as_of: "2026-08-23", source_url: "https://example.com" }, + }, + ); + const benchmarks = await fetchCodingBenchmarks("sk-fake", fetcher); + assert.equal(benchmarks.byBaseSlug["google/gemini-3.7-flash"].codingIndex, 62.4); + assert.ok("google/gemini-3.7-flash:batch".indexOf(":") >= 0, "the batch variant resolves through its base slug"); + assert.equal(benchmarks.meta.as_of, "2026-08-23"); +}); + +test("modelsText renders pricing, caps, support, and benchmark columns", () => { + const entries = [ + { + id: "google/gemini-3.7-flash:batch", + name: "Google: Gemini 3.7 Flash (batch)", + inputPerM: 0.1875, + outputPerM: 0.9375, + contextLength: 1_048_576, + maxCompletionTokens: 65_536, + supportedParameters: ["tools", "structured_outputs"], + expirationDate: null, + }, + ]; + const text = modelsText(entries, { + benchmarks: { byBaseSlug: { "google/gemini-3.7-flash": { codingIndex: 62.4 } }, meta: { as_of: "2026-08-23" } }, + defaultModel: "google/gemini-3.7-flash:batch", + }); + assert.match(text, /0\.188/); + assert.match(text, /64k/); + assert.match(text, /62\.4/); + assert.match(text, /\(default\)/); +}); + // ---------- batch client with a fake fetcher ---------- function fakeResponse(status, body) { From 32530b3e2e265ef4e3222c199cfbb656bb3f8512 Mon Sep 17 00:00:00 2001 From: James Sesler Date: Sun, 23 Aug 2026 04:01:09 -0400 Subject: [PATCH 6/7] feat: broadside truncation detection with fence-tolerant parsing (#103, #133) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reviewing OpenRouter's standard patterns (OpenRouterTeam/skills create-headless-agent) surfaced three adoptable ideas: - Fence-tolerant output parsing: lens content wrapped in markdown code fences now parses as JSON, mirroring the scaffold's --output-schema tolerance. Saved .json files hold clean parsed output instead of verbatim fenced text. - Truncation is spoken: output that still does not parse after fence stripping (the signature of a max_tokens cutoff) is saved verbatim but flagged truncated. Collect summaries, run-meta.json, and the synthesis prompt all report it, so an unscouted module is never mistaken for a clean one. - Invariants written down: the resubmission-safety rule (batch requests are pure, so retries are always safe — load-bearing if server tools ever arrive) and the distinction between the pre-flight max_cost estimate and OpenRouter's runtime cost accounting, in the module header, SKILL.md, and config.yaml. Catalog field shapes are attributed to the official skills repo. New feat issue #143 records the headless-agent lens-queue question and the hybrid verify-pass idea. 4 new tests; 376 total, all passing. --- .codecarto/broadside/SKILL.md | 22 +++++++ .codecarto/broadside/config.yaml | 4 ++ CHANGELOG.md | 1 + ROADMAP.md | 8 ++- core/broadside.ts | 105 +++++++++++++++++++++++++++---- mcp-server/server.ts | 1 + tests/broadside.test.mjs | 52 ++++++++++++++- 7 files changed, 177 insertions(+), 16 deletions(-) diff --git a/.codecarto/broadside/SKILL.md b/.codecarto/broadside/SKILL.md index 869a822..3bd41f7 100644 --- a/.codecarto/broadside/SKILL.md +++ b/.codecarto/broadside/SKILL.md @@ -82,3 +82,25 @@ collected file sizes against the model's live per-token pricing and refuses when the estimate exceeds `max_cost` (`config.yaml` or the tool parameter) unless `force` is passed. See `config.yaml` for the model, limit, and manual pricing-override keys. + +The `max_cost` guardrail is an **estimate-based pre-flight limit**, distinct +from OpenRouter's runtime cost tracking: it predicts from file sizes before +spend, it does not stop a batch mid-flight. Actual spend appears in +`run-meta.json` after collect. + +## Resilience notes + +- **Truncation is spoken.** A lens output whose JSON does not parse — even + after code-fence stripping — is saved verbatim but marked `truncated`: + the collect summary counts it, `run-meta.json` records it, and the + synthesis prompt is told its module is unrepresented, not clean. +- **Resubmission is always safe.** Batch requests are pure (no tools, no + filesystem, no side effects), so a failed or truncated slice can be + resubmitted freely. This is the same retry-safety rule OpenRouter's + headless-agent scaffold enforces for tool-using agents ("retry only + before tool calls"); Broad-Side satisfies it by construction. If + Broad-Side ever gains server tools, this invariant becomes load-bearing. +- **Field shapes** for the model catalog and benchmarks endpoints follow + the official OpenRouter skills (`OpenRouterTeam/skills`: + `openrouter-models`, `openrouter-benchmarks`) — consult them when + extending catalog parsing. diff --git a/.codecarto/broadside/config.yaml b/.codecarto/broadside/config.yaml index 8d82315..93604e0 100644 --- a/.codecarto/broadside/config.yaml +++ b/.codecarto/broadside/config.yaml @@ -39,6 +39,10 @@ # prints the per-lens breakdown; pass force: true to override, or set a # value here so every run is guarded by default. # +# This is a pre-flight estimate guardrail, not a runtime stop: OpenRouter +# bills actual usage, which may differ from the estimate either direction. +# Actual cost lands in each run's run-meta.json after collect. +# # max_cost: 1.00 # Manual pricing overrides in USD per MILLION tokens. Normally Broad-Side diff --git a/CHANGELOG.md b/CHANGELOG.md index ecc9fa3..6c52123 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ All notable changes to this project are documented here. The format is based on - **Broad-Side: batch reconnaissance over the OpenRouter Batch API** (#103). New `codecarto_broadside` MCP tool (actions: `submit`, `collect`, `status`) fires six single-turn analysis lenses — architecture, API surface, security, mechanical defect scan, convention extraction, porting — at any git repository as asynchronous batch jobs on a cheap batch model (~50% of sync pricing, unattended, 24h window), slices large modules by top-level directory, saves JSON plus rendered markdown to `.codecarto/broadside//`, and optionally synthesizes a cross-lens executive report. Works without an initialized workspace; needs an OpenRouter key via the `api_key` parameter, `OPENROUTER_API_KEY`, or `.codecarto/broadside/config.yaml`. Broad-Side findings are explicitly unverified scouting signals — file:line leads for the interactive pipeline to confirm, never evidence themselves. `codecarto_init` tolerates a `.codecarto/` that holds only `broadside/` (no force/backup needed), and scaffold refresh never touches broadside state, config, or results. - **Broad-Side: expense guardrails and live per-model pricing** (#103). `config.yaml` now accepts `model`, `max_cost`, and `pricing.input_per_m`/`output_per_m` overrides, and the MCP tool accepts `max_cost` and `force` parameters. Before submitting, Broad-Side estimates the run cost from collected file sizes (≈4 chars/token) against the configured model's per-token pricing — looked up live from OpenRouter's model catalog (cached 24h), so models like `openai/gpt-5.2-pro:batch` at ~$84/M output are priced correctly, not at the default model's rates. A submit whose estimate exceeds `max_cost` refuses with a per-lens breakdown and creates no run entry unless `force: true`. The submit response now reports the pricing used and its source (built-in/config/live/cache). - **Broad-Side: model catalog action and capability pre-flight** (#103). New `models` action lists every `:batch` model on OpenRouter — pricing per million tokens, context window, completion ceiling, structured-output support, and optional Artificial Analysis coding indices (via `GET /api/v1/benchmarks`, attribution preserved) — cheapest first with the configured model marked. Submits now pre-flight the chosen model against that catalog: lens `max_tokens` clamps to the provider's completion ceiling, deprecated models are flagged, and models that do not advertise structured-output support are refused outright, since every lens depends on `json_schema` response_format. The catalog cache is shared between the `models` action and submit-time pricing resolution. +- **Broad-Side: truncation detection with fence-tolerant parsing** (#103, #133). Lens output is now parsed tolerantly — markdown code fences are stripped before JSON parsing, mirroring the tolerance in OpenRouter's headless-agent scaffold. Parseable output is saved as clean JSON; output that still does not parse (the signature of a `max_tokens` cutoff) is saved verbatim but marked `truncated`. The collect summary and `run-meta.json` report truncation counts, and the synthesis prompt is told which modules are unrepresented rather than clean. Also documented the retry-safety invariant (batch requests are pure, resubmission always safe) and the distinction between Broad-Side's pre-flight `max_cost` estimate and OpenRouter's runtime cost accounting. ## [0.16.0] — 2026-08-17 diff --git a/ROADMAP.md b/ROADMAP.md index d267d9b..a2377b1 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -33,7 +33,7 @@ file only moves when a tier completes. | Item | Issue | Notes | |---|---|---| | **Triage lens** — prioritized fix queue (impact × difficulty, grouped by module) | [#135](https://github.com/HuginnIndustries/CodeCartographer/issues/135) | Highest-value next lens: turns leads into a work order | -| **Truncation repair** — detect max_tokens-cutoff JSON, resubmit slices, report truncation in summaries | [#133](https://github.com/HuginnIndustries/CodeCartographer/issues/133) | Found in the self-scan: 3/10 defect slices truncated | +| **Truncation repair** — detect max_tokens-cutoff JSON, resubmit slices, report truncation in summaries | [#133](https://github.com/HuginnIndustries/CodeCartographer/issues/133) | Partially shipped: fence-tolerant parsing + `truncated` flagging in collect, meta, and synthesis. Remaining: automatic resubmit of truncated slices | | **Concurrent polling** — poll all in-flight batches round-robin against one deadline | [#136](https://github.com/HuginnIndustries/CodeCartographer/issues/136) | Submissions already parallel; polling is sequential today | | **Per-language prompts** — Go/Python/Rust/TS lens prompts; globs already adapt | [#137](https://github.com/HuginnIndustries/CodeCartographer/issues/137) | Schemas stay shared so synthesis is unaffected | @@ -53,6 +53,12 @@ file only moves when a tier completes. | **Incremental re-scouting** — diff against previous run's HEAD, rescan changed modules only | [#142](https://github.com/HuginnIndustries/CodeCartographer/issues/142) | Makes recurring scans O(delta) | | **CodeCartoShow pipeline stage** — BATCH-SCOUT between SELECT and the interactive run | [CodeCartoShow#1](https://github.com/HuginnIndustries/CodeCartoShow/issues/1) | `scripts/batch-analyze.py` proved it; evidence rules apply unchanged | +## Tier 4 — open questions, not commitments + +| Item | Issue | Notes | +|---|---|---| +| **Headless-agent lens queue** — sync-priced, tool-using variant via `@openrouter/agent` | [#143](https://github.com/HuginnIndustries/CodeCartographer/issues/143) | 2× batch pricing; overlaps the interactive pipeline. The likelier winner is the hybrid: batch sweeps + one sync-priced verification pass on the top N findings | + ## Principles 1. **Leads, never evidence.** Every Broad-Side artifact carries the diff --git a/core/broadside.ts b/core/broadside.ts index e370b3f..b17d696 100644 --- a/core/broadside.ts +++ b/core/broadside.ts @@ -14,6 +14,19 @@ // of labor is the point: a ~$0.50 unattended sweep that tells the expensive // interactive run where to look. // +// Field shapes for the model catalog and benchmarks endpoints follow the +// official OpenRouter skills (OpenRouterTeam/skills: openrouter-models, +// openrouter-benchmarks). +// +// RESUBMISSION INVARIANT: batch requests are pure functions of their input — +// no tools, no filesystem, no side effects — so resubmitting a failed or +// truncated slice is always safe. This is the retry rule OpenRouter's own +// headless-agent scaffold states the hard way (retry only before tool calls, +// because replaying a mutating tool would double-execute it); here the rule is +// satisfied by construction. If Broad-Side ever gains server tools +// (openrouter:web_search etc.), this invariant becomes load-bearing and the +// resubmit path must gate on whether any tool executed. +// // Deliberately not in .codecarto/ template prose: Broad-Side requires runtime // code, so it lives on the executable surfaces (MCP today, Pi on the roadmap). @@ -206,7 +219,12 @@ export type BroadsideCollectResult = { status: string; totalCost: number; resultCount: number; - lensOutcomes: Partial>; + /** Results whose JSON did not parse even after fence stripping — + * the signature of an output cut off at max_tokens. */ + truncatedCount: number; + lensOutcomes: Partial< + Record + >; synthesis: BroadsideSynthesisEntry; topFindings: { title: string; severity: string; sourceLens: string; summary: string }[]; }; @@ -1656,6 +1674,9 @@ export type StoredLensResult = { moduleName: string; content: string; raw: Record; + /** True when the content is not parseable JSON even after fence stripping — + * the telltale of an output cut off at max_tokens. */ + truncated: boolean; }; function extractContent(result: Record): string | null { @@ -1667,6 +1688,25 @@ function extractContent(result: Record): string | null { return typeof message?.content === "string" ? message.content : null; } +/** + * Parse lens content as JSON, tolerating the markdown code fences some models + * wrap structured output in (the same tolerance OpenRouter's headless-agent + * scaffold ships for --output-schema). Returns null when the content is not + * JSON at all — which for a strict json_schema request means the output was + * truncated at max_tokens, not that the model chose prose. + */ +export function parseLensJson(content: string): unknown | null { + const trimmed = content.trim(); + const fenced = /^```(?:json)?\s*\n?([\s\S]*?)\n?```\s*$/.exec(trimmed); + const candidate = fenced ? fenced[1].trim() : trimmed; + if (!candidate.startsWith("{") && !candidate.startsWith("[")) return null; + try { + return JSON.parse(candidate); + } catch { + return null; + } +} + export async function saveLensResults( runDir: string, lensId: BroadsideLensId, @@ -1683,9 +1723,24 @@ export async function saveLensResults( } continue; } - await writeFile(join(runDir, `${sanitizeId(customId)}.json`), `${content}\n`, "utf8"); + const parsed = parseLensJson(content); + const truncated = parsed === null; + if (parsed !== null) { + await writeFile(join(runDir, `${sanitizeId(customId)}.json`), `${JSON.stringify(parsed, null, "\t")}\n`, "utf8"); + } else { + // Save the raw bytes verbatim so nothing is lost, but name the + // gap: an unparseable strict-schema response is a truncation. + await writeFile(join(runDir, `${sanitizeId(customId)}.json`), `${content}\n`, "utf8"); + } await writeFile(join(runDir, `${sanitizeId(customId)}.md`), renderFindingsMarkdown(content), "utf8"); - out.push({ lensId, customId, moduleName: String(customId).replace(/^[a-z]+-/, ""), content, raw: result }); + out.push({ + lensId, + customId, + moduleName: String(customId).replace(/^[a-z]+-/, ""), + content, + raw: result, + truncated, + }); } return out; } @@ -1713,6 +1768,7 @@ export async function runBroadsideCollect( const deadline = Date.now() + (opts.waitMs ?? BROADSIDE_DEFAULT_POLL_BUDGET_MS); let totalCost = 0; let resultCount = 0; + let truncatedCount = 0; const lensOutcomes: BroadsideCollectResult["lensOutcomes"] = {}; const allLensResults: StoredLensResult[] = []; @@ -1745,18 +1801,21 @@ export async function runBroadsideCollect( entry.completedAt = new Date().toISOString(); const stored = await saveLensResults(runDir, lensId, batch); entry.resultCount = stored.length; + const truncated = stored.filter((s) => s.truncated).length; allLensResults.push(...stored); resultCount += stored.length; + truncatedCount += truncated; totalCost += cost ?? 0; await writeFile( join(runDir, `raw-${lensId}.json`), `${JSON.stringify(batch, null, "\t")}\n`, "utf8", ); + lensOutcomes[lensId] = { status, cost: entry.cost, resultCount: entry.resultCount, truncated }; } else if (batch.error) { entry.error = batch.error; + lensOutcomes[lensId] = { status, cost: entry.cost, resultCount: entry.resultCount }; } - lensOutcomes[lensId] = { status, cost: entry.cost, resultCount: entry.resultCount }; await saveBroadsideState(broadsideDir, state); } @@ -1771,6 +1830,12 @@ export async function runBroadsideCollect( const findingsText = allLensResults .map((r) => `## ${r.lensId} — ${r.customId}\n\n${r.content}\n`) .join("\n"); + const truncatedNote = + truncatedCount > 0 + ? `\n\nNOTE: ${truncatedCount} lens result(s) were truncated at the output token limit and are ` + + "not included above. Any gap they would have covered is unrepresented — do not treat " + + "silence on a module as a clean bill.\n" + : ""; const request: BatchRequest = { custom_id: "synthesis", body: { @@ -1794,7 +1859,8 @@ export async function runBroadsideCollect( content: "Synthesize these analysis reports into a single summary.\n\n" + findingsText + - "\n\nReturn the synthesis_report JSON schema.", + truncatedNote + + "\nReturn the synthesis_report JSON schema.", }, ], response_format: { type: "json_schema", json_schema: SCHEMAS.synthesis }, @@ -1855,6 +1921,7 @@ export async function runBroadsideCollect( status: run.status, total_cost: totalCost, result_count: resultCount, + truncated_count: truncatedCount, lenses: run.lenses, disclaimer: "Findings are unverified scouting signals from a batch model, not validated claims. " + @@ -1866,7 +1933,16 @@ export async function runBroadsideCollect( "utf8", ); - return { runId: run.id, status: run.status, totalCost, resultCount, lensOutcomes, synthesis: run.synthesis, topFindings }; + return { + runId: run.id, + status: run.status, + totalCost, + resultCount, + truncatedCount, + lensOutcomes, + synthesis: run.synthesis, + topFindings, + }; } export async function runBroadsideStatus(cwd: string): Promise<{ state: BroadsideStateFile }> { @@ -1878,12 +1954,8 @@ export async function runBroadsideStatus(cwd: string): Promise<{ state: Broadsid // ---------- rendering ---------- export function renderFindingsMarkdown(content: string): string { - let parsed: unknown; - try { - parsed = JSON.parse(content); - } catch { - return content; - } + const parsed = parseLensJson(content); + if (parsed === null) return content; return formatAsMarkdown(parsed); } @@ -2019,10 +2091,17 @@ export function collectResultText(result: BroadsideCollectResult): string { for (const lensId of BROADSIDE_LENS_IDS) { const outcome = result.lensOutcomes[lensId]; if (!outcome) continue; + const truncation = outcome.truncated ? `, ${outcome.truncated} truncated` : ""; lines.push( ` ${lensId}: ${outcome.status}` + (outcome.cost !== undefined ? `, $${outcome.cost.toFixed(6)}` : "") + - (outcome.resultCount !== undefined ? `, ${outcome.resultCount} result(s)` : ""), + (outcome.resultCount !== undefined ? `, ${outcome.resultCount} result(s)` : "") + + truncation, + ); + } + if (result.truncatedCount > 0) { + lines.push( + ` ⚠ ${result.truncatedCount} result(s) truncated at the output limit — their modules are unscouted, not clean.`, ); } if (result.synthesis.status === "completed") { diff --git a/mcp-server/server.ts b/mcp-server/server.ts index 84d0ab9..9540dbb 100644 --- a/mcp-server/server.ts +++ b/mcp-server/server.ts @@ -1087,6 +1087,7 @@ export async function handleBroadside(args: { status: collect.status, totalCost: collect.totalCost, resultCount: collect.resultCount, + truncatedCount: collect.truncatedCount, lensOutcomes: collect.lensOutcomes, synthesis: collect.synthesis, topFindings: collect.topFindings, diff --git a/tests/broadside.test.mjs b/tests/broadside.test.mjs index 779a798..489306f 100644 --- a/tests/broadside.test.mjs +++ b/tests/broadside.test.mjs @@ -5,7 +5,7 @@ import { test } from "node:test"; import assert from "node:assert/strict"; -import { mkdtemp, mkdir, rm, writeFile } from "node:fs/promises"; +import { mkdtemp, mkdir, readFile, rm, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import { dirname, join, resolve } from "node:path"; import { fileURLToPath, pathToFileURL } from "node:url"; @@ -701,7 +701,7 @@ test("runBroadsideCollect polls, saves results, and runs synthesis", async () => } }); -// ---------- markdown rendering ---------- +// ---------- markdown rendering & fence-tolerant parsing ---------- test("renderFindingsMarkdown turns parsed JSON into readable text", () => { const md = renderFindingsMarkdown(JSON.stringify({ title: "T", severity: "high", nested: { a: "b" }, list: [{ title: "x" }] })); @@ -714,3 +714,51 @@ test("renderFindingsMarkdown turns parsed JSON into readable text", () => { test("renderFindingsMarkdown passes invalid JSON through untouched", () => { assert.equal(renderFindingsMarkdown("not json"), "not json"); }); + +test("parseLensJson strips markdown code fences", () => { + const fenced = '```json\n{"title": "F", "severity": "low"}\n```'; + const parsed = core.parseLensJson(fenced); + assert.deepEqual(parsed, { title: "F", severity: "low" }); + const withoutLang = '```\n{"title": "F"}\n```'; + assert.deepEqual(core.parseLensJson(withoutLang), { title: "F" }); +}); + +test("parseLensJson returns null for truncated or non-JSON content", () => { + assert.equal(core.parseLensJson('```json\n{"title": "unterminated\n```'), null); + assert.equal(core.parseLensJson("The findings are numerous."), null); +}); + +test("saveLensResults marks truncated content and writes parsed JSON cleanly", async () => { + const dir = await mkdtemp(join(tmpdir(), "broadside-trunc-")); + try { + const batch = { + results: [ + { + custom_id: "defect-core-1", + response: { + status_code: 200, + body: { choices: [{ message: { content: '```json\n{"module": "core", "findings": []}\n```' } }] }, + }, + error: null, + }, + { + custom_id: "defect-core-2", + response: { + status_code: 200, + body: { choices: [{ message: { content: '{"module": "core-2", "findin' } }] }, + }, + error: null, + }, + ], + }; + const stored = await core.saveLensResults(dir, "defect", batch); + assert.equal(stored.length, 2); + assert.equal(stored[0].truncated, false); + assert.equal(stored[1].truncated, true); + + const written = JSON.parse(await readFile(join(dir, "defect-core-1.json"), "utf8")); + assert.equal(written.module, "core", "fenced JSON must be saved parsed, not verbatim"); + } finally { + await rm(dir, { recursive: true, force: true }); + } +}); From d1b02a14460eea708fbefb478c9067a7ecced4de Mon Sep 17 00:00:00 2001 From: James Sesler Date: Sun, 23 Aug 2026 05:05:33 -0400 Subject: [PATCH 7/7] =?UTF-8?q?feat:=20broadside=20triage=20pass=20?= =?UTF-8?q?=E2=80=94=20prioritized=20work=20order=20from=20scouting=20find?= =?UTF-8?q?ings=20(#103,=20#135)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Collect now runs a second cross-lens post-pass alongside synthesis. Triage scores every lead by impact × fix difficulty and emits a work order: P0–P3 priority, effort estimates, per-module grouping, deduplicated leads, and explicit omitted notes for dropped items — saved as triage.json/triage.md, surfaced in the collect summary, and framed in-prompt as a starting point for re-verification, never a commitment. Both post-passes submit as separate batches together and poll independently; the state file tracks each so a resumed collect can finish whichever is still pending (older runs without a triage entry are upgraded in place). Skip with include_triage: false on collect. 2 new tests (triage flow, include_triage opt-out); 35 broadside tests, 377 total, all passing. --- .codecarto/broadside/SKILL.md | 12 +- CHANGELOG.md | 1 + ROADMAP.md | 6 +- core/broadside.ts | 280 ++++++++++++++++++++++++++++------ mcp-server/server.ts | 10 ++ tests/broadside.test.mjs | 90 ++++++++++- 6 files changed, 341 insertions(+), 58 deletions(-) diff --git a/.codecarto/broadside/SKILL.md b/.codecarto/broadside/SKILL.md index 3bd41f7..e42d896 100644 --- a/.codecarto/broadside/SKILL.md +++ b/.codecarto/broadside/SKILL.md @@ -28,14 +28,18 @@ not replace any phase; it tells phases where to look. 1. Read `synthesis.md` first. It carries the executive summary, severity counts, the top cross-lens findings, and per-module risk levels. -2. Read the per-lens files behind anything that matters to your current phase: +2. Read `triage.md` for the work order: each lead scored by impact × + difficulty with a P0–P3 priority and an effort estimate. It is a starting + point for re-verification, not a commitment — every item still needs + confirmation against the source before work begins. +3. Read the per-lens files behind anything that matters to your current phase: - `architecture-*.json` → the architecture phase's seed of prior knowledge - `api-*.json` → endpoints and data types (contracts/protocols phases) - `security-*.json` → auth, trust boundaries (defect-scan-semantic pass 5) - `defect-*.json` → mechanical defect leads (defect-scan-mechanical) - `conventions-*.json` → naming/idiom candidates for CONVENTIONS.md - `porting-*.json` → platform coupling (porting phase) -3. `run-meta.json` records scope: which lenses ran, at what cost, with what +4. `run-meta.json` records scope: which lenses ran, at what cost, with what coverage caps. ## How to use the leads @@ -73,6 +77,10 @@ Submits pre-flight the chosen model: pricing comes from the live catalog model that does not advertise structured-output support is refused outright, because every lens depends on `json_schema` response_format. +Collect runs two cross-lens post-passes by default: **synthesis** (the +executive report) and **triage** (the prioritized work order). Pass +`include_synthesis: false` or `include_triage: false` on collect to skip one. + It works on any git repository — no initialized workspace required — and needs an OpenRouter API key via the `api_key` parameter, the `OPENROUTER_API_KEY` environment variable, or `api_key` in this directory's `config.yaml`. diff --git a/CHANGELOG.md b/CHANGELOG.md index 6c52123..e035549 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ All notable changes to this project are documented here. The format is based on - **Broad-Side: expense guardrails and live per-model pricing** (#103). `config.yaml` now accepts `model`, `max_cost`, and `pricing.input_per_m`/`output_per_m` overrides, and the MCP tool accepts `max_cost` and `force` parameters. Before submitting, Broad-Side estimates the run cost from collected file sizes (≈4 chars/token) against the configured model's per-token pricing — looked up live from OpenRouter's model catalog (cached 24h), so models like `openai/gpt-5.2-pro:batch` at ~$84/M output are priced correctly, not at the default model's rates. A submit whose estimate exceeds `max_cost` refuses with a per-lens breakdown and creates no run entry unless `force: true`. The submit response now reports the pricing used and its source (built-in/config/live/cache). - **Broad-Side: model catalog action and capability pre-flight** (#103). New `models` action lists every `:batch` model on OpenRouter — pricing per million tokens, context window, completion ceiling, structured-output support, and optional Artificial Analysis coding indices (via `GET /api/v1/benchmarks`, attribution preserved) — cheapest first with the configured model marked. Submits now pre-flight the chosen model against that catalog: lens `max_tokens` clamps to the provider's completion ceiling, deprecated models are flagged, and models that do not advertise structured-output support are refused outright, since every lens depends on `json_schema` response_format. The catalog cache is shared between the `models` action and submit-time pricing resolution. - **Broad-Side: truncation detection with fence-tolerant parsing** (#103, #133). Lens output is now parsed tolerantly — markdown code fences are stripped before JSON parsing, mirroring the tolerance in OpenRouter's headless-agent scaffold. Parseable output is saved as clean JSON; output that still does not parse (the signature of a `max_tokens` cutoff) is saved verbatim but marked `truncated`. The collect summary and `run-meta.json` report truncation counts, and the synthesis prompt is told which modules are unrepresented rather than clean. Also documented the retry-safety invariant (batch requests are pure, resubmission always safe) and the distinction between Broad-Side's pre-flight `max_cost` estimate and OpenRouter's runtime cost accounting. +- **Broad-Side: triage pass** (#103, #135). Collect now runs a second cross-lens post-pass alongside synthesis (skip with `include_triage: false`): every finding is scored by impact × fix difficulty and turned into a prioritized work order — P0–P3 priority, effort estimate, per-module grouping, deduplicated leads, and explicit `omitted` notes for dropped items — saved as `triage.json`/`triage.md` and surfaced in the collect summary. The triage prompt frames the queue as a starting point for re-verification, never a commitment. Both post-passes submit as separate batches together and poll independently, and the state file tracks each so a resumed collect can finish whichever is still pending. ## [0.16.0] — 2026-08-17 diff --git a/ROADMAP.md b/ROADMAP.md index a2377b1..4848f7b 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -26,13 +26,15 @@ file only moves when a tier completes. structured-output support, and optional Artificial Analysis coding benchmarks; submit pre-flight refuses models without structured outputs and clamps lens `max_tokens` to the provider's completion ceiling. -- Tests: 31 unit tests (fake-fetcher based), opt-in live smoke script. +- Triage post-pass on collect: findings scored by impact × difficulty into + a P0–P3 work order with effort estimates, saved as triage.json/md. +- Tests: 35 unit tests (fake-fetcher based), opt-in live smoke script. ## Tier 1 — make Broad-Side better at what it does | Item | Issue | Notes | |---|---|---| -| **Triage lens** — prioritized fix queue (impact × difficulty, grouped by module) | [#135](https://github.com/HuginnIndustries/CodeCartographer/issues/135) | Highest-value next lens: turns leads into a work order | +| **Triage lens** — prioritized fix queue (impact × difficulty, grouped by module) | [#135](https://github.com/HuginnIndustries/CodeCartographer/issues/135) | **Shipped**: triage pass runs on collect alongside synthesis (`include_triage` to skip) | | **Truncation repair** — detect max_tokens-cutoff JSON, resubmit slices, report truncation in summaries | [#133](https://github.com/HuginnIndustries/CodeCartographer/issues/133) | Partially shipped: fence-tolerant parsing + `truncated` flagging in collect, meta, and synthesis. Remaining: automatic resubmit of truncated slices | | **Concurrent polling** — poll all in-flight batches round-robin against one deadline | [#136](https://github.com/HuginnIndustries/CodeCartographer/issues/136) | Submissions already parallel; polling is sequential today | | **Per-language prompts** — Go/Python/Rust/TS lens prompts; globs already adapt | [#137](https://github.com/HuginnIndustries/CodeCartographer/issues/137) | Schemas stay shared so synthesis is unaffected | diff --git a/core/broadside.ts b/core/broadside.ts index b17d696..92abb0d 100644 --- a/core/broadside.ts +++ b/core/broadside.ts @@ -168,6 +168,24 @@ export type BroadsideSynthesisEntry = { cost?: number; }; +/** One triage item — a scouting lead turned into a work-order entry. */ +export type TriageItem = { + title: string; + severity: string; + module: string; + impact: "high" | "medium" | "low"; + difficulty: "high" | "medium" | "low"; + priority: string; + effort_estimate: string; + rationale: string; +}; + +export type BroadsideTriageEntry = { + batchId?: string; + status: "pending" | "submitted" | "completed" | "failed"; + cost?: number; +}; + export type BroadsideRun = { id: string; createdAt: string; @@ -177,6 +195,7 @@ export type BroadsideRun = { outputDir: string; // relative to .codecarto/broadside/ batches: Partial>; synthesis: BroadsideSynthesisEntry; + triage: BroadsideTriageEntry; totalCost?: number; pricing?: ModelPricing; maxCost?: number; @@ -226,7 +245,9 @@ export type BroadsideCollectResult = { Record >; synthesis: BroadsideSynthesisEntry; + triage: BroadsideTriageEntry; topFindings: { title: string; severity: string; sourceLens: string; summary: string }[]; + topTriageItems: TriageItem[]; }; // ---------- JSON schemas (one per lens, plus synthesis) ---------- @@ -532,6 +553,41 @@ const SCHEMAS: Record = { additionalProperties: false, }, }, + triage: { + name: "triage_report", + strict: true, + schema: { + type: "object", + properties: { + summary: { type: "string" }, + items: { + type: "array", + items: { + type: "object", + properties: { + title: { type: "string" }, + severity: { type: "string" }, + module: { type: "string" }, + impact: { type: "string", enum: ["high", "medium", "low"] }, + difficulty: { type: "string", enum: ["high", "medium", "low"] }, + priority: { type: "string" }, + effort_estimate: { type: "string" }, + rationale: { type: "string" }, + }, + required: ["title", "severity", "module", "impact", "difficulty", "priority", "rationale"], + additionalProperties: false, + }, + }, + omitted: { + type: "array", + items: { type: "string" }, + description: "Leads deliberately dropped from the queue and why (duplicates, too vague, out of scope)", + }, + }, + required: ["summary", "items"], + additionalProperties: false, + }, + }, }; // ---------- lens definitions ---------- @@ -1600,6 +1656,7 @@ export async function runBroadsideSubmit( outputDir: runId, batches: {}, synthesis: { status: "pending" }, + triage: { status: "pending" }, pricing, maxCost: limit > 0 ? limit : undefined, }; @@ -1745,12 +1802,105 @@ export async function saveLensResults( return out; } +// ---------- post-lens passes: synthesis + triage ---------- + +function buildSynthesisRequest(findingsText: string, truncatedNote: string, model: string): BatchRequest { + return { + custom_id: "synthesis", + body: { + model, + messages: [ + { + role: "system", + content: + "You are a technical editor synthesizing multiple analysis reports about a single " + + "codebase into one coherent summary. The reports come from different lenses — " + + "architecture, API surface, security review, defect scanning, convention extraction, " + + "and porting assessment. Cross-reference findings across lenses: if a security issue " + + "also appears as a defect, merge them. Produce a JSON object following the " + + "synthesis_report schema. Prioritize the most actionable findings. " + + "Be honest about gaps — if a lens found nothing, say 'no issues found' rather than " + + "inventing problems. These are scouting signals from a batch model, not verified " + + "claims; note that in the summary.", + }, + { + role: "user", + content: + "Synthesize these analysis reports into a single summary.\n\n" + + findingsText + + truncatedNote + + "\nReturn the synthesis_report JSON schema.", + }, + ], + response_format: { type: "json_schema", json_schema: SCHEMAS.synthesis }, + max_tokens: 12_000, + }, + }; +} + +function buildTriageRequest(findingsText: string, truncatedNote: string, model: string): BatchRequest { + return { + custom_id: "triage", + body: { + model, + messages: [ + { + role: "system", + content: + "You are a senior engineering lead turning unverified scouting findings into a " + + "prioritized work order. Given the findings below, produce a JSON object following " + + "the triage_report schema. Score every lead by impact and fix difficulty, assign a " + + "priority (P0 urgent/safety-critical to P3 nice-to-have), give a rough effort " + + "estimate, group the queue by module where sensible, and justify each call in the " + + "rationale. Merge duplicate leads instead of listing them twice. Drop leads that are " + + "too vague to act on and record each drop in omitted with the reason. These findings " + + "are UNVERIFIED scouting signals from a cheap batch model: the queue is a starting " + + "point for re-verification, not a commitment — say so in the summary, and never " + + "inflate a severity you cannot see evidence for.", + }, + { + role: "user", + content: + "Triage these scouting findings into a prioritized work order.\n\n" + + findingsText + + truncatedNote + + "\nReturn the triage_report JSON schema.", + }, + ], + response_format: { type: "json_schema", json_schema: SCHEMAS.triage }, + max_tokens: 10_000, + }, + }; +} + +function parseTriageItems(content: string): TriageItem[] { + try { + const parsed = JSON.parse(content) as Record; + const items = Array.isArray(parsed.items) ? (parsed.items as Array>) : []; + return items + .filter((item) => typeof item.title === "string") + .map((item) => ({ + title: String(item.title), + severity: String(item.severity ?? "unknown"), + module: String(item.module ?? "unknown"), + impact: (["high", "medium", "low"].includes(String(item.impact)) ? String(item.impact) : "medium") as TriageItem["impact"], + difficulty: (["high", "medium", "low"].includes(String(item.difficulty)) ? String(item.difficulty) : "medium") as TriageItem["difficulty"], + priority: String(item.priority ?? "?"), + effort_estimate: String(item.effort_estimate ?? ""), + rationale: String(item.rationale ?? ""), + })); + } catch { + return []; + } +} + export async function runBroadsideCollect( cwd: string, apiKey: string, opts: { waitMs?: number; includeSynthesis?: boolean; + includeTriage?: boolean; onStatus?: (lensId: string, status: string, counts: Record) => void; fetcher?: FetchLike; } = {}, @@ -1819,14 +1969,19 @@ export async function runBroadsideCollect( await saveBroadsideState(broadsideDir, state); } - // Synthesis: one cross-lens report, only after every lens batch is terminal. + // Synthesis + triage: cross-lens post-passes, only after every lens batch + // is terminal. Triage turns the leads into a prioritized work order. + run.triage ??= { status: "pending" }; let topFindings: BroadsideCollectResult["topFindings"] = []; - if (opts.includeSynthesis !== false && allLensResults.length > 0) { + let topTriageItems: BroadsideCollectResult["topTriageItems"] = []; + const wantSynthesis = opts.includeSynthesis !== false; + const wantTriage = opts.includeTriage !== false; + if ((wantSynthesis || wantTriage) && allLensResults.length > 0) { const allTerminal = run.lenses.every((lensId) => { const entry = run.batches[lensId]; return entry && ["completed", "failed", "expired", "cancelled", "auth-failed", "skipped", "rejected"].includes(entry.status); }); - if (allTerminal && run.synthesis.status === "pending") { + if (allTerminal && (run.synthesis.status === "pending" || run.triage.status === "pending")) { const findingsText = allLensResults .map((r) => `## ${r.lensId} — ${r.customId}\n\n${r.content}\n`) .join("\n"); @@ -1836,65 +1991,77 @@ export async function runBroadsideCollect( "not included above. Any gap they would have covered is unrepresented — do not treat " + "silence on a module as a clean bill.\n" : ""; - const request: BatchRequest = { - custom_id: "synthesis", - body: { - model: run.model, - messages: [ - { - role: "system", - content: - "You are a technical editor synthesizing multiple analysis reports about a single " + - "codebase into one coherent summary. The reports come from different lenses — " + - "architecture, API surface, security review, defect scanning, convention extraction, " + - "and porting assessment. Cross-reference findings across lenses: if a security issue " + - "also appears as a defect, merge them. Produce a JSON object following the " + - "synthesis_report schema. Prioritize the most actionable findings. " + - "Be honest about gaps — if a lens found nothing, say 'no issues found' rather than " + - "inventing problems. These are scouting signals from a batch model, not verified " + - "claims; note that in the summary.", - }, - { - role: "user", - content: - "Synthesize these analysis reports into a single summary.\n\n" + - findingsText + - truncatedNote + - "\nReturn the synthesis_report JSON schema.", - }, - ], - response_format: { type: "json_schema", json_schema: SCHEMAS.synthesis }, - max_tokens: 12_000, - }, - }; - run.synthesis.status = "submitted"; + + // Both post-passes consume the same findings; they run as two + // batches (different response_format schemas cannot share one) + // submitted together and polled in turn. + const passes: Array<{ + kind: "synthesis" | "triage"; + request: BatchRequest; + entry: BroadsideSynthesisEntry; + }> = [ + ...(wantSynthesis && run.synthesis.status === "pending" + ? [{ + kind: "synthesis" as const, + request: buildSynthesisRequest(findingsText, truncatedNote, run.model), + entry: run.synthesis, + }] + : []), + ...(wantTriage && run.triage.status === "pending" + ? [{ + kind: "triage" as const, + request: buildTriageRequest(findingsText, truncatedNote, run.model), + entry: run.triage, + }] + : []), + ]; + + const submitted = new Map(); + await Promise.allSettled( + passes.map(async (pass) => { + pass.entry.status = "submitted"; + try { + const { batchId, error } = await submitBatch([pass.request], apiKey, opts.fetcher, run.model); + if (error) { + pass.entry.status = "failed"; + return; + } + pass.entry.batchId = batchId; + submitted.set(batchId, { batchId, pass }); + } catch { + pass.entry.status = "failed"; + } + }), + ); await saveBroadsideState(broadsideDir, state); - const { batchId, error } = await submitBatch([request], apiKey, opts.fetcher, run.model); - if (error) { - run.synthesis.status = "failed"; - } else { - run.synthesis.batchId = batchId; + + for (const { batchId, pass } of submitted.values()) { const batch = await pollBatchUntilTerminal(batchId, apiKey, { deadlineMs: BROADSIDE_DEFAULT_POLL_BUDGET_MS, - onStatus: (status, counts) => opts.onStatus?.("synthesis", status, counts), + onStatus: (status, counts) => opts.onStatus?.(pass.kind, status, counts), fetcher: opts.fetcher, }); if (batch.status === "completed") { const usage = (batch.usage ?? {}) as Record; const cost = typeof usage.cost === "number" ? usage.cost : undefined; - run.synthesis.status = "completed"; - run.synthesis.cost = cost; + pass.entry.status = "completed"; + pass.entry.cost = cost; totalCost += cost ?? 0; const results = Array.isArray(batch.results) ? (batch.results as Array>) : []; const content = results.length > 0 ? extractContent(results[0]) : null; if (content !== null) { - await writeFile(join(runDir, "synthesis.json"), `${content}\n`, "utf8"); - await writeFile(join(runDir, "synthesis.md"), renderFindingsMarkdown(content), "utf8"); - topFindings = parseSynthesisTopFindings(content); + await writeFile(join(runDir, `${pass.kind}.json`), `${content}\n`, "utf8"); + await writeFile(join(runDir, `${pass.kind}.md`), renderFindingsMarkdown(content), "utf8"); + if (pass.kind === "synthesis") { + topFindings = parseSynthesisTopFindings(content); + } else { + topTriageItems = parseTriageItems(content); + } } } else if (batch.error) { - run.synthesis.status = "failed"; + pass.entry.status = "failed"; } + await saveBroadsideState(broadsideDir, state); } } } @@ -1922,6 +2089,8 @@ export async function runBroadsideCollect( total_cost: totalCost, result_count: resultCount, truncated_count: truncatedCount, + synthesis: run.synthesis, + triage: run.triage, lenses: run.lenses, disclaimer: "Findings are unverified scouting signals from a batch model, not validated claims. " + @@ -1941,7 +2110,9 @@ export async function runBroadsideCollect( truncatedCount, lensOutcomes, synthesis: run.synthesis, + triage: run.triage, topFindings, + topTriageItems, }; } @@ -2113,6 +2284,20 @@ export function collectResultText(result: BroadsideCollectResult): string { } } } + if (result.triage.status === "completed") { + lines.push(` triage: completed, $${(result.triage.cost ?? 0).toFixed(6)}`); + if (result.topTriageItems.length > 0) { + lines.push("", "Triage — prioritized work order (re-verify before acting):"); + for (const item of result.topTriageItems.slice(0, 10)) { + lines.push( + ` ${item.priority} [${item.severity}/${item.module}] ${item.title}` + + (item.effort_estimate ? ` (${item.effort_estimate})` : ""), + ); + } + } + } else if (result.triage.status === "failed") { + lines.push(" triage: failed"); + } lines.push("", "Disclaimer: Broad-Side findings are unverified scouting signals from a batch model, not validated claims."); return lines.join("\n"); } @@ -2130,6 +2315,7 @@ export function statusText(state: BroadsideStateFile): string { lines.push(` ${lensId}: ${entry.status}${entry.batchId ? ` (${entry.batchId})` : ""}${entry.cost !== undefined ? `, $${entry.cost.toFixed(6)}` : ""}`); } lines.push(` synthesis: ${run.synthesis.status}`); + lines.push(` triage: ${run.triage?.status ?? "pending"}`); if (run.totalCost !== undefined) lines.push(` total cost: $${run.totalCost.toFixed(6)}`); } return lines.join("\n"); diff --git a/mcp-server/server.ts b/mcp-server/server.ts index 9540dbb..e990da5 100644 --- a/mcp-server/server.ts +++ b/mcp-server/server.ts @@ -998,6 +998,7 @@ export async function handleBroadside(args: { api_key?: string; wait_seconds?: number; include_synthesis?: boolean; + include_triage?: boolean; max_cost?: number; force?: boolean; include_benchmarks?: boolean; @@ -1060,6 +1061,7 @@ export async function handleBroadside(args: { const collect = await runBroadsideCollect(cwd, apiKey, { waitMs, includeSynthesis: args.include_synthesis !== false, + includeTriage: args.include_triage !== false, onStatus: (lensId, status, counts) => lines.push(` ${lensId}: ${status} (${counts.completed ?? 0}/${counts.total ?? "?"})`), }); @@ -1079,6 +1081,7 @@ export async function handleBroadside(args: { const collect = await runBroadsideCollect(cwd, apiKey, { waitMs, includeSynthesis: args.include_synthesis !== false, + includeTriage: args.include_triage !== false, }).catch((error) => { throw new McpError(ErrorCode.InvalidRequest, error instanceof Error ? error.message : String(error)); }); @@ -1090,7 +1093,9 @@ export async function handleBroadside(args: { truncatedCount: collect.truncatedCount, lensOutcomes: collect.lensOutcomes, synthesis: collect.synthesis, + triage: collect.triage, topFindings: collect.topFindings, + topTriageItems: collect.topTriageItems, }); } @@ -1412,6 +1417,11 @@ const TOOLS = [ type: "boolean", description: "Run the cross-lens synthesis pass once all lens batches complete (default true).", }, + include_triage: { + type: "boolean", + description: + "Run the triage pass once all lens batches complete: turns the findings into a prioritized work order (impact × difficulty, P0-P3, effort estimates). Default true.", + }, max_cost: { type: "number", description: diff --git a/tests/broadside.test.mjs b/tests/broadside.test.mjs index 489306f..62e12a0 100644 --- a/tests/broadside.test.mjs +++ b/tests/broadside.test.mjs @@ -638,11 +638,9 @@ test("runBroadsideSubmit records a run and fires one batch per lens", async () = // ---------- collect with fake fetcher ---------- -test("runBroadsideCollect polls, saves results, and runs synthesis", async () => { +test("runBroadsideCollect polls, saves results, and runs synthesis + triage", async () => { const dir = await makeFixture(); try { - let submits = 0; - let gets = 0; const lensPayload = { results: [ { @@ -675,16 +673,36 @@ test("runBroadsideCollect polls, saves results, and runs synthesis", async () => usage: { cost: 0.002 }, request_counts: { total: 1, completed: 1, failed: 0 }, }; + const triagePayload = { + results: [ + { + custom_id: "triage", + response: { + status_code: 200, + body: { + choices: [{ message: { role: "assistant", content: JSON.stringify({ summary: "work order", items: [{ title: "fix the thing", severity: "high", module: "server", impact: "high", difficulty: "low", priority: "P0", effort_estimate: "2h", rationale: "obvious" }] }) } }], + }, + }, + error: null, + }, + ], + usage: { cost: 0.003 }, + request_counts: { total: 1, completed: 1, failed: 0 }, + }; + let postCount = 0; const fetcher = async (url, init) => { if (init.method === "POST") { - submits += 1; - return fakeResponse(202, { id: submits === 1 ? "batch-lens" : "batch-synth", status: "validating" }); + postCount += 1; + const payload = JSON.parse(init.body); + const id = payload.requests[0].custom_id === "synthesis" ? "batch-synth" : payload.requests[0].custom_id === "triage" ? "batch-triage" : "batch-lens"; + return fakeResponse(202, { id, status: "validating" }); } - gets += 1; - // First GET per batch returns completed immediately. if (String(url).includes("batch-lens")) { return fakeResponse(200, { id: "batch-lens", status: "completed", ...lensPayload }); } + if (String(url).includes("batch-triage")) { + return fakeResponse(200, { id: "batch-triage", status: "completed", ...triagePayload }); + } return fakeResponse(200, { id: "batch-synth", status: "completed", ...synthPayload }); }; @@ -696,6 +714,64 @@ test("runBroadsideCollect polls, saves results, and runs synthesis", async () => assert.equal(collect.synthesis.status, "completed"); assert.equal(collect.topFindings.length, 1); assert.equal(collect.topFindings[0].title, "lead"); + assert.equal(collect.triage.status, "completed"); + assert.equal(collect.topTriageItems.length, 1); + assert.equal(collect.topTriageItems[0].title, "fix the thing"); + assert.equal(collect.topTriageItems[0].priority, "P0"); + assert.equal(postCount, 3, "lens + synthesis + triage batches"); + } finally { + await rm(dir, { recursive: true, force: true }); + } +}); + +test("runBroadsideCollect skips triage when include_triage is false", async () => { + const dir = await makeFixture(); + try { + const lensPayload = { + results: [ + { + custom_id: "architecture-root", + response: { + status_code: 200, + body: { choices: [{ message: { content: JSON.stringify({ tech_stack: { language: "Go", build_system: "x" }, module_architecture: [], data_flow: "x", entry_points: [] }) } }] }, + }, + error: null, + }, + ], + usage: { cost: 0.001 }, + request_counts: { total: 1, completed: 1, failed: 0 }, + }; + const synthPayload = { + results: [ + { + custom_id: "synthesis", + response: { + status_code: 200, + body: { choices: [{ message: { content: JSON.stringify({ executive_summary: "ok", severity_summary: { critical: 0, high: 0, medium: 0, low: 0 }, top_findings: [] }) } }] }, + }, + error: null, + }, + ], + usage: { cost: 0.002 }, + request_counts: { total: 1, completed: 1, failed: 0 }, + }; + const seenPosts = []; + const fetcher = async (url, init) => { + if (init.method === "POST") { + const payload = JSON.parse(init.body); + seenPosts.push(payload.requests[0].custom_id); + return fakeResponse(202, { id: `batch-${payload.requests[0].custom_id}`, status: "validating" }); + } + if (String(url).includes("batch-synthesis")) { + return fakeResponse(200, { id: "batch-synthesis", status: "completed", ...synthPayload }); + } + return fakeResponse(200, { id: "batch-architecture-root", status: "completed", ...lensPayload }); + }; + + await runBroadsideSubmit(dir, "sk-fake", { lenses: ["architecture"], fetcher }); + const collect = await runBroadsideCollect(dir, "sk-fake", { fetcher, includeTriage: false }); + assert.deepEqual(seenPosts, ["architecture-root", "synthesis"], "no triage batch may be submitted"); + assert.equal(collect.triage.status, "pending"); } finally { await rm(dir, { recursive: true, force: true }); }