diff --git a/.codecarto/.gitignore b/.codecarto/.gitignore index b66c5f7..39f6ab6 100644 --- a/.codecarto/.gitignore +++ b/.codecarto/.gitignore @@ -17,6 +17,13 @@ findings/config-model/config-model.md scratch/* !scratch/.gitkeep +# Broad-Side machine-local state and generated results (batch ids, run +# directories, costs). The SKILL.md guidance and the config template are +# tracked; the API key inside config.yaml is your own risk to commit. +broadside/* +!broadside/SKILL.md +!broadside/config.yaml + # Orchestrator session pointer (machine-local, written by /codecarto-init # when run from the Pi extension; the MCP path doesn't write it). Contains # absolute paths into the user's Pi session storage, so it must never be diff --git a/.codecarto/GUIDE.md b/.codecarto/GUIDE.md index 92d8301..9194c78 100644 --- a/.codecarto/GUIDE.md +++ b/.codecarto/GUIDE.md @@ -57,6 +57,7 @@ Read these files in order before doing any analysis: 4. `scratch/checkpoints/.md`, if present, to resume durable in-phase progress after compaction or interruption. 5. The current phase's `SKILL.md` for detailed instructions on what to analyze and produce. 6. The output template from `templates/` for the current phase (if starting a new output). +7. `broadside/synthesis.md`, if a Broad-Side batch reconnaissance run has completed — it carries unverified scouting leads (see `broadside/SKILL.md`) that tell you where the interactive phases should spend attention. All paths in this guide are relative to `.codecarto/` unless stated otherwise. @@ -71,7 +72,7 @@ Some files in this workspace are **read-only instructions** and must not be modi | Category | Files | Access | |---|---|---| | Orchestration (read-only) | `GUIDE.md`, `CONTRIBUTING.md`, `LICENSE` | Read only. Never modify. | -| Skills (read-only) | `findings/*/SKILL.md`, `findings/defect-scan/passes/*.md`, `skills/*/SKILL.md` | Read only. Never modify. | +| Skills (read-only) | `findings/*/SKILL.md`, `findings/defect-scan/passes/*.md`, `skills/*/SKILL.md`, `broadside/SKILL.md` | Read only. Never modify. | | Templates (read-only) | `templates/*.md` | Read only. Never modify. | | Pipeline definitions (read-only) | `workflow/pipeline*.yaml`, `workflow/VALIDATE.md` | Read only. Never modify. | | Source code (read-only) | `../` (everything outside `.codecarto/`) | Read only. Analyze but never modify. | @@ -313,6 +314,10 @@ your-repo/ VALIDATE.md # Validation protocol. Run after every phase. closeouts/ # Per-session closeout files (replaces monolithic THREAD_LOG body). -.md + broadside/ # Batch reconnaissance state and results (see broadside/SKILL.md). + SKILL.md # How to read Broad-Side scouting leads (unverified, not evidence). + config.yaml # Broad-Side model/key/lens configuration. + / # Per-run JSON + markdown findings, run-meta.json, synthesis report. CONVENTIONS.md # (Optional, project-grown) Cross-cutting invariants. Orchestrator-maintained. DECISIONS.md # (Optional, project-grown) Numbered decisions log. Orchestrator-maintained. BACKLOG.md # (Optional) Deferred items with rationale. diff --git a/.codecarto/broadside/SKILL.md b/.codecarto/broadside/SKILL.md new file mode 100644 index 0000000..e42d896 --- /dev/null +++ b/.codecarto/broadside/SKILL.md @@ -0,0 +1,114 @@ +--- +name: broadside +description: Interpret a Broad-Side batch reconnaissance run. Use after codecarto_broadside collect has produced .codecarto/broadside// results, to triage scouting signals before or during an interactive CodeCartographer pipeline run. +--- + +# Broad-Side + +Broad-Side is CodeCartographer's batch reconnaissance pass. It fires six +analysis lenses — architecture, API surface, security, mechanical defect scan, +convention extraction, and porting — at the repository as single-turn prompts +over the OpenRouter Batch API (~50% of sync pricing, asynchronous, unattended), +then synthesizes one cross-lens report. Results live under +`.codecarto/broadside//` alongside this file. + +## What Broad-Side findings are — and are not + +Broad-Side findings are **unverified scouting signals**, not validated claims. +Every lens is one shot: no cross-file traversal, no runtime verification, no +builds, no tests, no follow-up questions. The batch model is cheap, not strong. +Treat every finding as a lead with a file:line pointer that the interactive +pipeline — or you — must confirm before it is a fact. + +This is the division of labor: Broad-Side is cheap enough to run on any repo to +decide where the expensive interactive run should spend its attention. It does +not replace any phase; it tells phases where to look. + +## Reading a Broad-Side run + +1. Read `synthesis.md` first. It carries the executive summary, severity counts, + the top cross-lens findings, and per-module risk levels. +2. Read `triage.md` for the work order: each lead scored by impact × + difficulty with a P0–P3 priority and an effort estimate. It is a starting + point for re-verification, not a commitment — every item still needs + confirmation against the source before work begins. +3. Read the per-lens files behind anything that matters to your current phase: + - `architecture-*.json` → the architecture phase's seed of prior knowledge + - `api-*.json` → endpoints and data types (contracts/protocols phases) + - `security-*.json` → auth, trust boundaries (defect-scan-semantic pass 5) + - `defect-*.json` → mechanical defect leads (defect-scan-mechanical) + - `conventions-*.json` → naming/idiom candidates for CONVENTIONS.md + - `porting-*.json` → platform coupling (porting phase) +4. `run-meta.json` records scope: which lenses ran, at what cost, with what + coverage caps. + +## How to use the leads + +- **A finding that matches your phase's scope is a starting point, not an answer.** + Re-derive it from the source yourself; cite the source, not the Broad-Side + report. Broad-Side output is not evidence. +- **Route, don't believe.** A Broad-Side "high" that your phase can neither + confirm nor dismiss becomes an open question with `needs-runtime-test` or + `needs-maintainer-decision` — never a finding. +- **Promotable conventions are candidates only.** CONVENTIONS.md promotions + still require the orchestrator's review against the code, per the usual + promotion rules. +- **Coverage caps are real.** Directory-sliced lenses cap each slice's input; + `run-meta.json` and the synthesis `coverage` field say what was scanned. + Everything outside that is unscouted, not clean. + +## Running Broad-Side + +Broad-Side is an executable-surface feature (MCP today): + +``` +codecarto_broadside {cwd, action: "submit", lenses: [...]} # fire the batches +codecarto_broadside {cwd, action: "collect"} # poll, save, synthesize +codecarto_broadside {cwd, action: "status"} # show recorded runs +codecarto_broadside {cwd, action: "models"} # compare batch models +``` + +The `models` action lists every `:batch` variant on OpenRouter — pricing per +million tokens, context window, output ceiling, structured-output support, and +(optionally) Artificial Analysis coding indices — cheapest first, with the +configured model marked. Use it before switching models in `config.yaml`. +Submits pre-flight the chosen model: pricing comes from the live catalog +(cached 24h), requests clamp to the provider's completion ceiling, and a +model that does not advertise structured-output support is refused outright, +because every lens depends on `json_schema` response_format. + +Collect runs two cross-lens post-passes by default: **synthesis** (the +executive report) and **triage** (the prioritized work order). Pass +`include_synthesis: false` or `include_triage: false` on collect to skip one. + +It works on any git repository — no initialized workspace required — and needs +an OpenRouter API key via the `api_key` parameter, the `OPENROUTER_API_KEY` +environment variable, or `api_key` in this directory's `config.yaml`. + +Submits are priced before they fire: Broad-Side estimates the run from the +collected file sizes against the model's live per-token pricing and refuses +when the estimate exceeds `max_cost` (`config.yaml` or the tool parameter) +unless `force` is passed. See `config.yaml` for the model, limit, and manual +pricing-override keys. + +The `max_cost` guardrail is an **estimate-based pre-flight limit**, distinct +from OpenRouter's runtime cost tracking: it predicts from file sizes before +spend, it does not stop a batch mid-flight. Actual spend appears in +`run-meta.json` after collect. + +## Resilience notes + +- **Truncation is spoken.** A lens output whose JSON does not parse — even + after code-fence stripping — is saved verbatim but marked `truncated`: + the collect summary counts it, `run-meta.json` records it, and the + synthesis prompt is told its module is unrepresented, not clean. +- **Resubmission is always safe.** Batch requests are pure (no tools, no + filesystem, no side effects), so a failed or truncated slice can be + resubmitted freely. This is the same retry-safety rule OpenRouter's + headless-agent scaffold enforces for tool-using agents ("retry only + before tool calls"); Broad-Side satisfies it by construction. If + Broad-Side ever gains server tools, this invariant becomes load-bearing. +- **Field shapes** for the model catalog and benchmarks endpoints follow + the official OpenRouter skills (`OpenRouterTeam/skills`: + `openrouter-models`, `openrouter-benchmarks`) — consult them when + extending catalog parsing. diff --git a/.codecarto/broadside/config.yaml b/.codecarto/broadside/config.yaml new file mode 100644 index 0000000..93604e0 --- /dev/null +++ b/.codecarto/broadside/config.yaml @@ -0,0 +1,54 @@ +# Broad-Side batch reconnaissance configuration. Optional. +# Missing keys fall back to the defaults listed below. + +# OpenRouter model to use for batch requests. The default is Google Gemini +# 3.7 Flash (batch) — the cheapest batch model with tool-calling support and +# a 1M-token context window. Change this to another OpenRouter batch model +# if you need a different cost/capability trade-off. To compare what's +# available, run codecarto_broadside with action "models" — it lists every +# :batch variant with pricing, context, output caps, structured-output +# support, and optional coding benchmarks. Beware the expensive end of that +# list — some batch models exceed $80 per million output tokens — and note +# that every lens requires structured-output (json_schema) support, which +# submit refuses to proceed without. +# +# model: google/gemini-3.7-flash:batch + +# OpenRouter API key. Prefer the OPENROUTER_API_KEY environment variable — +# keys in this file are committed if you track .codecarto/ after init. +# The codecarto_broadside tool also accepts api_key as a parameter. +# +# api_key: "" + +# Default lens set for codecarto_broadside submit when no lenses are +# specified. All six lenses are on by default. Remove a lens id to skip it +# globally, or pass an explicit lenses array on the submit call to override. +# +# default_lenses: +# - architecture +# - api +# - security +# - defect +# - conventions +# - porting + +# Approximate run expense limit in USD (0 = no limit). Before submitting, +# Broad-Side estimates the run cost from the collected file sizes and the +# model's per-token pricing — fetched live from OpenRouter's model catalog +# and cached for 24h. If the estimate exceeds max_cost, submit refuses and +# prints the per-lens breakdown; pass force: true to override, or set a +# value here so every run is guarded by default. +# +# This is a pre-flight estimate guardrail, not a runtime stop: OpenRouter +# bills actual usage, which may differ from the estimate either direction. +# Actual cost lands in each run's run-meta.json after collect. +# +# max_cost: 1.00 + +# Manual pricing overrides in USD per MILLION tokens. Normally Broad-Side +# looks the model's pricing up automatically; set both fields only when the +# lookup fails (offline, private model) or you want to assert a ceiling. +# +# pricing: +# input_per_m: 0.1875 +# output_per_m: 0.9375 \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md index 4979ac1..e035549 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,14 @@ All notable changes to this project are documented here. The format is based on ## [Unreleased] +### Added + +- **Broad-Side: batch reconnaissance over the OpenRouter Batch API** (#103). New `codecarto_broadside` MCP tool (actions: `submit`, `collect`, `status`) fires six single-turn analysis lenses — architecture, API surface, security, mechanical defect scan, convention extraction, porting — at any git repository as asynchronous batch jobs on a cheap batch model (~50% of sync pricing, unattended, 24h window), slices large modules by top-level directory, saves JSON plus rendered markdown to `.codecarto/broadside//`, and optionally synthesizes a cross-lens executive report. Works without an initialized workspace; needs an OpenRouter key via the `api_key` parameter, `OPENROUTER_API_KEY`, or `.codecarto/broadside/config.yaml`. Broad-Side findings are explicitly unverified scouting signals — file:line leads for the interactive pipeline to confirm, never evidence themselves. `codecarto_init` tolerates a `.codecarto/` that holds only `broadside/` (no force/backup needed), and scaffold refresh never touches broadside state, config, or results. +- **Broad-Side: expense guardrails and live per-model pricing** (#103). `config.yaml` now accepts `model`, `max_cost`, and `pricing.input_per_m`/`output_per_m` overrides, and the MCP tool accepts `max_cost` and `force` parameters. Before submitting, Broad-Side estimates the run cost from collected file sizes (≈4 chars/token) against the configured model's per-token pricing — looked up live from OpenRouter's model catalog (cached 24h), so models like `openai/gpt-5.2-pro:batch` at ~$84/M output are priced correctly, not at the default model's rates. A submit whose estimate exceeds `max_cost` refuses with a per-lens breakdown and creates no run entry unless `force: true`. The submit response now reports the pricing used and its source (built-in/config/live/cache). +- **Broad-Side: model catalog action and capability pre-flight** (#103). New `models` action lists every `:batch` model on OpenRouter — pricing per million tokens, context window, completion ceiling, structured-output support, and optional Artificial Analysis coding indices (via `GET /api/v1/benchmarks`, attribution preserved) — cheapest first with the configured model marked. Submits now pre-flight the chosen model against that catalog: lens `max_tokens` clamps to the provider's completion ceiling, deprecated models are flagged, and models that do not advertise structured-output support are refused outright, since every lens depends on `json_schema` response_format. The catalog cache is shared between the `models` action and submit-time pricing resolution. +- **Broad-Side: truncation detection with fence-tolerant parsing** (#103, #133). Lens output is now parsed tolerantly — markdown code fences are stripped before JSON parsing, mirroring the tolerance in OpenRouter's headless-agent scaffold. Parseable output is saved as clean JSON; output that still does not parse (the signature of a `max_tokens` cutoff) is saved verbatim but marked `truncated`. The collect summary and `run-meta.json` report truncation counts, and the synthesis prompt is told which modules are unrepresented rather than clean. Also documented the retry-safety invariant (batch requests are pure, resubmission always safe) and the distinction between Broad-Side's pre-flight `max_cost` estimate and OpenRouter's runtime cost accounting. +- **Broad-Side: triage pass** (#103, #135). Collect now runs a second cross-lens post-pass alongside synthesis (skip with `include_triage: false`): every finding is scored by impact × fix difficulty and turned into a prioritized work order — P0–P3 priority, effort estimate, per-module grouping, deduplicated leads, and explicit `omitted` notes for dropped items — saved as `triage.json`/`triage.md` and surfaced in the collect summary. The triage prompt frames the queue as a starting point for re-verification, never a commitment. Both post-passes submit as separate batches together and poll independently, and the state file tracks each so a resumed collect can finish whichever is still pending. + ## [0.16.0] — 2026-08-17 The field-test round. Immediately after 0.15.0 shipped, the same 7-phase deepseek-harness analysis was re-run on a fresh worktree through the published binary — this time with the driving chat as orchestrator — and the run's own gaps became this release (#111–#114): the very first completion appended decision rows without their promised heading, both full runs ended with no dashboard ever rendered, the analysis→publish→synthesis library loop was unreachable from any served text, and the terminal completion message named nothing actionable while skills, amendments, a publishable spec, and the usage log all sat unused. diff --git a/README.md b/README.md index 74c4d68..02ce3c3 100644 --- a/README.md +++ b/README.md @@ -357,6 +357,7 @@ Implements MCP spec revision [`2025-11-25`](https://modelcontextprotocol.io/spec | `codecarto_publish` | MCP-only library publish | | `codecarto_library_list` | MCP-only library listing | | `codecarto_library_reindex` | MCP-only library reindex | +| `codecarto_broadside` | MCP-only batch reconnaissance (Broad-Side) | Each workflow tool accepts an absolute `cwd` for the target repository. `codecarto_init` requires `force: true` to overwrite an existing `.codecarto/` (instead of Pi's interactive confirmation). The library tools accept an explicit absolute `library_path` or resolve `library.path` from `.codecarto/workflow/config.yaml` / `~/.codecarto/config.yaml`. The library schema is experimental and may break before v2. diff --git a/ROADMAP.md b/ROADMAP.md new file mode 100644 index 0000000..4848f7b --- /dev/null +++ b/ROADMAP.md @@ -0,0 +1,73 @@ +# Roadmap — Broad-Side + +Broad-Side is CodeCartographer's batch reconnaissance feature: a cheap, +unattended multi-lens scan over the OpenRouter Batch API that produces +unverified scouting leads for the interactive pipeline to confirm. Shipped +behind the `feat/103-broadside` branch (MCP surface first). + +This roadmap is the working agreement on what comes next. Items are tracked +as GitHub issues labeled `feat`; status changes happen in the issues, this +file only moves when a tier completes. + +## Shipped + +- `codecarto_broadside` MCP tool — `submit` / `collect` / `status` actions. +- Six lenses: architecture, api, security, defect, conventions, porting. +- Directory slicing with overflow splitting (no truncation of coverage). +- Cross-lens synthesis report. +- Repo-local state file (`broadside/state.json`) with resumable collect. +- Works without an initialized workspace; `codecarto_init` tolerates a + scout-only `.codecarto/`; scaffold refresh never touches broadside state. +- Auth-expiry fast bail, empty-lens skip, submission-throw guard. +- Live per-model pricing lookup (OpenRouter catalog, 24h cache) with + `max_cost` expense guardrail and `force` override; configurable model + (`config.yaml` `model` key, now wired through submissions). +- `models` action: batch-model catalog with pricing, context, output caps, + structured-output support, and optional Artificial Analysis coding + benchmarks; submit pre-flight refuses models without structured outputs + and clamps lens `max_tokens` to the provider's completion ceiling. +- Triage post-pass on collect: findings scored by impact × difficulty into + a P0–P3 work order with effort estimates, saved as triage.json/md. +- Tests: 35 unit tests (fake-fetcher based), opt-in live smoke script. + +## Tier 1 — make Broad-Side better at what it does + +| Item | Issue | Notes | +|---|---|---| +| **Triage lens** — prioritized fix queue (impact × difficulty, grouped by module) | [#135](https://github.com/HuginnIndustries/CodeCartographer/issues/135) | **Shipped**: triage pass runs on collect alongside synthesis (`include_triage` to skip) | +| **Truncation repair** — detect max_tokens-cutoff JSON, resubmit slices, report truncation in summaries | [#133](https://github.com/HuginnIndustries/CodeCartographer/issues/133) | Partially shipped: fence-tolerant parsing + `truncated` flagging in collect, meta, and synthesis. Remaining: automatic resubmit of truncated slices | +| **Concurrent polling** — poll all in-flight batches round-robin against one deadline | [#136](https://github.com/HuginnIndustries/CodeCartographer/issues/136) | Submissions already parallel; polling is sequential today | +| **Per-language prompts** — Go/Python/Rust/TS lens prompts; globs already adapt | [#137](https://github.com/HuginnIndustries/CodeCartographer/issues/137) | Schemas stay shared so synthesis is unaffected | + +## Tier 2 — integration depth + +| Item | Issue | Notes | +|---|---|---| +| **Pi extension** — `/codecarto-broadside` command with lens picker and live progress | [#138](https://github.com/HuginnIndustries/CodeCartographer/issues/138) | Agreed order: MCP first (shipped), Pi second | +| **Pipeline phase** — `broadside-scout` phase feeding later phases via `required_reads` | [#139](https://github.com/HuginnIndustries/CodeCartographer/issues/139) | SKILL.md contract stays: leads, never evidence | +| **Zero-config executive** — meta-pass picks lenses and slicing resolution from repo shape | [#140](https://github.com/HuginnIndustries/CodeCartographer/issues/140) | Decision recorded in `run-meta.json` for reproducibility | + +## Tier 3 — cost and coverage economics + +| Item | Issue | Notes | +|---|---|---| +| **Multi-model** — DeepSeek/Anthropic batch endpoints behind the lens registry | [#141](https://github.com/HuginnIndustries/CodeCartographer/issues/141) | Partially shipped: catalog lookup, `models` action, pricing + capability pre-flight. Remaining: per-model prompt tweaks and a stronger default for semantic lenses | +| **Incremental re-scouting** — diff against previous run's HEAD, rescan changed modules only | [#142](https://github.com/HuginnIndustries/CodeCartographer/issues/142) | Makes recurring scans O(delta) | +| **CodeCartoShow pipeline stage** — BATCH-SCOUT between SELECT and the interactive run | [CodeCartoShow#1](https://github.com/HuginnIndustries/CodeCartoShow/issues/1) | `scripts/batch-analyze.py` proved it; evidence rules apply unchanged | + +## Tier 4 — open questions, not commitments + +| Item | Issue | Notes | +|---|---|---| +| **Headless-agent lens queue** — sync-priced, tool-using variant via `@openrouter/agent` | [#143](https://github.com/HuginnIndustries/CodeCartographer/issues/143) | 2× batch pricing; overlaps the interactive pipeline. The likelier winner is the hybrid: batch sweeps + one sync-priced verification pass on the top N findings | + +## Principles + +1. **Leads, never evidence.** Every Broad-Side artifact carries the + disclaimer; nothing downstream may cite a Broad-Side report as fact. +2. **Coverage is spoken, not implied.** Truncations, skipped lenses, and + unscouted scope appear in `run-meta.json` and the collect summary. +3. **Cost before submission.** Estimates are shown on submit; no silent + spend. The live smoke script stays opt-in. +4. **The cheap model is a feature.** Gemini batch is weak but ~50% price; + its job is to tell the expensive run where to look, not to be right. diff --git a/core/broadside.ts b/core/broadside.ts new file mode 100644 index 0000000..92abb0d --- /dev/null +++ b/core/broadside.ts @@ -0,0 +1,2322 @@ +// Broad-Side: cheap batch reconnaissance over the OpenRouter Batch API. +// +// Broad-Side fires every analysis lens at a repository at once. Each lens is a +// single-turn prompt with a structured-output JSON schema, submitted as an +// asynchronous batch job (Google Gemini's batch endpoint, ~50% of sync pricing) +// and polled to completion. Results land in `.codecarto/broadside//` as +// JSON plus rendered markdown, and an optional synthesis pass cross-references +// every lens into one executive report. +// +// This is deliberately NOT the interactive CodeCartographer pipeline. The batch +// API is text-in/text-out: no filesystem access, no multi-turn exploration, no +// runtime verification. Broad-Side findings are unverified scouting signals — +// file:line leads that a real analysis (or a human) must confirm. That division +// of labor is the point: a ~$0.50 unattended sweep that tells the expensive +// interactive run where to look. +// +// Field shapes for the model catalog and benchmarks endpoints follow the +// official OpenRouter skills (OpenRouterTeam/skills: openrouter-models, +// openrouter-benchmarks). +// +// RESUBMISSION INVARIANT: batch requests are pure functions of their input — +// no tools, no filesystem, no side effects — so resubmitting a failed or +// truncated slice is always safe. This is the retry rule OpenRouter's own +// headless-agent scaffold states the hard way (retry only before tool calls, +// because replaying a mutating tool would double-execute it); here the rule is +// satisfied by construction. If Broad-Side ever gains server tools +// (openrouter:web_search etc.), this invariant becomes load-bearing and the +// resubmit path must gate on whether any tool executed. +// +// Deliberately not in .codecarto/ template prose: Broad-Side requires runtime +// code, so it lives on the executable surfaces (MCP today, Pi on the roadmap). + +import { mkdir, readFile, readdir, writeFile } from "node:fs/promises"; +import { execFile } from "node:child_process"; +import { promisify } from "node:util"; +import { dirname, join } from "node:path"; +import { pathExists, sleep } from "./utils.ts"; +import { loadYamlFile } from "./yaml.ts"; + +const execFileAsync = promisify(execFile); + +// ---------- constants ---------- + +export const BROADSIDE_MODEL = "google/gemini-3.7-flash:batch"; +export const BROADSIDE_BATCH_URL = "https://openrouter.ai/api/beta/batches"; +export const BROADSIDE_DIR = "broadside"; // relative to .codecarto/ +export const BROADSIDE_STATE_FILE = "state.json"; +export const BROADSIDE_CONFIG_FILE = "config.yaml"; +export const BROADSIDE_STATE_SCHEMA_VERSION = 1; + +// Per-token pricing in USD (OpenRouter, google/gemini-3.7-flash:batch). +export const BROADSIDE_INPUT_PRICE_PER_M = 0.1875; +export const BROADSIDE_OUTPUT_PRICE_PER_M = 0.9375; + +// OpenRouter's public model catalog; pricing, context, and capabilities live +// per model id. The benchmarks endpoint adds coding/intelligence indices. +export const BROADSIDE_MODELS_URL = "https://openrouter.ai/api/v1/models"; +export const BROADSIDE_BENCHMARKS_URL = "https://openrouter.ai/api/v1/benchmarks"; + +export const BROADSIDE_CATALOG_CACHE_FILE = "model-catalog.json"; +export const BROADSIDE_CATALOG_CACHE_TTL_MS = 24 * 60 * 60 * 1000; + +export const BROADSIDE_LENS_IDS = [ + "architecture", + "api", + "security", + "defect", + "conventions", + "porting", +] as const; +export type BroadsideLensId = (typeof BROADSIDE_LENS_IDS)[number]; + +export const BROADSIDE_POLL_INTERVAL_MS = 15_000; +export const BROADSIDE_DEFAULT_POLL_BUDGET_MS = 25 * 60 * 1000; + +// ---------- types ---------- + +export type ModelPricing = { + /** USD per million input tokens. */ + inputPerM: number; + /** USD per million output tokens. */ + outputPerM: number; + /** Where the numbers came from — affects what the submit text claims. */ + source: "built-in" | "config" | "live" | "cache"; +}; + +/** The subset of the OpenRouter model catalog Broad-Side actually uses. */ +export type CatalogEntry = { + id: string; + name: string; + inputPerM: number; + outputPerM: number; + cachedInputPerM?: number; + contextLength?: number; + maxCompletionTokens?: number; + /** Empty array means unknown, not "supports nothing". */ + supportedParameters: string[]; + expirationDate?: string | null; +}; + +export type CodingBenchmarks = { + /** Base model slug (batch suffix stripped) → indices. */ + byBaseSlug: Record; + /** Citation/attribution metadata from the benchmarks endpoint. */ + meta: Record; +}; + +export type BroadsideCatalogResult = { + model: string; + source: "built-in" | "config" | "live" | "cache"; + entry: CatalogEntry | null; + benchmarks?: CodingBenchmarks; +}; + +export type JsonSchemaDef = { + name: string; + strict: boolean; + schema: Record; +}; + +export type RepoInfo = { + name: string; + path: string; + language: string; + manifest: { path: string; content: string } | null; + mainFile: string; + readmeFirst: string; + fileTree: string; + fileCounts: Record; + sourceGlob: string; + sourceExts: string[]; +}; + +export type FileSlice = { + moduleName: string; + content: string; + fileCount: number; + chars: number; +}; + +export type BatchRequest = { + custom_id: string; + body: { + model: string; + messages: { role: "system" | "user"; content: string }[]; + response_format: { type: "json_schema"; json_schema: JsonSchemaDef }; + max_tokens: number; + }; +}; + +export type BatchTerminalStatus = "completed" | "failed" | "expired" | "cancelled"; + +export type BroadsideBatchEntry = { + batchId: string; + requests: number; + status: string; + submittedAt: string; + completedAt?: string; + estimatedCost: number; + cost?: number; + resultCount?: number; + error?: unknown; +}; + +export type BroadsideSynthesisEntry = { + batchId?: string; + status: "pending" | "submitted" | "completed" | "failed"; + cost?: number; +}; + +/** One triage item — a scouting lead turned into a work-order entry. */ +export type TriageItem = { + title: string; + severity: string; + module: string; + impact: "high" | "medium" | "low"; + difficulty: "high" | "medium" | "low"; + priority: string; + effort_estimate: string; + rationale: string; +}; + +export type BroadsideTriageEntry = { + batchId?: string; + status: "pending" | "submitted" | "completed" | "failed"; + cost?: number; +}; + +export type BroadsideRun = { + id: string; + createdAt: string; + model: string; + lenses: BroadsideLensId[]; + status: "in-flight" | "completed" | "partial" | "failed"; + outputDir: string; // relative to .codecarto/broadside/ + batches: Partial>; + synthesis: BroadsideSynthesisEntry; + triage: BroadsideTriageEntry; + totalCost?: number; + pricing?: ModelPricing; + maxCost?: number; +}; + +export type BroadsideStateFile = { + schema_version: number; + runs: BroadsideRun[]; +}; + +export type BroadsideConfig = { + model: string; + apiKey: string; + defaultLenses: BroadsideLensId[]; + /** Approximate run expense limit in USD; 0 means no limit. */ + maxCost: number; + /** Manual pricing overrides (USD per million). Live lookup is preferred. */ + pricing: { inputPerM: number; outputPerM: number } | null; +}; + +export type BroadsideSubmitResult = { + runId: string; + outputDir: string; + batches: Partial>; + estimatedTotalCost: number; + estimatedInputTokens: number; + estimatedOutputTokens: number; + pricing: ModelPricing; + maxCost?: number; + modelInfo: { + contextLength?: number; + maxCompletionTokens?: number; + supportsStructuredOutputs?: boolean; + expirationDate?: string | null; + }; +}; + +export type BroadsideCollectResult = { + runId: string; + status: string; + totalCost: number; + resultCount: number; + /** Results whose JSON did not parse even after fence stripping — + * the signature of an output cut off at max_tokens. */ + truncatedCount: number; + lensOutcomes: Partial< + Record + >; + synthesis: BroadsideSynthesisEntry; + triage: BroadsideTriageEntry; + topFindings: { title: string; severity: string; sourceLens: string; summary: string }[]; + topTriageItems: TriageItem[]; +}; + +// ---------- JSON schemas (one per lens, plus synthesis) ---------- + +const SCHEMAS: Record = { + architecture: { + name: "architecture_report", + strict: true, + schema: { + type: "object", + properties: { + tech_stack: { + type: "object", + properties: { + language: { type: "string" }, + version: { type: "string" }, + build_system: { type: "string" }, + key_dependencies: { type: "array", items: { type: "string" } }, + }, + required: ["language", "build_system"], + additionalProperties: false, + }, + module_architecture: { + type: "array", + items: { + type: "object", + properties: { + name: { type: "string" }, + role: { type: "string" }, + file_count: { type: "integer" }, + depends_on: { type: "array", items: { type: "string" } }, + }, + required: ["name", "role"], + additionalProperties: false, + }, + }, + data_flow: { type: "string" }, + entry_points: { type: "array", items: { type: "string" } }, + notable_patterns: { type: "array", items: { type: "string" } }, + }, + required: ["tech_stack", "module_architecture", "data_flow", "entry_points"], + additionalProperties: false, + }, + }, + api_surface: { + name: "api_surface_report", + strict: true, + schema: { + type: "object", + properties: { + endpoints: { + type: "array", + items: { + type: "object", + properties: { + method: { type: "string" }, + path: { type: "string" }, + handler: { type: "string" }, + auth_required: { type: "boolean" }, + description: { type: "string" }, + }, + required: ["method", "path", "handler", "auth_required"], + additionalProperties: false, + }, + }, + data_types: { + type: "array", + items: { + type: "object", + properties: { + name: { type: "string" }, + kind: { type: "string" }, + fields_summary: { type: "string" }, + }, + required: ["name", "kind"], + additionalProperties: false, + }, + }, + authentication_flow: { type: "string" }, + error_handling: { type: "string" }, + }, + required: ["endpoints"], + additionalProperties: false, + }, + }, + security: { + name: "security_review_report", + strict: true, + schema: { + type: "object", + properties: { + findings: { + type: "array", + items: { + type: "object", + properties: { + severity: { type: "string", enum: ["critical", "high", "medium", "low"] }, + category: { type: "string" }, + title: { type: "string" }, + location: { type: "string" }, + description: { type: "string" }, + }, + required: ["severity", "title", "description"], + additionalProperties: false, + }, + }, + overall_assessment: { type: "string" }, + coverage_note: { type: "string" }, + }, + required: ["findings", "overall_assessment"], + additionalProperties: false, + }, + }, + defect_mechanical: { + name: "defect_scan_report", + strict: true, + schema: { + type: "object", + properties: { + module: { type: "string" }, + findings: { + type: "array", + items: { + type: "object", + properties: { + severity: { type: "string", enum: ["high", "medium", "low"] }, + pattern: { type: "string" }, + title: { type: "string" }, + location: { type: "string" }, + description: { type: "string" }, + suggestion: { type: "string" }, + }, + required: ["severity", "pattern", "title", "description"], + additionalProperties: false, + }, + }, + patterns_checked: { type: "array", items: { type: "string" } }, + files_scanned: { type: "integer" }, + overall_notes: { type: "string" }, + }, + required: ["module", "findings", "patterns_checked", "files_scanned"], + additionalProperties: false, + }, + }, + conventions: { + name: "conventions_report", + strict: true, + schema: { + type: "object", + properties: { + module: { type: "string" }, + naming_conventions: { + type: "object", + properties: { + packages: { type: "string" }, + types: { type: "string" }, + functions: { type: "string" }, + variables: { type: "string" }, + files: { type: "string" }, + tests: { type: "string" }, + }, + additionalProperties: false, + }, + error_handling_pattern: { type: "string" }, + logging_approach: { type: "string" }, + test_patterns: { type: "string" }, + code_organization: { type: "string" }, + idioms: { type: "array", items: { type: "string" } }, + inconsistencies: { + type: "array", + items: { + type: "object", + properties: { + description: { type: "string" }, + locations: { type: "array", items: { type: "string" } }, + }, + required: ["description"], + additionalProperties: false, + }, + }, + promotable_conventions: { + type: "array", + items: { + type: "object", + properties: { + title: { type: "string" }, + rule: { type: "string" }, + evidence: { type: "string" }, + }, + required: ["title", "rule"], + additionalProperties: false, + }, + }, + files_scanned: { type: "integer" }, + }, + required: ["module", "naming_conventions", "files_scanned"], + additionalProperties: false, + }, + }, + porting: { + name: "porting_surface_report", + strict: true, + schema: { + type: "object", + properties: { + module: { type: "string" }, + platform_coupling: { + type: "array", + items: { + type: "object", + properties: { + platform: { type: "string" }, + mechanisms: { type: "array", items: { type: "string" } }, + files: { type: "array", items: { type: "string" } }, + }, + required: ["platform", "mechanisms"], + additionalProperties: false, + }, + }, + external_dependencies: { + type: "array", + items: { + type: "object", + properties: { + name: { type: "string" }, + role: { type: "string" }, + replaceability: { type: "string" }, + }, + required: ["name"], + additionalProperties: false, + }, + }, + build_system_complexity: { type: "string" }, + porting_risk_areas: { + type: "array", + items: { + type: "object", + properties: { + area: { type: "string" }, + risk: { type: "string", enum: ["low", "medium", "high"] }, + notes: { type: "string" }, + }, + required: ["area", "risk"], + additionalProperties: false, + }, + }, + files_scanned: { type: "integer" }, + }, + required: ["module", "platform_coupling", "files_scanned"], + additionalProperties: false, + }, + }, + synthesis: { + name: "synthesis_report", + strict: true, + schema: { + type: "object", + properties: { + executive_summary: { type: "string" }, + severity_summary: { + type: "object", + properties: { + critical: { type: "integer" }, + high: { type: "integer" }, + medium: { type: "integer" }, + low: { type: "integer" }, + }, + required: ["critical", "high", "medium", "low"], + additionalProperties: false, + }, + top_findings: { + type: "array", + items: { + type: "object", + properties: { + title: { type: "string" }, + severity: { type: "string" }, + source_lens: { type: "string" }, + summary: { type: "string" }, + }, + required: ["title", "severity", "source_lens", "summary"], + additionalProperties: false, + }, + }, + module_assessments: { + type: "array", + items: { + type: "object", + properties: { + module: { type: "string" }, + quality_notes: { type: "string" }, + risk_level: { type: "string", enum: ["low", "medium", "high"] }, + }, + required: ["module", "risk_level"], + additionalProperties: false, + }, + }, + porting_readiness: { type: "string" }, + gaps_and_unknowns: { type: "array", items: { type: "string" } }, + coverage: { type: "string" }, + }, + required: ["executive_summary", "severity_summary", "top_findings"], + additionalProperties: false, + }, + }, + triage: { + name: "triage_report", + strict: true, + schema: { + type: "object", + properties: { + summary: { type: "string" }, + items: { + type: "array", + items: { + type: "object", + properties: { + title: { type: "string" }, + severity: { type: "string" }, + module: { type: "string" }, + impact: { type: "string", enum: ["high", "medium", "low"] }, + difficulty: { type: "string", enum: ["high", "medium", "low"] }, + priority: { type: "string" }, + effort_estimate: { type: "string" }, + rationale: { type: "string" }, + }, + required: ["title", "severity", "module", "impact", "difficulty", "priority", "rationale"], + additionalProperties: false, + }, + }, + omitted: { + type: "array", + items: { type: "string" }, + description: "Leads deliberately dropped from the queue and why (duplicates, too vague, out of scope)", + }, + }, + required: ["summary", "items"], + additionalProperties: false, + }, + }, +}; + +// ---------- lens definitions ---------- + +type LensDefinition = { + id: BroadsideLensId; + name: string; + description: string; + schemaName: string; + sliceBy: "none" | "directory"; + maxChars: number; + maxTokens: number; + // Test files rarely carry the surface a lens audits — they bulk up the + // batch and the bill. Convention extraction is the exception: it exists + // partly to catalog test patterns. + skipTestFiles?: boolean; + // Globs are matched against repo-relative forward-slash paths. + globsFor: (info: RepoInfo) => string[]; + systemPrompt: (info: RepoInfo) => string; + userPrompt: (info: RepoInfo, source: string, moduleName: string) => string; +}; + +const LENSES: Record = { + architecture: { + id: "architecture", + name: "Architecture, tech stack & module map", + description: "Repo-wide structural analysis from the manifest, entry point, README, and file tree.", + schemaName: "architecture", + sliceBy: "none", + maxChars: 0, // repo-info lens; no file slurping + maxTokens: 8000, + globsFor: () => [], + systemPrompt: () => + "You are a senior software architect performing a structural analysis of a " + + "codebase. You receive the project manifest, entry point, README excerpt, and " + + "file tree. Return a JSON object following the architecture_report schema " + + "exactly. All findings must be traceable to the provided files — cite file " + + "paths. If you can't determine something, say so rather than guessing.", + userPrompt: (info) => { + const manifest = info.manifest + ? `## ${info.manifest.path}\n\`\`\`\n${info.manifest.content}\n\`\`\`\n\n` + : "## Manifest\n[no manifest found]\n\n"; + return ( + "Analyze the architecture of this project.\n\n" + + manifest + + `## Entry point\n\`\`\`\n${info.mainFile || "[missing]"}\n\`\`\`\n\n` + + `## README (first 4000 chars)\n${info.readmeFirst || "[missing]"}\n\n` + + `## File tree (depth 3, capped)\n${info.fileTree || "[missing]"}\n\n` + + "## File counts by extension\n```json\n" + + JSON.stringify(info.fileCounts) + + "\n```\n\n" + + "Return the architecture_report JSON schema." + ); + }, + }, + api: { + id: "api", + name: "API surface audit", + description: "Endpoint catalog, request/response types, auth flow, error handling.", + schemaName: "api_surface", + sliceBy: "none", + maxChars: 70_000, + maxTokens: 8000, + skipTestFiles: true, + globsFor: (info) => + info.language === "go" + ? ["server/**/*.go", "server/*.go", "api/**/*.go", "api/*.go"] + : [ + "server/**", + "api/**", + "src/server/**", + "src/api/**", + "mcp-server/**", + "**/*routes*", + "**/*router*", + "**/*handler*", + "**/*endpoint*", + ], + systemPrompt: () => + "You are a senior API auditor. Given source files from an HTTP server, " + + "extract every HTTP endpoint (method, path, handler function, auth requirement) " + + "and every key request/response data type. Return a JSON object following the " + + "api_surface_report schema exactly. Cite specific file:line locations.", + userPrompt: (info, source, moduleName) => + "Extract the full API surface from these server source files:\n\n" + + source + + "\n\nReturn the api_surface_report JSON schema.", + }, + security: { + id: "security", + name: "Security review", + description: "Auth, authorization, input validation, TLS, secrets, trust boundaries.", + schemaName: "security", + sliceBy: "none", + maxChars: 70_000, + maxTokens: 8000, + skipTestFiles: true, + globsFor: (info) => + info.language === "go" + ? ["server/**/*.go", "server/*.go", "**/auth*.go", "**/middleware/**/*.go", "SECURITY.md"] + : ["server/**", "**/auth*", "**/middleware/**", "SECURITY.md"], + systemPrompt: () => + "You are a security engineer performing a first-pass review of a codebase. " + + "Given source files, identify potential security issues — focusing on " + + "authentication, authorization, input validation, TLS, secrets handling, " + + "and trust boundaries. Return a JSON object following the security_review_report " + + "schema. Rate severity as critical/high/medium/low. Be specific: cite file:line. " + + "If the provided files don't cover an area, state the gap in coverage_note.", + userPrompt: (info, source, moduleName) => + "Review these server source files for security issues:\n\n" + + source + + "\n\nReturn the security_review_report JSON schema.", + }, + defect: { + id: "defect", + name: "Mechanical defect scan", + description: "Nil derefs, error gaps, leaks, races, panics — pattern-based, sliced per module.", + schemaName: "defect_mechanical", + sliceBy: "directory", + maxChars: 60_000, + maxTokens: 6000, + globsFor: (info) => [info.sourceGlob], + systemPrompt: (info) => + `You are a senior code reviewer performing an automated defect scan on ${info.language} ` + + "source files. Look for these specific patterns:\n" + + " 1. Nil/null pointer dereference risks (unchecked returns, missing guards)\n" + + " 2. Error handling gaps (ignored errors, deferred errors unchecked)\n" + + " 3. Resource leaks (unclosed files, connections, goroutines without ctx)\n" + + " 4. Race conditions (shared state without sync, channel misuse)\n" + + " 5. Integer overflow/underflow in arithmetic or bounds\n" + + " 6. Unsafe type assertions without ok check\n" + + " 7. Panic-prone code (slice out of bounds, map access without ok)\n" + + " 8. Timezone/locale assumptions\n\n" + + "Return a JSON object following the defect_scan_report schema. " + + "Cite file:line for every finding. List which patterns you checked. " + + "If the code looks clean for a pattern, say so rather than staying silent. " + + "Prefer precision over volume — 3 solid findings beat 15 vague ones.", + userPrompt: (info, source, moduleName) => + `Scan this ${info.language} module for mechanical defects.\n\n` + + `Module: ${moduleName}\n\n` + + "## Source files\n\n" + + source + + "\n\nReturn the defect_scan_report JSON schema.", + }, + conventions: { + id: "conventions", + name: "Convention extraction", + description: "Naming, error handling, idioms, inconsistencies, promotable conventions.", + schemaName: "conventions", + sliceBy: "directory", + maxChars: 60_000, + maxTokens: 6000, + globsFor: (info) => [info.sourceGlob], + systemPrompt: () => + "You are a code style analyst extracting conventions from source files. " + + "Catalog: naming conventions per category (packages, types, functions, variables, " + + "source files, test files), the dominant error-handling pattern, logging approach, " + + "test organization patterns, file/package organization rules, and recurring idioms. " + + "Also flag inconsistencies — places where the same convention is violated. " + + "If you find well-established conventions worth formalizing, list them as " + + "promotable_conventions with a title, rule, and evidence from the code. " + + "Return a JSON object following the conventions_report schema.", + userPrompt: (info, source, moduleName) => + "Extract coding conventions from this module.\n\n" + + `Module: ${moduleName}\n\n` + + "## Source files\n\n" + + source + + "\n\nReturn the conventions_report JSON schema.", + }, + porting: { + id: "porting", + name: "Porting surface assessment", + description: "Platform coupling, external deps, build complexity, porting risk areas.", + schemaName: "porting", + sliceBy: "directory", + maxChars: 60_000, + maxTokens: 6000, + skipTestFiles: true, + globsFor: (info) => [ + info.sourceGlob, + "**/*.c", + "**/*.h", + "**/*.cpp", + "**/*.cc", + "**/*.m", + "**/*.mm", + "**/CMakeLists.txt", + "**/*.cmake", + "go.mod", + ], + systemPrompt: () => + "You are a software portability analyst. Examine source files and " + + "identify everything that ties this codebase to a specific platform, OS, " + + "architecture, or external dependency. Catalog: platform-specific build tags, " + + "FFI usage, OS-specific syscalls, external library bindings, and " + + "compile-time constants that encode platform assumptions. " + + "For each external dependency, note whether it could be replaced by a " + + "cross-platform alternative. Assess the build system complexity. " + + "Return a JSON object following the porting_surface_report schema.", + userPrompt: (info, source, moduleName) => + "Assess porting surface for this module.\n\n" + + `Module: ${moduleName}\n\n` + + "## Source files\n\n" + + source + + "\n\nReturn the porting_surface_report JSON schema.", + }, +}; + +export function getLens(lensId: BroadsideLensId): LensDefinition { + return LENSES[lensId]; +} + +export function listLenses(): LensDefinition[] { + return BROADSIDE_LENS_IDS.map((id) => LENSES[id]); +} + +// ---------- repo info ---------- + +const SKIP_DIR_NAMES = new Set([ + ".git", + ".github", + ".claude", + ".opencode", + ".codecarto", + "node_modules", + "vendor", + "dist", + "build", + "target", + "testdata", + "__pycache__", +]); + +const SKIP_FILE_EXTENSIONS = new Set([ + ".png", + ".jpg", + ".jpeg", + ".gif", + ".svg", + ".ico", + ".icns", + ".bmp", + ".webp", + ".mp3", + ".mp4", + ".mov", + ".avi", + ".wav", + ".ogg", + ".zip", + ".gz", + ".tar", + ".bz2", + ".xz", + ".7z", + ".pdf", + ".woff", + ".woff2", + ".ttf", + ".eot", + ".otf", + ".bin", + ".exe", + ".dll", + ".so", + ".dylib", + ".a", + ".o", + ".obj", + ".class", + ".jar", + ".war", + ".pyc", + ".wasm", + ".model", + ".bpe", +]); + +const MANIFEST_CANDIDATES = [ + ["go.mod", "go"], + ["package.json", "typescript"], + ["Cargo.toml", "rust"], + ["pyproject.toml", "python"], + ["setup.py", "python"], + ["requirements.txt", "python"], +]; + +const SOURCE_SPECS: Record = { + go: { glob: "**/*.go", exts: [".go"] }, + python: { glob: "**/*.py", exts: [".py"] }, + rust: { glob: "**/*.rs", exts: [".rs"] }, + typescript: { glob: "**/*.ts", exts: [".ts", ".tsx"] }, + javascript: { glob: "**/*.js", exts: [".js", ".jsx"] }, +}; + +async function listRepoFiles(targetDir: string): Promise { + // git ls-tree is the fast path; fall back to a bounded walk for non-git trees. + try { + const { stdout } = await execFileAsync("git", ["-C", targetDir, "ls-tree", "-r", "--name-only", "HEAD"], { + maxBuffer: 64 * 1024 * 1024, + }); + return stdout.split("\n").filter(Boolean); + } catch { + return walkFiles(targetDir, targetDir, 0, 30_000); + } +} + +async function walkFiles( + rootDir: string, + dir: string, + depth: number, + remaining: number, +): Promise { + if (remaining <= 0) return []; + let out: string[] = []; + let entries: import("node:fs").Dirent[] = []; + try { + entries = await readdir(dir, { withFileTypes: true }); + } catch { + return out; + } + for (const entry of entries) { + if (entry.name.startsWith(".") && entry.name !== ".github") continue; + if (entry.isDirectory()) { + if (SKIP_DIR_NAMES.has(entry.name)) continue; + if (depth > 8) continue; + const children = await walkFiles(rootDir, join(dir, entry.name), depth + 1, remaining - out.length); + out = out.concat(children); + } else if (entry.isFile()) { + const rel = join(dir, entry.name).slice(rootDir.length + 1).split("\\").join("/"); + out.push(rel); + } + } + return out; +} + +function detectLanguage(fileCounts: Record, manifestPath: string | null): string { + if (manifestPath) { + for (const [candidate, lang] of MANIFEST_CANDIDATES) { + if (manifestPath === candidate) return lang; + } + } + const counts: Record = { go: fileCounts[".go"] ?? 0, python: fileCounts[".py"] ?? 0, rust: fileCounts[".rs"] ?? 0, typescript: (fileCounts[".ts"] ?? 0) + (fileCounts[".tsx"] ?? 0) }; + const best = Object.entries(counts).sort((a, b) => b[1] - a[1])[0]; + return best && best[1] > 0 ? best[0] : "unknown"; +} + +export async function collectRepoInfo(targetDir: string): Promise { + const allFiles = await listRepoFiles(targetDir); + + const fileCounts: Record = {}; + for (const f of allFiles) { + const slash = f.lastIndexOf("/"); + const base = slash >= 0 ? f.slice(slash + 1) : f; + const dot = base.lastIndexOf("."); + const ext = dot > 0 ? base.slice(dot).toLowerCase() : "(no ext)"; + fileCounts[ext] = (fileCounts[ext] ?? 0) + 1; + } + const sortedCounts: Record = {}; + for (const [ext, n] of Object.entries(fileCounts).sort((a, b) => b[1] - a[1])) { + sortedCounts[ext] = n; + } + + let manifest: { path: string; content: string } | null = null; + for (const [candidate] of MANIFEST_CANDIDATES) { + const p = join(targetDir, candidate); + if (await pathExists(p)) { + try { + manifest = { path: candidate, content: await readFile(p, "utf8") }; + } catch { + manifest = null; + } + break; + } + } + + let mainFile = ""; + for (const candidate of ["main.go", "main.py", "src/main.rs", "src/index.ts", "index.ts"]) { + const p = join(targetDir, candidate); + if (await pathExists(p)) { + try { + mainFile = await readFile(p, "utf8"); + } catch { + mainFile = ""; + } + break; + } + } + + let readmeFirst = ""; + const readmePath = join(targetDir, "README.md"); + if (await pathExists(readmePath)) { + try { + readmeFirst = (await readFile(readmePath, "utf8")).slice(0, 4000); + } catch { + readmeFirst = ""; + } + } + + const fileTree = buildFileTree(allFiles); + + const language = detectLanguage(sortedCounts, manifest?.path ?? null); + const sourceSpec = SOURCE_SPECS[language] ?? SOURCE_SPECS.go; + const name = targetDir.split(/[\\/]/).filter(Boolean).pop() ?? "repo"; + + return { + name, + path: targetDir, + language, + manifest, + mainFile, + readmeFirst, + fileTree, + fileCounts: sortedCounts, + sourceGlob: sourceSpec.glob, + sourceExts: sourceSpec.exts, + }; +} + +function buildFileTree(allFiles: string[], maxDepth = 3, maxLines = 200): string { + const lines: string[] = []; + let count = 0; + for (const f of allFiles) { + if (f.split("/").length - 1 > maxDepth) continue; + if (f.startsWith(".git/") || f.startsWith(".github/")) continue; + if (f.endsWith(".sum") || f.endsWith(".lock")) continue; + lines.push(f); + count += 1; + if (count >= maxLines) { + lines.push(`... (${allFiles.length} total files, showing first ${maxLines})`); + break; + } + } + return lines.join("\n"); +} + +// ---------- glob matching & file slurping ---------- + +function globToRegExp(glob: string): RegExp { + let re = ""; + for (let i = 0; i < glob.length; i++) { + const c = glob[i]; + if (c === "*") { + if (glob[i + 1] === "*") { + // `**/` matches zero or more directories; a trailing `**` + // matches anything including slashes. + if (glob[i + 2] === "/") { + re += "(?:.*/)?"; + i += 2; + } else { + re += ".*"; + i += 1; + } + } else { + re += "[^/]*"; + } + } else if (c === "?") { + re += "[^/]"; + } else { + re += c.replace(/[.+^${}()|[\]\\]/g, "\\$&"); + } + } + return new RegExp(`^${re}$`); +} + +function matchesAnyGlob(path: string, globs: string[]): boolean { + for (const glob of globs) { + if (globToRegExp(glob).test(path)) return true; + } + return false; +} + +function isSlurpable(relPath: string): boolean { + const segments = relPath.split("/"); + for (const seg of segments) { + if (SKIP_DIR_NAMES.has(seg)) return false; + } + const slash = relPath.lastIndexOf("/"); + const base = slash >= 0 ? relPath.slice(slash + 1) : relPath; + const dot = base.lastIndexOf("."); + if (dot > 0 && SKIP_FILE_EXTENSIONS.has(base.slice(dot).toLowerCase())) return false; + return true; +} + +function sanitizeId(segment: string): string { + return segment.replace(/[^a-zA-Z0-9_-]+/g, "-").replace(/^-+|-+$/g, "") || "root"; +} + +function topLevelModule(relPath: string): string { + const slash = relPath.indexOf("/"); + return slash >= 0 ? relPath.slice(0, slash) : "root"; +} + +type CollectedFile = { relPath: string; moduleName: string }; + +function isTestFile(relPath: string): boolean { + const base = relPath.slice(relPath.lastIndexOf("/") + 1); + return /[._](test|spec)\.[a-z]+$/i.test(base) || base.includes("_test."); +} + +function collectLensFiles(allFiles: string[], lens: LensDefinition, info: RepoInfo): CollectedFile[] { + const globs = lens.globsFor(info); + if (globs.length === 0) return []; + const out: CollectedFile[] = []; + for (const f of allFiles) { + if (!isSlurpable(f)) continue; + if (lens.skipTestFiles && isTestFile(f)) continue; + if (!matchesAnyGlob(f, globs)) continue; + out.push({ relPath: f, moduleName: lens.sliceBy === "directory" ? topLevelModule(f) : info.name }); + } + return out; +} + +async function slurpFileList( + targetDir: string, + files: CollectedFile[], + maxChars: number, +): Promise { + const slices: FileSlice[] = []; + let currentModule = ""; + let parts: string[] = []; + let running = 0; + let fileCount = 0; + + const flush = () => { + if (parts.length === 0) return; + slices.push({ + moduleName: currentModule, + content: parts.join("\n"), + fileCount, + chars: running, + }); + parts = []; + running = 0; + fileCount = 0; + }; + + for (const file of files) { + let content = ""; + try { + content = await readFile(join(targetDir, file.relPath), "utf8"); + } catch { + content = "[BINARY or UNREADABLE]"; + } + const block = `=== ${file.relPath} ===\n${content}\n`; + + if (file.moduleName !== currentModule && parts.length > 0) { + flush(); + } + currentModule = file.moduleName; + + if (running + block.length > maxChars && parts.length > 0) { + // Slice is full: flush it and start another slice for the same module + // rather than truncating, so big modules get full coverage. + flush(); + currentModule = file.moduleName; + } + parts.push(block); + running += block.length; + fileCount += 1; + } + flush(); + return slices; +} + +export async function gatherSlices(targetDir: string, lens: LensDefinition, info: RepoInfo): Promise { + if (lens.sliceBy === "none" && lens.globsFor(info).length === 0) { + // Repo-info lens (architecture): the prompt is built from info alone. + return [{ moduleName: "root", content: "", fileCount: 0, chars: 0 }]; + } + const allFiles = await listRepoFiles(targetDir); + const files = collectLensFiles(allFiles, lens, info); + return slurpFileList(targetDir, files, lens.maxChars); +} + +// ---------- request building ---------- + +export function buildBatchRequest( + lens: LensDefinition, + info: RepoInfo, + slice: FileSlice, + index: number, + sliceCount: number, + model: string = BROADSIDE_MODEL, + maxTokensOverride?: number, +): BatchRequest { + const moduleTag = sanitizeId(slice.moduleName); + const customId = sliceCount > 1 ? `${lens.id}-${moduleTag}-${index + 1}` : `${lens.id}-${moduleTag}`; + return { + custom_id: customId, + body: { + model, + messages: [ + { role: "system", content: lens.systemPrompt(info) }, + { role: "user", content: lens.userPrompt(info, slice.content, slice.moduleName) }, + ], + response_format: { type: "json_schema", json_schema: SCHEMAS[lens.schemaName] }, + max_tokens: maxTokensOverride ?? lens.maxTokens, + }, + }; +} + +export function estimateCost( + lens: LensDefinition, + slices: FileSlice[], + pricing: ModelPricing, + maxTokensOverride?: number, +): { + inputTokens: number; + outputTokens: number; + cost: number; +} { + const inputTokens = Math.ceil(slices.reduce((sum, s) => sum + (lens.maxChars === 0 ? 6000 : s.chars), 0) / 4); + const outputTokens = Math.ceil((maxTokensOverride ?? lens.maxTokens) * 0.75); + const cost = + (inputTokens / 1_000_000) * pricing.inputPerM + + (outputTokens / 1_000_000) * pricing.outputPerM; + return { inputTokens, outputTokens, cost }; +} + +// ---------- state & config ---------- + +export function broadsideDirFor(cwd: string): string { + return join(cwd, ".codecarto", BROADSIDE_DIR); +} + +export function defaultBroadsideState(): BroadsideStateFile { + return { schema_version: BROADSIDE_STATE_SCHEMA_VERSION, runs: [] }; +} + +export async function loadBroadsideState(broadsideDir: string): Promise { + const statePath = join(broadsideDir, BROADSIDE_STATE_FILE); + if (!(await pathExists(statePath))) return defaultBroadsideState(); + try { + const raw = JSON.parse(await readFile(statePath, "utf8")); + if (!raw || typeof raw !== "object" || !Array.isArray(raw.runs)) return defaultBroadsideState(); + return raw as BroadsideStateFile; + } catch { + return defaultBroadsideState(); + } +} + +export async function saveBroadsideState(broadsideDir: string, state: BroadsideStateFile): Promise { + await mkdir(broadsideDir, { recursive: true }); + await writeFile(join(broadsideDir, BROADSIDE_STATE_FILE), `${JSON.stringify(state, null, "\t")}\n`, "utf8"); +} + +export async function loadBroadsideConfig(broadsideDir: string): Promise { + const configPath = join(broadsideDir, BROADSIDE_CONFIG_FILE); + let raw: Record = {}; + if (await pathExists(configPath)) { + try { + raw = (await loadYamlFile>(configPath)) ?? {}; + } catch { + raw = {}; + } + } + const lenses = Array.isArray(raw.default_lenses) + ? (raw.default_lenses.filter((l): l is BroadsideLensId => BROADSIDE_LENS_IDS.includes(l as BroadsideLensId))) + : []; + const rawPricing = (raw.pricing ?? {}) as Record; + const inputOverride = typeof rawPricing.input_per_m === "number" ? rawPricing.input_per_m : undefined; + const outputOverride = typeof rawPricing.output_per_m === "number" ? rawPricing.output_per_m : undefined; + return { + model: typeof raw.model === "string" && raw.model.trim() ? raw.model.trim() : BROADSIDE_MODEL, + apiKey: typeof raw.api_key === "string" ? raw.api_key.trim() : "", + defaultLenses: lenses.length > 0 ? lenses : [...BROADSIDE_LENS_IDS], + maxCost: typeof raw.max_cost === "number" && raw.max_cost > 0 ? raw.max_cost : 0, + pricing: + inputOverride !== undefined && outputOverride !== undefined + ? { inputPerM: inputOverride, outputPerM: outputOverride } + : null, + }; +} + +// ---------- model catalog, pricing, benchmarks ---------- + +type CatalogCacheFile = { + schema_version: number; + fetched_at: string; + models: Record; +}; + +async function readCatalogCache(broadsideDir: string): Promise { + const cachePath = join(broadsideDir, BROADSIDE_CATALOG_CACHE_FILE); + if (!(await pathExists(cachePath))) return null; + try { + const parsed = JSON.parse(await readFile(cachePath, "utf8")) as CatalogCacheFile; + if (!parsed || typeof parsed !== "object" || typeof parsed.models !== "object") return null; + return parsed; + } catch { + return null; + } +} + +async function writeCatalogCache(broadsideDir: string, cache: CatalogCacheFile): Promise { + await mkdir(broadsideDir, { recursive: true }); + await writeFile(join(broadsideDir, BROADSIDE_CATALOG_CACHE_FILE), `${JSON.stringify(cache, null, "\t")}\n`, "utf8"); +} + +function parseCatalogEntry(raw: Record): CatalogEntry | null { + const id = String(raw.id ?? ""); + if (!id) return null; + const p = (raw.pricing ?? {}) as { prompt?: unknown; completion?: unknown; cached_input?: unknown }; + const input = typeof p.prompt === "string" ? Number(p.prompt) : NaN; + const output = typeof p.completion === "string" ? Number(p.completion) : NaN; + if (!Number.isFinite(input) || !Number.isFinite(output)) return null; + const cached = typeof p.cached_input === "string" ? Number(p.cached_input) : NaN; + const topProvider = (raw.top_provider ?? {}) as Record; + const contextLength = typeof raw.context_length === "number" ? raw.context_length : undefined; + const maxCompletion = + typeof topProvider.max_completion_tokens === "number" ? topProvider.max_completion_tokens : undefined; + return { + id, + name: String(raw.name ?? id), + inputPerM: input * 1_000_000, + outputPerM: output * 1_000_000, + cachedInputPerM: Number.isFinite(cached) ? cached * 1_000_000 : undefined, + contextLength, + maxCompletionTokens: maxCompletion, + supportedParameters: Array.isArray(raw.supported_parameters) + ? raw.supported_parameters.map((entry) => String(entry)) + : [], + expirationDate: typeof raw.expiration_date === "string" ? raw.expiration_date : null, + }; +} + +export function builtInCatalogEntry(model: string): CatalogEntry | null { + // The default model's rates are compile-time constants; its capabilities + // are asserted from the shipped configuration (1M context, 64K output, + // structured outputs used by every lens). + if (model !== BROADSIDE_MODEL) return null; + return { + id: BROADSIDE_MODEL, + name: "Google: Gemini 3.7 Flash (batch)", + inputPerM: BROADSIDE_INPUT_PRICE_PER_M, + outputPerM: BROADSIDE_OUTPUT_PRICE_PER_M, + contextLength: 1_048_576, + maxCompletionTokens: 65_536, + supportedParameters: ["tools", "structured_outputs", "json_schema", "response_format"], + expirationDate: null, + }; +} + +export function builtInPricing(model: string): ModelPricing | null { + const entry = builtInCatalogEntry(model); + if (!entry) return null; + return { inputPerM: entry.inputPerM, outputPerM: entry.outputPerM, source: "built-in" }; +} + +export async function resolveCatalogEntry( + broadsideDir: string, + config: BroadsideConfig, + model: string, + apiKey: string, + fetcher: FetchLike = fetch as FetchLike, +): Promise { + // Manual overrides always win for pricing — the user is asserting a rate, + // and a config assertion is cheaper to respect than to second-guess. + // Capabilities stay unknown in that case: nothing is refused, nothing + // is clamped, and the submit text says the pricing came from config. + if (config.pricing) { + return { + model, + source: "config", + entry: { + id: model, + name: model, + inputPerM: config.pricing.inputPerM, + outputPerM: config.pricing.outputPerM, + supportedParameters: [], + }, + }; + } + + const builtIn = builtInCatalogEntry(model); + if (builtIn) return { model, source: "built-in", entry: builtIn }; + + // Unknown model: on-disk cache first, then the live catalog. + const cache = await readCatalogCache(broadsideDir); + const cached = cache?.models[model]; + if (cached && Date.now() - new Date(cache!.fetched_at).getTime() < BROADSIDE_CATALOG_CACHE_TTL_MS) { + return { model, source: "cache", entry: cached }; + } + + let live: CatalogEntry | null = null; + try { + const resp = await fetcher(BROADSIDE_MODELS_URL, { + method: "GET", + headers: { Authorization: `Bearer ${apiKey}` }, + signal: AbortSignal.timeout(30_000), + }); + const data = (await resp.json()) as { data?: Array> }; + const hit = (data.data ?? []).find((m) => String(m.id) === model); + if (hit) live = parseCatalogEntry(hit); + } catch { + live = null; + } + + if (live) { + const updated: CatalogCacheFile = { + schema_version: 2, + fetched_at: new Date().toISOString(), + models: { ...(cache?.models ?? {}) }, + }; + updated.models[model] = live; + await writeCatalogCache(broadsideDir, updated); + return { model, source: "live", entry: live }; + } + + throw new Error( + `Could not resolve per-token pricing for batch model "${model}". ` + + "Set pricing.input_per_m and pricing.output_per_m in .codecarto/broadside/config.yaml " + + "(USD per million tokens), or check the model id against https://openrouter.ai/models?variant=batch.", + ); +} + +export async function resolveModelPricing( + broadsideDir: string, + config: BroadsideConfig, + model: string, + apiKey: string, + fetcher: FetchLike = fetch as FetchLike, +): Promise { + const { source, entry } = await resolveCatalogEntry(broadsideDir, config, model, apiKey, fetcher); + if (!entry) throw new Error(`No pricing resolved for ${model}.`); + return { inputPerM: entry.inputPerM, outputPerM: entry.outputPerM, source }; +} + +/** Base slug with the OpenRouter variant suffix (e.g. `:batch`) stripped. */ +function baseSlug(modelId: string): string { + const idx = modelId.indexOf(":"); + return idx >= 0 ? modelId.slice(0, idx) : modelId; +} + +export async function fetchCodingBenchmarks( + apiKey: string, + fetcher: FetchLike = fetch as FetchLike, +): Promise { + try { + const resp = await fetcher(`${BROADSIDE_BENCHMARKS_URL}?source=artificial-analysis&task_type=coding`, { + method: "GET", + headers: { Authorization: `Bearer ${apiKey}` }, + signal: AbortSignal.timeout(30_000), + }); + const data = (await resp.json()) as { data?: Array>; meta?: Record }; + const byBaseSlug: CodingBenchmarks["byBaseSlug"] = {}; + for (const row of data.data ?? []) { + const slug = baseSlug(String(row.model_permaslug ?? "")); + if (!slug) continue; + const toIndex = (v: unknown) => (typeof v === "number" && Number.isFinite(v) ? v : undefined); + byBaseSlug[slug] = { + codingIndex: toIndex(row.coding_index), + intelligenceIndex: toIndex(row.intelligence_index), + }; + } + return { byBaseSlug, meta: data.meta ?? {} }; + } catch { + return null; + } +} + +export async function listBatchModels( + broadsideDir: string, + config: BroadsideConfig, + apiKey: string, + opts: { includeBenchmarks?: boolean; fetcher?: FetchLike } = {}, +): Promise<{ entries: CatalogEntry[]; source: string; benchmarks: CodingBenchmarks | null; defaultModel: string }> { + const fetcher = opts.fetcher ?? (fetch as FetchLike); + const resp = await fetcher(BROADSIDE_MODELS_URL, { + method: "GET", + headers: { Authorization: `Bearer ${apiKey}` }, + signal: AbortSignal.timeout(30_000), + }); + const data = (await resp.json()) as { data?: Array> }; + const entries: CatalogEntry[] = []; + const seen = new Set(); + for (const raw of data.data ?? []) { + const entry = parseCatalogEntry(raw); + if (!entry || seen.has(entry.id)) continue; + seen.add(entry.id); + if (!entry.id.endsWith(":batch")) continue; + entries.push(entry); + } + entries.sort((a, b) => a.inputPerM + a.outputPerM - (b.inputPerM + b.outputPerM)); + + // Persist the catalog so the next submit's pricing resolution hits cache. + const cache: CatalogCacheFile = { schema_version: 2, fetched_at: new Date().toISOString(), models: {} }; + for (const entry of entries) cache.models[entry.id] = entry; + await writeCatalogCache(broadsideDir, cache); + + const benchmarks = opts.includeBenchmarks ? await fetchCodingBenchmarks(apiKey, fetcher) : null; + return { entries, source: "live", benchmarks, defaultModel: config.model }; +} + +// ---------- batch client ---------- + +export type FetchLike = (url: string, init: Record) => Promise; + +export async function submitBatch( + batchRequests: BatchRequest[], + apiKey: string, + fetcher: FetchLike = fetch as FetchLike, + model: string = BROADSIDE_MODEL, +): Promise<{ batchId: string; status: string; error?: unknown }> { + // The OpenRouter batch endpoint stream-parses the body and requires + // `endpoint` and `model` to serialize before `requests` — key order matters. + const payload = { + endpoint: "/v1/chat/completions", + model, + requests: batchRequests, + }; + const resp = await fetcher(BROADSIDE_BATCH_URL, { + method: "POST", + headers: { + Authorization: `Bearer ${apiKey}`, + "Content-Type": "application/json", + }, + body: JSON.stringify(payload), + signal: AbortSignal.timeout(30_000), + }); + const data = (await resp.json()) as Record; + if (resp.status !== 202) { + return { batchId: "", status: "rejected", error: data }; + } + return { batchId: String(data.id), status: String(data.status) }; +} + +export async function fetchBatch( + batchId: string, + apiKey: string, + fetcher: FetchLike = fetch as FetchLike, +): Promise> { + const resp = await fetcher(`${BROADSIDE_BATCH_URL}/${batchId}`, { + method: "GET", + headers: { Authorization: `Bearer ${apiKey}` }, + signal: AbortSignal.timeout(30_000), + }); + const data = (await resp.json()) as Record; + // Surface the HTTP status so the poller can bail fast on auth expiry + // instead of retrying a dead key for the whole budget. + data.http_status = resp.status; + return data; +} + +export async function pollBatchUntilTerminal( + batchId: string, + apiKey: string, + opts: { + deadlineMs?: number; + onStatus?: (status: string, counts: Record) => void; + fetcher?: FetchLike; + } = {}, +): Promise> { + const deadline = Date.now() + (opts.deadlineMs ?? BROADSIDE_DEFAULT_POLL_BUDGET_MS); + const fetcher = opts.fetcher ?? (fetch as FetchLike); + for (;;) { + let batch: Record; + try { + batch = await fetchBatch(batchId, apiKey, fetcher); + } catch { + if (Date.now() >= deadline) return { id: batchId, status: "timeout" }; + await sleep(BROADSIDE_POLL_INTERVAL_MS); + continue; + } + const httpStatus = Number(batch.http_status ?? 200); + if (httpStatus === 401 || httpStatus === 403) { + return { id: batchId, status: "auth-failed", error: batch.error ?? batch }; + } + const status = String(batch.status ?? "unknown"); + const counts = (batch.request_counts ?? {}) as Record; + opts.onStatus?.(status, counts); + if (["completed", "failed", "expired", "cancelled", "auth-failed"].includes(status)) return batch; + if (Date.now() >= deadline) return { id: batchId, status: "timeout" }; + await sleep(BROADSIDE_POLL_INTERVAL_MS); + } +} + +// ---------- run orchestration ---------- + +export async function runBroadsideSubmit( + cwd: string, + apiKey: string, + opts: { + lenses?: BroadsideLensId[]; + fetcher?: FetchLike; + model?: string; + /** Approximate run expense limit in USD; 0 means no limit. */ + maxCost?: number; + /** Submit even when the estimate exceeds maxCost. */ + force?: boolean; + } = {}, +): Promise { + const info = await collectRepoInfo(cwd); + const lensIds = opts.lenses ?? BROADSIDE_LENS_IDS; + const broadsideDir = broadsideDirFor(cwd); + const model = opts.model ?? BROADSIDE_MODEL; + + // Resolve the model's catalog entry before anything is submitted: the + // guardrail must know real per-token rates, and every lens requires + // structured-output support that not all batch models offer. + const config = await loadBroadsideConfig(broadsideDir); + const catalog = await resolveCatalogEntry(broadsideDir, config, model, apiKey, opts.fetcher); + const pricing: ModelPricing = { + inputPerM: catalog.entry!.inputPerM, + outputPerM: catalog.entry!.outputPerM, + source: catalog.source, + }; + const limit = opts.maxCost ?? config.maxCost; + + const entry = catalog.entry!; + const supportsStructuredOutputs = + entry.supportedParameters.length === 0 || + entry.supportedParameters.some((p) => + ["structured_outputs", "json_schema", "response_format", "structuredoutputs"].includes(p.toLowerCase()), + ); + if (!supportsStructuredOutputs) { + throw new Error( + `Batch model "${model}" does not advertise structured-output support ` + + `(supported_parameters: ${entry.supportedParameters.join(", ") || "unknown"}), but every ` + + "Broad-Side lens requires json_schema response_format. Choose another batch model " + + "(codecarto_broadside action 'models') or pass a pricing override only if you know it works.", + ); + } + // Respect the provider's completion ceiling: a request asking for more + // output than the model can produce fails the whole batch. + const outputCap = entry.maxCompletionTokens; + + // Slice offline first so the estimate covers every request we would send. + const slicesByLens = new Map(); + let estimatedInputTokens = 0; + let estimatedOutputTokens = 0; + let estimatedTotalCost = 0; + const perLensEstimate: Array<{ lens: LensDefinition; cost: number; maxTokens: number }> = []; + for (const lensId of lensIds) { + const lens = getLens(lensId); + const slices = await gatherSlices(cwd, lens, info); + slicesByLens.set(lensId, slices); + const maxTokens = outputCap ? Math.min(lens.maxTokens, outputCap) : lens.maxTokens; + const estimate = estimateCost(lens, slices, pricing, maxTokens); + estimatedInputTokens += estimate.inputTokens; + estimatedOutputTokens += estimate.outputTokens; + estimatedTotalCost += estimate.cost; + perLensEstimate.push({ lens, cost: estimate.cost, maxTokens }); + } + + if (limit > 0 && !opts.force && estimatedTotalCost > limit) { + const breakdown = perLensEstimate + .map(({ lens, cost }) => ` ${lens.name}: ~$${cost.toFixed(4)}`) + .join("\n"); + throw new Error( + `Estimated Broad-Side cost ~$${estimatedTotalCost.toFixed(4)} exceeds the run limit ` + + `$${limit.toFixed(2)}. Nothing was submitted.\nBreakdown:\n${breakdown}\n` + + `Pass force: true to submit anyway, or raise max_cost in .codecarto/broadside/config.yaml.`, + ); + } + + const state = await loadBroadsideState(broadsideDir); + const runId = new Date().toISOString().replace(/[:.]/g, "-"); + const run: BroadsideRun = { + id: runId, + createdAt: new Date().toISOString(), + model, + lenses: [...lensIds], + status: "in-flight", + outputDir: runId, + batches: {}, + synthesis: { status: "pending" }, + triage: { status: "pending" }, + pricing, + maxCost: limit > 0 ? limit : undefined, + }; + state.runs.push(run); + await saveBroadsideState(broadsideDir, state); + + const submissions: Promise[] = []; + for (const lensId of lensIds) { + const lens = getLens(lensId); + const slices = slicesByLens.get(lensId) ?? []; + const maxTokens = outputCap ? Math.min(lens.maxTokens, outputCap) : lens.maxTokens; + const requests = slices.map((s, i) => buildBatchRequest(lens, info, s, i, slices.length, model, maxTokens)); + const estimate = estimateCost(lens, slices, pricing, maxTokens); + + const entry: BroadsideBatchEntry = { + batchId: "", + requests: requests.length, + status: "submitting", + submittedAt: new Date().toISOString(), + estimatedCost: estimate.cost, + }; + run.batches[lensId] = entry; + + if (requests.length === 0) { + // No files matched the lens's globs. That is a coverage gap to + // report, not a batch to submit — the API rejects empty batches. + entry.status = "skipped"; + continue; + } + + submissions.push( + (async () => { + // A network-level throw (DNS, abort, TLS) must not strand the + // entry in "submitting" forever — allSettled would swallow the + // rejection and collect would never see a terminal status. + try { + const { batchId, status, error } = await submitBatch(requests, apiKey, opts.fetcher, model); + entry.batchId = batchId; + entry.status = status; + if (error) entry.error = error; + } catch (error) { + entry.status = "rejected"; + entry.error = error instanceof Error ? error.message : String(error); + } + })(), + ); + } + await Promise.allSettled(submissions); + await saveBroadsideState(broadsideDir, state); + + return { + runId, + outputDir: join(".codecarto", BROADSIDE_DIR, runId), + batches: run.batches, + estimatedTotalCost, + estimatedInputTokens, + estimatedOutputTokens, + pricing, + maxCost: limit > 0 ? limit : undefined, + modelInfo: { + contextLength: entry.contextLength, + maxCompletionTokens: entry.maxCompletionTokens, + supportsStructuredOutputs: entry.supportedParameters.length === 0 ? undefined : supportsStructuredOutputs, + expirationDate: entry.expirationDate ?? null, + }, + }; +} + +export type StoredLensResult = { + lensId: BroadsideLensId; + customId: string; + moduleName: string; + content: string; + raw: Record; + /** True when the content is not parseable JSON even after fence stripping — + * the telltale of an output cut off at max_tokens. */ + truncated: boolean; +}; + +function extractContent(result: Record): string | null { + const response = result.response as Record | undefined; + if (!response?.body) return null; + const body = response.body as Record; + const choices = body.choices as Array> | undefined; + const message = choices?.[0]?.message as Record | undefined; + return typeof message?.content === "string" ? message.content : null; +} + +/** + * Parse lens content as JSON, tolerating the markdown code fences some models + * wrap structured output in (the same tolerance OpenRouter's headless-agent + * scaffold ships for --output-schema). Returns null when the content is not + * JSON at all — which for a strict json_schema request means the output was + * truncated at max_tokens, not that the model chose prose. + */ +export function parseLensJson(content: string): unknown | null { + const trimmed = content.trim(); + const fenced = /^```(?:json)?\s*\n?([\s\S]*?)\n?```\s*$/.exec(trimmed); + const candidate = fenced ? fenced[1].trim() : trimmed; + if (!candidate.startsWith("{") && !candidate.startsWith("[")) return null; + try { + return JSON.parse(candidate); + } catch { + return null; + } +} + +export async function saveLensResults( + runDir: string, + lensId: BroadsideLensId, + batch: Record, +): Promise { + const results = Array.isArray(batch.results) ? (batch.results as Array>) : []; + const out: StoredLensResult[] = []; + for (const result of results) { + const customId = String(result.custom_id ?? "unknown"); + const content = extractContent(result); + if (content === null) { + if (result.error) { + await writeFile(join(runDir, `${sanitizeId(customId)}.error.json`), `${JSON.stringify(result.error, null, "\t")}\n`, "utf8"); + } + continue; + } + const parsed = parseLensJson(content); + const truncated = parsed === null; + if (parsed !== null) { + await writeFile(join(runDir, `${sanitizeId(customId)}.json`), `${JSON.stringify(parsed, null, "\t")}\n`, "utf8"); + } else { + // Save the raw bytes verbatim so nothing is lost, but name the + // gap: an unparseable strict-schema response is a truncation. + await writeFile(join(runDir, `${sanitizeId(customId)}.json`), `${content}\n`, "utf8"); + } + await writeFile(join(runDir, `${sanitizeId(customId)}.md`), renderFindingsMarkdown(content), "utf8"); + out.push({ + lensId, + customId, + moduleName: String(customId).replace(/^[a-z]+-/, ""), + content, + raw: result, + truncated, + }); + } + return out; +} + +// ---------- post-lens passes: synthesis + triage ---------- + +function buildSynthesisRequest(findingsText: string, truncatedNote: string, model: string): BatchRequest { + return { + custom_id: "synthesis", + body: { + model, + messages: [ + { + role: "system", + content: + "You are a technical editor synthesizing multiple analysis reports about a single " + + "codebase into one coherent summary. The reports come from different lenses — " + + "architecture, API surface, security review, defect scanning, convention extraction, " + + "and porting assessment. Cross-reference findings across lenses: if a security issue " + + "also appears as a defect, merge them. Produce a JSON object following the " + + "synthesis_report schema. Prioritize the most actionable findings. " + + "Be honest about gaps — if a lens found nothing, say 'no issues found' rather than " + + "inventing problems. These are scouting signals from a batch model, not verified " + + "claims; note that in the summary.", + }, + { + role: "user", + content: + "Synthesize these analysis reports into a single summary.\n\n" + + findingsText + + truncatedNote + + "\nReturn the synthesis_report JSON schema.", + }, + ], + response_format: { type: "json_schema", json_schema: SCHEMAS.synthesis }, + max_tokens: 12_000, + }, + }; +} + +function buildTriageRequest(findingsText: string, truncatedNote: string, model: string): BatchRequest { + return { + custom_id: "triage", + body: { + model, + messages: [ + { + role: "system", + content: + "You are a senior engineering lead turning unverified scouting findings into a " + + "prioritized work order. Given the findings below, produce a JSON object following " + + "the triage_report schema. Score every lead by impact and fix difficulty, assign a " + + "priority (P0 urgent/safety-critical to P3 nice-to-have), give a rough effort " + + "estimate, group the queue by module where sensible, and justify each call in the " + + "rationale. Merge duplicate leads instead of listing them twice. Drop leads that are " + + "too vague to act on and record each drop in omitted with the reason. These findings " + + "are UNVERIFIED scouting signals from a cheap batch model: the queue is a starting " + + "point for re-verification, not a commitment — say so in the summary, and never " + + "inflate a severity you cannot see evidence for.", + }, + { + role: "user", + content: + "Triage these scouting findings into a prioritized work order.\n\n" + + findingsText + + truncatedNote + + "\nReturn the triage_report JSON schema.", + }, + ], + response_format: { type: "json_schema", json_schema: SCHEMAS.triage }, + max_tokens: 10_000, + }, + }; +} + +function parseTriageItems(content: string): TriageItem[] { + try { + const parsed = JSON.parse(content) as Record; + const items = Array.isArray(parsed.items) ? (parsed.items as Array>) : []; + return items + .filter((item) => typeof item.title === "string") + .map((item) => ({ + title: String(item.title), + severity: String(item.severity ?? "unknown"), + module: String(item.module ?? "unknown"), + impact: (["high", "medium", "low"].includes(String(item.impact)) ? String(item.impact) : "medium") as TriageItem["impact"], + difficulty: (["high", "medium", "low"].includes(String(item.difficulty)) ? String(item.difficulty) : "medium") as TriageItem["difficulty"], + priority: String(item.priority ?? "?"), + effort_estimate: String(item.effort_estimate ?? ""), + rationale: String(item.rationale ?? ""), + })); + } catch { + return []; + } +} + +export async function runBroadsideCollect( + cwd: string, + apiKey: string, + opts: { + waitMs?: number; + includeSynthesis?: boolean; + includeTriage?: boolean; + onStatus?: (lensId: string, status: string, counts: Record) => void; + fetcher?: FetchLike; + } = {}, +): Promise { + const broadsideDir = broadsideDirFor(cwd); + const state = await loadBroadsideState(broadsideDir); + const run = state.runs[state.runs.length - 1]; + if (!run) { + throw new Error("No Broad-Side run recorded. Call codecarto_broadside with action 'submit' first."); + } + + const runDir = join(broadsideDir, run.outputDir); + await mkdir(runDir, { recursive: true }); + + const deadline = Date.now() + (opts.waitMs ?? BROADSIDE_DEFAULT_POLL_BUDGET_MS); + let totalCost = 0; + let resultCount = 0; + let truncatedCount = 0; + const lensOutcomes: BroadsideCollectResult["lensOutcomes"] = {}; + + const allLensResults: StoredLensResult[] = []; + + for (const lensId of run.lenses) { + const entry = run.batches[lensId]; + if (!entry || !entry.batchId) { + lensOutcomes[lensId] = { status: entry?.status ?? "failed", resultCount: 0 }; + continue; + } + if (["completed", "failed", "expired", "cancelled", "auth-failed", "skipped", "rejected"].includes(entry.status)) { + totalCost += entry.cost ?? 0; + resultCount += entry.resultCount ?? 0; + lensOutcomes[lensId] = { status: entry.status, cost: entry.cost, resultCount: entry.resultCount }; + continue; + } + + const batch = await pollBatchUntilTerminal(entry.batchId, apiKey, { + deadlineMs: Math.max(0, deadline - Date.now()), + onStatus: (status, counts) => opts.onStatus?.(lensId, status, counts), + fetcher: opts.fetcher, + }); + + const status = String(batch.status ?? "unknown"); + entry.status = status; + if (status === "completed") { + const usage = (batch.usage ?? {}) as Record; + const cost = typeof usage.cost === "number" ? usage.cost : undefined; + entry.cost = cost; + entry.completedAt = new Date().toISOString(); + const stored = await saveLensResults(runDir, lensId, batch); + entry.resultCount = stored.length; + const truncated = stored.filter((s) => s.truncated).length; + allLensResults.push(...stored); + resultCount += stored.length; + truncatedCount += truncated; + totalCost += cost ?? 0; + await writeFile( + join(runDir, `raw-${lensId}.json`), + `${JSON.stringify(batch, null, "\t")}\n`, + "utf8", + ); + lensOutcomes[lensId] = { status, cost: entry.cost, resultCount: entry.resultCount, truncated }; + } else if (batch.error) { + entry.error = batch.error; + lensOutcomes[lensId] = { status, cost: entry.cost, resultCount: entry.resultCount }; + } + await saveBroadsideState(broadsideDir, state); + } + + // Synthesis + triage: cross-lens post-passes, only after every lens batch + // is terminal. Triage turns the leads into a prioritized work order. + run.triage ??= { status: "pending" }; + let topFindings: BroadsideCollectResult["topFindings"] = []; + let topTriageItems: BroadsideCollectResult["topTriageItems"] = []; + const wantSynthesis = opts.includeSynthesis !== false; + const wantTriage = opts.includeTriage !== false; + if ((wantSynthesis || wantTriage) && allLensResults.length > 0) { + const allTerminal = run.lenses.every((lensId) => { + const entry = run.batches[lensId]; + return entry && ["completed", "failed", "expired", "cancelled", "auth-failed", "skipped", "rejected"].includes(entry.status); + }); + if (allTerminal && (run.synthesis.status === "pending" || run.triage.status === "pending")) { + const findingsText = allLensResults + .map((r) => `## ${r.lensId} — ${r.customId}\n\n${r.content}\n`) + .join("\n"); + const truncatedNote = + truncatedCount > 0 + ? `\n\nNOTE: ${truncatedCount} lens result(s) were truncated at the output token limit and are ` + + "not included above. Any gap they would have covered is unrepresented — do not treat " + + "silence on a module as a clean bill.\n" + : ""; + + // Both post-passes consume the same findings; they run as two + // batches (different response_format schemas cannot share one) + // submitted together and polled in turn. + const passes: Array<{ + kind: "synthesis" | "triage"; + request: BatchRequest; + entry: BroadsideSynthesisEntry; + }> = [ + ...(wantSynthesis && run.synthesis.status === "pending" + ? [{ + kind: "synthesis" as const, + request: buildSynthesisRequest(findingsText, truncatedNote, run.model), + entry: run.synthesis, + }] + : []), + ...(wantTriage && run.triage.status === "pending" + ? [{ + kind: "triage" as const, + request: buildTriageRequest(findingsText, truncatedNote, run.model), + entry: run.triage, + }] + : []), + ]; + + const submitted = new Map(); + await Promise.allSettled( + passes.map(async (pass) => { + pass.entry.status = "submitted"; + try { + const { batchId, error } = await submitBatch([pass.request], apiKey, opts.fetcher, run.model); + if (error) { + pass.entry.status = "failed"; + return; + } + pass.entry.batchId = batchId; + submitted.set(batchId, { batchId, pass }); + } catch { + pass.entry.status = "failed"; + } + }), + ); + await saveBroadsideState(broadsideDir, state); + + for (const { batchId, pass } of submitted.values()) { + const batch = await pollBatchUntilTerminal(batchId, apiKey, { + deadlineMs: BROADSIDE_DEFAULT_POLL_BUDGET_MS, + onStatus: (status, counts) => opts.onStatus?.(pass.kind, status, counts), + fetcher: opts.fetcher, + }); + if (batch.status === "completed") { + const usage = (batch.usage ?? {}) as Record; + const cost = typeof usage.cost === "number" ? usage.cost : undefined; + pass.entry.status = "completed"; + pass.entry.cost = cost; + totalCost += cost ?? 0; + const results = Array.isArray(batch.results) ? (batch.results as Array>) : []; + const content = results.length > 0 ? extractContent(results[0]) : null; + if (content !== null) { + await writeFile(join(runDir, `${pass.kind}.json`), `${content}\n`, "utf8"); + await writeFile(join(runDir, `${pass.kind}.md`), renderFindingsMarkdown(content), "utf8"); + if (pass.kind === "synthesis") { + topFindings = parseSynthesisTopFindings(content); + } else { + topTriageItems = parseTriageItems(content); + } + } + } else if (batch.error) { + pass.entry.status = "failed"; + } + await saveBroadsideState(broadsideDir, state); + } + } + } + + const terminal = run.lenses.every((lensId) => { + const entry = run.batches[lensId]; + return entry && ["completed", "failed", "expired", "cancelled", "auth-failed", "skipped", "rejected"].includes(entry.status); + }); + run.status = terminal ? (resultCount > 0 ? "completed" : "failed") : "partial"; + run.totalCost = totalCost; + await saveBroadsideState(broadsideDir, state); + + await writeFile( + join(runDir, "run-meta.json"), + `${JSON.stringify( + { + experimental: true, + method: "Broad-Side (OpenRouter Batch API)", + model: run.model, + pricing: run.pricing, + max_cost: run.maxCost, + run_id: run.id, + created_at: run.createdAt, + status: run.status, + total_cost: totalCost, + result_count: resultCount, + truncated_count: truncatedCount, + synthesis: run.synthesis, + triage: run.triage, + lenses: run.lenses, + disclaimer: + "Findings are unverified scouting signals from a batch model, not validated claims. " + + "Re-verify every file:line lead with the interactive pipeline or by hand.", + }, + null, + "\t", + )}\n`, + "utf8", + ); + + return { + runId: run.id, + status: run.status, + totalCost, + resultCount, + truncatedCount, + lensOutcomes, + synthesis: run.synthesis, + triage: run.triage, + topFindings, + topTriageItems, + }; +} + +export async function runBroadsideStatus(cwd: string): Promise<{ state: BroadsideStateFile }> { + const broadsideDir = broadsideDirFor(cwd); + const state = await loadBroadsideState(broadsideDir); + return { state }; +} + +// ---------- rendering ---------- + +export function renderFindingsMarkdown(content: string): string { + const parsed = parseLensJson(content); + if (parsed === null) return content; + return formatAsMarkdown(parsed); +} + +function formatAsMarkdown(value: unknown, depth = 0): string { + const indent = "\t".repeat(depth); + if (Array.isArray(value)) { + const lines: string[] = []; + for (let i = 0; i < value.length; i++) { + const item = value[i] as Record; + if (item && typeof item === "object") { + const title = (item.title ?? item.name ?? item.module ?? item.area ?? item.platform ?? "") as string; + lines.push(`${indent}${i + 1}. ${title}`); + lines.push(formatAsMarkdown(item, depth + 1)); + } else { + lines.push(`${indent}- ${String(item)}`); + } + } + return lines.join("\n"); + } + if (value && typeof value === "object") { + const lines: string[] = []; + for (const [key, entryValue] of Object.entries(value as Record)) { + if (entryValue && typeof entryValue === "object") { + lines.push(`${indent}**${key}**:`); + lines.push(formatAsMarkdown(entryValue, depth + 1)); + } else { + lines.push(`${indent}- **${key}**: ${String(entryValue)}`); + } + } + return lines.join("\n"); + } + return `${indent}${String(value)}`; +} + +function parseSynthesisTopFindings( + content: string, +): BroadsideCollectResult["topFindings"] { + try { + const parsed = JSON.parse(content) as Record; + const findings = Array.isArray(parsed.top_findings) + ? (parsed.top_findings as Array>) + : []; + return findings + .filter((f) => typeof f.title === "string") + .map((f) => ({ + title: String(f.title), + severity: String(f.severity ?? "unknown"), + sourceLens: String(f.source_lens ?? "unknown"), + summary: String(f.summary ?? ""), + })); + } catch { + return []; + } +} + +// ---------- formatting helpers for tool output ---------- + +export function estimateSubmitText(result: BroadsideSubmitResult, lenses: LensDefinition[]): string { + const lines = [ + `Broad-Side submitted ${result.batches ? Object.keys(result.batches).length : 0} batch(es).`, + ]; + for (const lens of lenses) { + const entry = result.batches[lens.id]; + if (!entry) continue; + const status = entry.batchId ? `batch ${entry.batchId}` : entry.status; + lines.push(` ${lens.name}: ${status} (${entry.requests} request(s), ~$${entry.estimatedCost.toFixed(4)})`); + } + lines.push( + `Estimated total: ~$${result.estimatedTotalCost.toFixed(4)}`, + `Pricing: $${result.pricing.inputPerM.toFixed(4)}/M in, $${result.pricing.outputPerM.toFixed(4)}/M out (${result.pricing.source})`, + ); + if (result.modelInfo.contextLength) { + lines.push(`Model: ${result.modelInfo.contextLength.toLocaleString()} context, ${result.modelInfo.maxCompletionTokens?.toLocaleString() ?? "?"} max output`); + } + if (result.modelInfo.supportsStructuredOutputs === false) { + lines.push("Warning: model does not advertise structured-output support; lens JSON may be unreliable."); + } + if (result.modelInfo.expirationDate) { + lines.push(`Warning: this model is deprecated (expires ${result.modelInfo.expirationDate}).`); + } + if (result.maxCost) { + lines.push(`Run limit: $${result.maxCost.toFixed(2)} (enforced on estimate; pass force to override)`); + } + lines.push( + `Results will land in ${result.outputDir}/`, + "Call codecarto_broadside with action 'collect' once batches finish, or pass wait_seconds on submit to block.", + "Disclaimer: Broad-Side findings are unverified scouting signals from a batch model, not validated claims.", + ); + return lines.join("\n"); +} + +export function modelsText( + entries: CatalogEntry[], + opts: { benchmarks: CodingBenchmarks | null; defaultModel: string }, +): string { + const lines = [ + `Batch models on OpenRouter (${entries.length}, cheapest first).`, + "", + "id | $/M in | $/M out | ctx | max out | structured | coding idx", + ]; + for (const entry of entries) { + const bench = opts.benchmarks?.byBaseSlug[baseSlug(entry.id)]; + const structured = entry.supportedParameters.length === 0 + ? "?" + : entry.supportedParameters.some((p) => ["structured_outputs", "json_schema", "response_format", "structuredoutputs"].includes(p.toLowerCase())) + ? "yes" + : "no"; + const coding = bench?.codingIndex !== undefined ? bench.codingIndex.toFixed(1) : "-"; + const ctx = entry.contextLength + ? entry.contextLength >= 1_000_000 + ? `${(entry.contextLength / 1_000_000).toFixed(1)}M` + : `${(entry.contextLength / 1024).toFixed(0)}k` + : "?"; + const out = entry.maxCompletionTokens ? `${(entry.maxCompletionTokens / 1024).toFixed(0)}k` : "?"; + const tag = entry.id === opts.defaultModel ? " (default)" : ""; + const exp = entry.expirationDate ? " [deprecated]" : ""; + lines.push( + `${entry.id}${tag}${exp} | ${entry.inputPerM.toFixed(3)} | ${entry.outputPerM.toFixed(3)} | ${ctx} | ${out} | ${structured} | ${coding}`, + ); + } + if (opts.benchmarks?.meta.as_of) { + lines.push("", `Benchmarks: Artificial Analysis coding index (as of ${String(opts.benchmarks.meta.as_of)}).`); + } + lines.push("", "Set the batch model in .codecarto/broadside/config.yaml (model key). Higher coding index ≠ better scout: precision, context, and structured-output support matter most here."); + return lines.join("\n"); +} + +export function collectResultText(result: BroadsideCollectResult): string { + const lines = [ + `Broad-Side run ${result.runId}: ${result.status}`, + ` Results: ${result.resultCount} | Total cost: $${result.totalCost.toFixed(6)}`, + ]; + for (const lensId of BROADSIDE_LENS_IDS) { + const outcome = result.lensOutcomes[lensId]; + if (!outcome) continue; + const truncation = outcome.truncated ? `, ${outcome.truncated} truncated` : ""; + lines.push( + ` ${lensId}: ${outcome.status}` + + (outcome.cost !== undefined ? `, $${outcome.cost.toFixed(6)}` : "") + + (outcome.resultCount !== undefined ? `, ${outcome.resultCount} result(s)` : "") + + truncation, + ); + } + if (result.truncatedCount > 0) { + lines.push( + ` ⚠ ${result.truncatedCount} result(s) truncated at the output limit — their modules are unscouted, not clean.`, + ); + } + if (result.synthesis.status === "completed") { + lines.push(` synthesis: completed, $${(result.synthesis.cost ?? 0).toFixed(6)}`); + if (result.topFindings.length > 0) { + lines.push("", "Top findings (unverified leads):"); + for (const f of result.topFindings.slice(0, 10)) { + lines.push(` [${f.severity}] ${f.title}`); + } + } + } + if (result.triage.status === "completed") { + lines.push(` triage: completed, $${(result.triage.cost ?? 0).toFixed(6)}`); + if (result.topTriageItems.length > 0) { + lines.push("", "Triage — prioritized work order (re-verify before acting):"); + for (const item of result.topTriageItems.slice(0, 10)) { + lines.push( + ` ${item.priority} [${item.severity}/${item.module}] ${item.title}` + + (item.effort_estimate ? ` (${item.effort_estimate})` : ""), + ); + } + } + } else if (result.triage.status === "failed") { + lines.push(" triage: failed"); + } + lines.push("", "Disclaimer: Broad-Side findings are unverified scouting signals from a batch model, not validated claims."); + return lines.join("\n"); +} + +export function statusText(state: BroadsideStateFile): string { + if (state.runs.length === 0) { + return "No Broad-Side runs recorded. Call codecarto_broadside with action 'submit' first."; + } + const lines: string[] = []; + for (const run of [...state.runs].reverse().slice(0, 3)) { + lines.push(`Run ${run.id} — ${run.status}`); + for (const lensId of BROADSIDE_LENS_IDS) { + const entry = run.batches[lensId]; + if (!entry) continue; + lines.push(` ${lensId}: ${entry.status}${entry.batchId ? ` (${entry.batchId})` : ""}${entry.cost !== undefined ? `, $${entry.cost.toFixed(6)}` : ""}`); + } + lines.push(` synthesis: ${run.synthesis.status}`); + lines.push(` triage: ${run.triage?.status ?? "pending"}`); + if (run.totalCost !== undefined) lines.push(` total cost: $${run.totalCost.toFixed(6)}`); + } + return lines.join("\n"); +} diff --git a/core/index.ts b/core/index.ts index 9ee3dcc..b5f4ed3 100644 --- a/core/index.ts +++ b/core/index.ts @@ -17,3 +17,4 @@ export * from "./guide.ts"; export * from "./dashboard.ts"; export * from "./library.ts"; export * from "./synthesis.ts"; +export * from "./broadside.ts"; diff --git a/core/workspace.ts b/core/workspace.ts index 39a80bd..9f31fe1 100644 --- a/core/workspace.ts +++ b/core/workspace.ts @@ -141,7 +141,9 @@ export async function seedOrchestratorFiles(workspaceDir: string): Promise entry !== BROADSIDE_DIR); + broadsideOnly = entries.length === 0 && (await pathExists(join(targetWorkspaceDir, BROADSIDE_DIR))); + } + + if (targetExists && !sameWorkspace && !broadsideOnly) { if (!args.force) { throw new McpError( ErrorCode.InvalidRequest, @@ -174,6 +198,10 @@ export async function handleInit(args: { cwd: string; pipeline?: string; force?: if (!(await pathExists(targetWorkspaceDir))) { await mkdir(cwd, { recursive: true }); await cp(packagedWorkspaceDir, targetWorkspaceDir, { recursive: true }); + } else if (broadsideOnly) { + // Merge the template into the scout-only .codecarto/, preserving the + // broadside state and results already on disk. + await cp(packagedWorkspaceDir, targetWorkspaceDir, { recursive: true }); } const statusPath = join(targetWorkspaceDir, "workflow", "status.yaml"); @@ -950,6 +978,127 @@ export async function handleAmend(args: { cwd: string; name: string }) { }); } +// ---------- broadside (batch reconnaissance) ---------- + +function resolveBroadsideApiKey(explicit: string | undefined, config: { apiKey: string }): string { + if (explicit && explicit.trim()) return explicit.trim(); + const fromEnv = process.env.OPENROUTER_API_KEY?.trim(); + if (fromEnv) return fromEnv; + if (config.apiKey) return config.apiKey; + throw new McpError( + ErrorCode.InvalidParams, + "No OpenRouter API key found. Pass api_key, set the OPENROUTER_API_KEY environment variable, or add api_key to .codecarto/broadside/config.yaml.", + ); +} + +export async function handleBroadside(args: { + cwd: string; + action: "submit" | "collect" | "status" | "models"; + lenses?: string[]; + api_key?: string; + wait_seconds?: number; + include_synthesis?: boolean; + include_triage?: boolean; + max_cost?: number; + force?: boolean; + include_benchmarks?: boolean; +}) { + const cwd = await validateCwd(args.cwd); + const action = args.action ?? "submit"; + if (!["submit", "collect", "status", "models"].includes(action)) { + throw new McpError(ErrorCode.InvalidParams, `Unknown action: ${action}. Valid actions: submit, collect, status, models.`); + } + + const config = await loadBroadsideConfig(broadsideDirFor(cwd)); + + if (action === "status") { + const { state } = await runBroadsideStatus(cwd); + return textResult(statusText(state), { state }); + } + + const apiKey = resolveBroadsideApiKey(args.api_key, config); + const waitMs = typeof args.wait_seconds === "number" && args.wait_seconds > 0 ? args.wait_seconds * 1000 : undefined; + + if (action === "models") { + const { entries, benchmarks } = await listBatchModels(broadsideDirFor(cwd), config, apiKey, { + includeBenchmarks: args.include_benchmarks === true, + }).catch((error) => { + throw new McpError(ErrorCode.InvalidRequest, error instanceof Error ? error.message : String(error)); + }); + return textResult(modelsText(entries, { benchmarks, defaultModel: config.model }), { + models: entries, + defaultModel: config.model, + benchmarkMeta: benchmarks?.meta ?? null, + }); + } + + if (action === "submit") { + let lenses: BroadsideLensId[]; + if (args.lenses && args.lenses.length > 0) { + const unknown = args.lenses.filter((l) => !BROADSIDE_LENS_IDS.includes(l as BroadsideLensId)); + if (unknown.length > 0) { + throw new McpError(ErrorCode.InvalidParams, `Unknown lens(es): ${unknown.join(", ")}. Valid: ${BROADSIDE_LENS_IDS.join(", ")}`); + } + lenses = args.lenses as BroadsideLensId[]; + } else { + lenses = config.defaultLenses; + } + + const maxCost = typeof args.max_cost === "number" && args.max_cost > 0 ? args.max_cost : config.maxCost; + + const result = await runBroadsideSubmit(cwd, apiKey, { + lenses, + model: config.model, + maxCost, + force: args.force === true, + }).catch((error) => { + throw new McpError(ErrorCode.InvalidRequest, error instanceof Error ? error.message : String(error)); + }); + + const lines = [estimateSubmitText(result, lenses.map(getLens))]; + if (waitMs) { + lines.push("", "Waiting for batches to complete..."); + const collect = await runBroadsideCollect(cwd, apiKey, { + waitMs, + includeSynthesis: args.include_synthesis !== false, + includeTriage: args.include_triage !== false, + onStatus: (lensId, status, counts) => + lines.push(` ${lensId}: ${status} (${counts.completed ?? 0}/${counts.total ?? "?"})`), + }); + lines.push("", collectResultText(collect)); + } + return textResult(lines.join("\n"), { + runId: result.runId, + outputDir: result.outputDir, + batches: result.batches, + estimatedTotalCost: result.estimatedTotalCost, + pricing: result.pricing, + maxCost: result.maxCost, + }); + } + + // action === "collect" + const collect = await runBroadsideCollect(cwd, apiKey, { + waitMs, + includeSynthesis: args.include_synthesis !== false, + includeTriage: args.include_triage !== false, + }).catch((error) => { + throw new McpError(ErrorCode.InvalidRequest, error instanceof Error ? error.message : String(error)); + }); + return textResult(collectResultText(collect), { + runId: collect.runId, + status: collect.status, + totalCost: collect.totalCost, + resultCount: collect.resultCount, + truncatedCount: collect.truncatedCount, + lensOutcomes: collect.lensOutcomes, + synthesis: collect.synthesis, + triage: collect.triage, + topFindings: collect.topFindings, + topTriageItems: collect.topTriageItems, + }); +} + // ---------- tool registry ---------- const TOOLS = [ @@ -1238,6 +1387,58 @@ const TOOLS = [ required: ["cwd"], }, }, + { + name: "codecarto_broadside", + description: + "Broad-Side: fire a cheap batch reconnaissance scan at a repository via the OpenRouter Batch API. Six lenses (architecture, api, security, defect, conventions, porting) run as asynchronous single-turn prompts with structured JSON schemas; results land in .codecarto/broadside// as JSON plus markdown, with an optional cross-lens synthesis report. Works on any git repository — no CodeCartographer workspace required. Requires an OpenRouter API key (api_key param, OPENROUTER_API_KEY env var, or .codecarto/broadside/config.yaml). Findings are unverified scouting signals from a batch model, not validated claims — they tell the interactive pipeline where to look. Actions: submit (fire batches, returns batch ids and cost estimate), collect (poll to completion, save results, optionally synthesize), status (show recorded runs), models (list batch-capable models with pricing, context, output caps, structured-output support, and optional coding benchmarks).", + inputSchema: { + type: "object", + properties: { + cwd: { type: "string", description: "Absolute path to the target repository." }, + action: { + type: "string", + enum: ["submit", "collect", "status", "models"], + description: "submit fires all lens batches and returns batch ids; collect polls submitted batches, saves results, and optionally runs the synthesis pass; status shows recorded runs; models lists batch-capable models with pricing and capabilities.", + }, + lenses: { + type: "array", + items: { type: "string", enum: [...BROADSIDE_LENS_IDS] }, + description: "Lenses to run (submit only). Defaults to all six.", + }, + api_key: { + type: "string", + description: "OpenRouter API key. Prefer the OPENROUTER_API_KEY environment variable or .codecarto/broadside/config.yaml.", + }, + wait_seconds: { + type: "number", + description: "For submit: after submitting, poll up to this many seconds before returning. For collect: poll up to this many seconds before returning with partial state.", + }, + include_synthesis: { + type: "boolean", + description: "Run the cross-lens synthesis pass once all lens batches complete (default true).", + }, + include_triage: { + type: "boolean", + description: + "Run the triage pass once all lens batches complete: turns the findings into a prioritized work order (impact × difficulty, P0-P3, effort estimates). Default true.", + }, + max_cost: { + type: "number", + description: + "Approximate run expense limit in USD. The submit action estimates the run cost from slice sizes and the configured model's per-token pricing (live OpenRouter lookup, cached 24h) and refuses to submit when the estimate exceeds the limit unless force is true. Falls back to max_cost in .codecarto/broadside/config.yaml.", + }, + force: { + type: "boolean", + description: "Submit even when the cost estimate exceeds max_cost (default false).", + }, + include_benchmarks: { + type: "boolean", + description: "For action 'models': annotate each model with its Artificial Analysis coding index (extra API call; default false).", + }, + }, + required: ["cwd", "action"], + }, + }, ] as const; const HANDLERS: Record Promise> = { @@ -1262,6 +1463,7 @@ const HANDLERS: Record Promise> = { codecarto_dashboard: handleDashboard, codecarto_list_skills: handleListSkills, codecarto_guide: handleGuide, + codecarto_broadside: handleBroadside, }; export async function handleGuide(args: { topic?: string }) { diff --git a/scripts/smoke-broadside.mjs b/scripts/smoke-broadside.mjs new file mode 100644 index 0000000..c2495ea --- /dev/null +++ b/scripts/smoke-broadside.mjs @@ -0,0 +1,52 @@ +// Broad-Side smoke test: drives the MCP handler through a real submit → +// collect → synthesis cycle against a live target repository using the +// OpenRouter Batch API. +// +// This spends real money (~$0.03 for the default two lenses). It is therefore +// opt-in: it skips cleanly unless OPENROUTER_API_KEY is set and a target is +// passed on the command line. +// +// OPENROUTER_API_KEY=sk-or-... node scripts/smoke-broadside.mjs /path/to/repo + +import assert from "node:assert/strict"; +import { dirname, join, resolve } from "node:path"; +import { fileURLToPath, pathToFileURL } from "node:url"; + +const REPO_ROOT = resolve(dirname(fileURLToPath(import.meta.url)), ".."); +const target = process.argv[2]; + +if (!process.env.OPENROUTER_API_KEY) { + console.log("SKIP: OPENROUTER_API_KEY not set — broadside smoke spends real money and is opt-in."); + process.exit(0); +} +if (!target) { + console.error("Usage: OPENROUTER_API_KEY=sk-or-... node scripts/smoke-broadside.mjs /path/to/repo"); + process.exit(1); +} + +const { handleBroadside } = await import(pathToFileURL(join(REPO_ROOT, "mcp-server/server.ts")).href); + +async function step(label, fn) { + console.log(`\n== ${label} ==`); + const result = await fn(); + console.log(result.content[0].text); + return result; +} + +const submit = await step("submit (architecture + api)", () => + handleBroadside({ cwd: target, action: "submit", lenses: ["architecture", "api"], api_key: process.env.OPENROUTER_API_KEY }), +); +assert.equal(submit.structuredContent.runId.length > 0, true, "submit must return a run id"); + +const status = await step("status", () => handleBroadside({ cwd: target, action: "status" })); +assert.match(status.content[0].text, /architecture/); + +const collect = await step("collect (polls to completion, synthesizes)", () => + handleBroadside({ cwd: target, action: "collect", api_key: process.env.OPENROUTER_API_KEY }), +); +assert.equal(collect.structuredContent.status, "completed", "collect must reach completed"); +assert.ok(collect.structuredContent.resultCount > 0, "collect must save at least one result"); +assert.equal(collect.structuredContent.synthesis.status, "completed", "synthesis must complete"); +assert.ok(collect.structuredContent.topFindings.length > 0, "synthesis must produce top findings"); + +console.log("\nSMOKE PASSED"); diff --git a/scripts/smoke-mcp.mjs b/scripts/smoke-mcp.mjs index 391a996..9b38c14 100644 --- a/scripts/smoke-mcp.mjs +++ b/scripts/smoke-mcp.mjs @@ -87,6 +87,7 @@ async function setupFixture() { const EXPECTED_TOOLS = [ "codecarto_amend", + "codecarto_broadside", "codecarto_complete", "codecarto_config", "codecarto_dashboard", diff --git a/tests/broadside.test.mjs b/tests/broadside.test.mjs new file mode 100644 index 0000000..62e12a0 --- /dev/null +++ b/tests/broadside.test.mjs @@ -0,0 +1,840 @@ +// Broad-Side unit tests. No network: the batch client is exercised through an +// injected fake fetcher, and file collection runs against temp fixtures built +// in-memory. The real OpenRouter API is covered by a manual smoke path, not +// this suite. + +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { mkdtemp, mkdir, readFile, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { dirname, join, resolve } from "node:path"; +import { fileURLToPath, pathToFileURL } from "node:url"; + +const REPO_ROOT = resolve(dirname(fileURLToPath(import.meta.url)), ".."); +const core = await import(pathToFileURL(join(REPO_ROOT, "core/index.ts")).href); + +const { + BROADSIDE_LENS_IDS, + BROADSIDE_MODEL, + buildBatchRequest, + builtInPricing, + collectRepoInfo, + defaultBroadsideState, + estimateCost, + fetchCodingBenchmarks, + gatherSlices, + getLens, + listBatchModels, + loadBroadsideConfig, + loadBroadsideState, + listLenses, + modelsText, + renderFindingsMarkdown, + resolveModelPricing, + runBroadsideCollect, + runBroadsideStatus, + runBroadsideSubmit, + saveBroadsideState, + submitBatch, +} = core; + +// ---------- lens registry ---------- + +test("the registry carries six lenses with unique schema names", () => { + const lenses = listLenses(); + assert.equal(lenses.length, 6); + assert.deepEqual( + lenses.map((l) => l.id).sort(), + [...BROADSIDE_LENS_IDS].sort(), + ); + const schemaNames = new Set(); + for (const lens of lenses) { + schemaNames.add(lens.schemaName); + assert.ok(lens.name.length > 5, "every lens needs a human-readable name"); + assert.ok(lens.systemPrompt({ language: "go" }).length > 100); + } + assert.equal(schemaNames.size, 6, "each lens must declare its own schema"); +}); + +// ---------- file collection ---------- + +async function makeFixture() { + const dir = await mkdtemp(join(tmpdir(), "broadside-fixture-")); + await mkdir(join(dir, "server"), { recursive: true }); + await mkdir(join(dir, "model", "deep"), { recursive: true }); + await writeFile(join(dir, "go.mod"), "module example.com/fixture\n\ngo 1.26.0\n"); + await writeFile(join(dir, "main.go"), "package main\n\nfunc main() {}\n"); + await writeFile(join(dir, "server", "routes.go"), "package server\n\n// GET /api/version\nfunc routes() {}\n"); + await writeFile(join(dir, "server", "auth.go"), "package server\n\nfunc auth() {}\n"); + await writeFile(join(dir, "model", "core.go"), "package model\n\nfunc core() {}\n"); + await writeFile(join(dir, "model", "deep", "nested.go"), "package deep\n\nfunc nested() {}\n"); + await writeFile(join(dir, "README.md"), "# Fixture\n"); + return dir; +} + +test("collectRepoInfo detects Go and gathers manifest, tree, and counts", async () => { + const dir = await makeFixture(); + try { + const info = await collectRepoInfo(dir); + assert.equal(info.language, "go"); + assert.ok(info.manifest, "go.mod must be found"); + assert.equal(info.manifest.path, "go.mod"); + assert.match(info.fileTree, /server\/routes\.go/); + assert.ok(info.fileCounts[".go"] >= 5, "five go files expected"); + assert.equal(info.sourceGlob, "**/*.go"); + } finally { + await rm(dir, { recursive: true, force: true }); + } +}); + +test("directory slicing puts nested files under their top-level module", async () => { + const dir = await makeFixture(); + try { + const info = await collectRepoInfo(dir); + const lens = getLens("defect"); + const slices = await gatherSlices(dir, lens, info); + const byModule = Object.fromEntries(slices.map((s) => [s.moduleName, s])); + assert.ok(byModule.server, "server/ must be its own slice"); + assert.ok(byModule.model, "model/ must be its own slice"); + assert.ok(byModule.root, "top-level main.go must land in the root slice"); + assert.match(byModule.server.content, /server\/routes\.go/); + assert.match(byModule.model.content, /model\/deep\/nested\.go/, "nested files belong to the top-level module"); + assert.match(byModule.root.content, /main\.go/); + for (const slice of slices) { + assert.ok(slice.chars <= lens.maxChars); + } + } finally { + await rm(dir, { recursive: true, force: true }); + } +}); + +test("oversized modules split into multiple slices instead of truncating", async () => { + const dir = await mkdtemp(join(tmpdir(), "broadside-split-")); + try { + await mkdir(join(dir, "big")); + await writeFile(join(dir, "go.mod"), "module x\n"); + for (let i = 0; i < 80; i++) { + await writeFile(join(dir, "big", `file${i}.go`), "package big\n" + `// ${"x".repeat(2000)}\n`); + } + const info = await collectRepoInfo(dir); + const lens = getLens("defect"); + const slices = await gatherSlices(dir, lens, info); + const big = slices.filter((s) => s.moduleName === "big"); + assert.ok(big.length >= 2, `expected split slices, got ${big.length}`); + const totalChars = big.reduce((sum, s) => sum + s.chars, 0); + assert.ok(totalChars > 20 * 2000, "split slices must carry all content, not drop it"); + } finally { + await rm(dir, { recursive: true, force: true }); + } +}); + +test("architecture lens needs no file slurping and builds from repo info", async () => { + const dir = await makeFixture(); + try { + const info = await collectRepoInfo(dir); + const lens = getLens("architecture"); + const slices = await gatherSlices(dir, lens, info); + assert.equal(slices.length, 1); + const request = buildBatchRequest(lens, info, slices[0], 0, 1); + assert.equal(request.custom_id, "architecture-root"); + assert.match(request.body.messages[1].content, /module example\.com\/fixture/); + assert.equal(request.body.model, BROADSIDE_MODEL); + assert.equal(request.body.response_format.json_schema.name, "architecture_report"); + } finally { + await rm(dir, { recursive: true, force: true }); + } +}); + +test("api and security lenses skip test files; conventions keeps them", async () => { + const dir = await makeFixture(); + try { + await writeFile(join(dir, "server", "routes_test.go"), "package server\n"); + const info = await collectRepoInfo(dir); + const apiSlices = await gatherSlices(dir, getLens("api"), info); + const apiText = apiSlices.map((s) => s.content).join("\n"); + assert.ok(!apiText.includes("routes_test.go"), "api lens must skip test files"); + const securitySlices = await gatherSlices(dir, getLens("security"), info); + const securityText = securitySlices.map((s) => s.content).join("\n"); + assert.ok(!securityText.includes("routes_test.go"), "security lens must skip test files"); + const conventionsSlices = await gatherSlices(dir, getLens("conventions"), info); + const convText = conventionsSlices.map((s) => s.content).join("\n"); + assert.ok(convText.includes("routes_test.go"), "conventions lens must catalog test files"); + } finally { + await rm(dir, { recursive: true, force: true }); + } +}); + +test("submit marks a lens with no matching files as skipped, never submitting an empty batch", async () => { + const dir = await mkdtemp(join(tmpdir(), "broadside-empty-")); + try { + await writeFile(join(dir, "go.mod"), "module x\n"); + await writeFile(join(dir, "main.go"), "package main\n"); + const posted = []; + const fetcher = async (url, init) => { + if (init.method === "POST") { + posted.push(JSON.parse(init.body)); + return fakeResponse(202, { id: `batch-${posted.length}`, status: "validating" }); + } + return fakeResponse(200, { id: "x", status: "completed" }); + }; + // api lens globs target server/ and api/ — neither exists here. + const result = await runBroadsideSubmit(dir, "sk-fake", { lenses: ["architecture", "api"], fetcher }); + assert.equal(result.batches.api.status, "skipped"); + assert.equal(posted.length, 1, "only the architecture batch may be submitted"); + assert.ok(posted[0].requests.length > 0, "submitted batches must be non-empty"); + } finally { + await rm(dir, { recursive: true, force: true }); + } +}); + +test("a network throw during submission marks the entry rejected, not stuck submitting", async () => { + const dir = await mkdtemp(join(tmpdir(), "broadside-throw-")); + try { + await writeFile(join(dir, "go.mod"), "module x\n"); + await writeFile(join(dir, "main.go"), "package main\n"); + const fetcher = async () => { + throw new Error("ECONNREFUSED"); + }; + const result = await runBroadsideSubmit(dir, "sk-fake", { lenses: ["architecture"], fetcher }); + assert.equal(result.batches.architecture.status, "rejected"); + assert.ok(result.batches.architecture.error, "the failure reason must be recorded"); + } finally { + await rm(dir, { recursive: true, force: true }); + } +}); + +// ---------- cost estimation ---------- + +test("estimateCost matches the documented per-token pricing", () => { + const lens = getLens("defect"); + const slices = [ + { moduleName: "a", content: "x".repeat(4000), fileCount: 1, chars: 4000 }, + { moduleName: "b", content: "y".repeat(4000), fileCount: 1, chars: 4000 }, + ]; + const pricing = { inputPerM: 0.1875, outputPerM: 0.9375, source: "built-in" }; + const { inputTokens, outputTokens, cost } = estimateCost(lens, slices, pricing); + assert.equal(inputTokens, 2000); // 8000 chars / 4 + assert.equal(outputTokens, 4500); // maxTokens 6000 * 0.75 + const expected = (2000 / 1e6) * 0.1875 + (4500 / 1e6) * 0.9375; + assert.ok(Math.abs(cost - expected) < 1e-12); +}); + +test("estimateCost scales with the pricing table, not the default model", () => { + const lens = getLens("defect"); + const slices = [{ moduleName: "a", content: "x".repeat(40000), fileCount: 1, chars: 40000 }]; + const cheap = estimateCost(lens, slices, { inputPerM: 0.1875, outputPerM: 0.9375, source: "built-in" }); + const expensive = estimateCost(lens, slices, { inputPerM: 3.75, outputPerM: 84, source: "live" }); + assert.ok(expensive.cost > cheap.cost * 20, "an $84/M output model must estimate far higher"); +}); + +// ---------- state & config ---------- + +test("state round-trips and defaults to an empty run list", async () => { + const dir = await mkdtemp(join(tmpdir(), "broadside-state-")); + try { + const state = await loadBroadsideState(dir); + assert.deepEqual(state, defaultBroadsideState()); + state.runs.push({ + id: "run-1", + createdAt: "2026-08-23T00:00:00Z", + model: BROADSIDE_MODEL, + lenses: ["architecture"], + status: "in-flight", + outputDir: "run-1", + batches: {}, + synthesis: { status: "pending" }, + }); + await saveBroadsideState(dir, state); + const reloaded = await loadBroadsideState(dir); + assert.equal(reloaded.runs.length, 1); + assert.equal(reloaded.runs[0].id, "run-1"); + } finally { + await rm(dir, { recursive: true, force: true }); + } +}); + +test("corrupt state files degrade to defaults, not crashes", async () => { + const dir = await mkdtemp(join(tmpdir(), "broadside-corrupt-")); + try { + await writeFile(join(dir, "state.json"), "{ this is not json"); + const state = await loadBroadsideState(dir); + assert.equal(state.runs.length, 0); + } finally { + await rm(dir, { recursive: true, force: true }); + } +}); + +test("config falls back to defaults and honors overrides", async () => { + const dir = await mkdtemp(join(tmpdir(), "broadside-config-")); + try { + const defaults = await loadBroadsideConfig(dir); + assert.equal(defaults.model, BROADSIDE_MODEL); + assert.equal(defaults.apiKey, ""); + assert.equal(defaults.defaultLenses.length, 6); + + await writeFile( + join(dir, "config.yaml"), + "model: custom/model\napi_key: sk-test\ndefault_lenses:\n - architecture\n - security\n", + ); + const overridden = await loadBroadsideConfig(dir); + assert.equal(overridden.model, "custom/model"); + assert.equal(overridden.apiKey, "sk-test"); + assert.deepEqual(overridden.defaultLenses, ["architecture", "security"]); + + await writeFile(join(dir, "config.yaml"), "default_lenses:\n - bogus\n - architecture\n"); + const filtered = await loadBroadsideConfig(dir); + assert.deepEqual(filtered.defaultLenses, ["architecture"], "unknown lens ids must be dropped"); + } finally { + await rm(dir, { recursive: true, force: true }); + } +}); + +// ---------- pricing resolution & expense limits ---------- + +function modelsCatalog(body) { + return { data: body }; +} + +test("builtInPricing covers only the default model", () => { + assert.ok(core.builtInPricing(BROADSIDE_MODEL)); + assert.equal(core.builtInPricing("openai/gpt-5.2-pro:batch"), null); +}); + +test("resolveModelPricing prefers config overrides over everything", async () => { + const dir = await mkdtemp(join(tmpdir(), "broadside-pricing-")); + try { + const config = { + model: "custom/model", + apiKey: "", + defaultLenses: ["architecture"], + maxCost: 0, + pricing: { inputPerM: 1.5, outputPerM: 42 }, + }; + const fetcher = async () => { + throw new Error("the network must not be touched when config pricing exists"); + }; + const pricing = await resolveModelPricing(dir, config, "custom/model", "sk-fake", fetcher); + assert.deepEqual(pricing, { inputPerM: 1.5, outputPerM: 42, source: "config" }); + } finally { + await rm(dir, { recursive: true, force: true }); + } +}); + +test("resolveModelPricing falls back to built-in for the default model without network", async () => { + const dir = await mkdtemp(join(tmpdir(), "broadside-pricing-")); + try { + const config = { model: BROADSIDE_MODEL, apiKey: "", defaultLenses: ["architecture"], maxCost: 0, pricing: null }; + const fetcher = async () => { + throw new Error("the default model needs no lookup"); + }; + const pricing = await resolveModelPricing(dir, config, BROADSIDE_MODEL, "sk-fake", fetcher); + assert.equal(pricing.source, "built-in"); + assert.equal(pricing.inputPerM, 0.1875); + } finally { + await rm(dir, { recursive: true, force: true }); + } +}); + +test("resolveModelPricing looks up unknown models live and caches the result", async () => { + const dir = await mkdtemp(join(tmpdir(), "broadside-pricing-")); + try { + const config = { model: "openai/gpt-5.2-pro:batch", apiKey: "", defaultLenses: ["architecture"], maxCost: 0, pricing: null }; + let fetches = 0; + const fetcher = async () => { + fetches += 1; + return fakeResponse( + 200, + modelsCatalog([{ id: "openai/gpt-5.2-pro:batch", pricing: { prompt: "0.00000375", completion: "0.000084" } }]), + ); + }; + const pricing = await resolveModelPricing(dir, config, "openai/gpt-5.2-pro:batch", "sk-fake", fetcher); + assert.equal(pricing.source, "live"); + assert.equal(pricing.inputPerM, 3.75); + assert.equal(pricing.outputPerM, 84); + assert.equal(fetches, 1); + + const cached = await resolveModelPricing(dir, config, "openai/gpt-5.2-pro:batch", "sk-fake", fetcher); + assert.equal(cached.source, "cache"); + assert.equal(fetches, 1, "second resolution must come from the 24h cache"); + } finally { + await rm(dir, { recursive: true, force: true }); + } +}); + +test("resolveModelPricing refuses unknown models it cannot price", async () => { + const dir = await mkdtemp(join(tmpdir(), "broadside-pricing-")); + try { + const config = { model: "vendor/mystery", apiKey: "", defaultLenses: ["architecture"], maxCost: 0, pricing: null }; + const fetcher = async () => fakeResponse(200, modelsCatalog([{ id: "other/model", pricing: { prompt: "0.000001", completion: "0.000002" } }])); + await assert.rejects( + () => resolveModelPricing(dir, config, "vendor/mystery", "sk-fake", fetcher), + /Could not resolve per-token pricing/, + ); + } finally { + await rm(dir, { recursive: true, force: true }); + } +}); + +test("submit refuses over-budget runs and creates no run entry; force bypasses", async () => { + const dir = await mkdtemp(join(tmpdir(), "broadside-limit-")); + try { + await writeFile(join(dir, "go.mod"), "module x\n"); + await mkdir(join(dir, "big"), { recursive: true }); + for (let i = 0; i < 40; i++) { + await writeFile(join(dir, "big", `file${i}.go`), "package big\n" + `// ${"y".repeat(2000)}\n`); + } + const fetcher = async (url, init) => { + if (init.method === "POST") { + return fakeResponse(202, { id: "batch-ok", status: "validating" }); + } + return fakeResponse(200, { id: "x", status: "in_progress" }); + }; + + await assert.rejects( + () => runBroadsideSubmit(dir, "sk-fake", { lenses: ["defect"], fetcher, maxCost: 0.0001 }), + /estimated.*exceeds the run limit|exceeds the run limit/i, + ); + let state = await loadBroadsideState(join(dir, ".codecarto", "broadside")); + assert.equal(state.runs.length, 0, "a refused submit must not create a run entry"); + + const forced = await runBroadsideSubmit(dir, "sk-fake", { lenses: ["defect"], fetcher, maxCost: 0.0001, force: true }); + assert.equal(forced.batches.defect.status, "validating"); + assert.equal(forced.maxCost, 0.0001); + + state = await loadBroadsideState(join(dir, ".codecarto", "broadside")); + assert.equal(state.runs.length, 1); + assert.equal(state.runs[0].pricing.source, "built-in"); + } finally { + await rm(dir, { recursive: true, force: true }); + } +}); + +test("submit passes the configured model into batch payloads", async () => { + const dir = await mkdtemp(join(tmpdir(), "broadside-model-")); + try { + await writeFile(join(dir, "go.mod"), "module x\n"); + await writeFile(join(dir, "main.go"), "package main\n"); + let payload; + const fetcher = async (url, init) => { + if (init.method === "POST") { + payload = JSON.parse(init.body); + return fakeResponse(202, { id: "batch-m", status: "validating" }); + } + return fakeResponse(200, { id: "x", status: "in_progress" }); + }; + // config pricing override: submit must not hit the network for pricing. + await mkdir(join(dir, ".codecarto", "broadside"), { recursive: true }); + await writeFile( + join(dir, ".codecarto", "broadside", "config.yaml"), + "model: openai/gpt-5.2-pro:batch\npricing:\n input_per_m: 3.75\n output_per_m: 84\n", + ); + const result = await runBroadsideSubmit(dir, "sk-fake", { lenses: ["architecture"], fetcher, model: "openai/gpt-5.2-pro:batch" }); + assert.equal(payload.model, "openai/gpt-5.2-pro:batch"); + assert.equal(payload.requests[0].body.model, "openai/gpt-5.2-pro:batch"); + assert.equal(result.pricing.source, "config"); + } finally { + await rm(dir, { recursive: true, force: true }); + } +}); + +// ---------- model catalog & models action ---------- + +function catalogWith(...models) { + return { data: models }; +} + +const BATCH_MODEL_SHAPE = (id, prompt, completion, extra = {}) => ({ + id, + name: id, + pricing: { prompt: String(prompt), completion: String(completion) }, + context_length: 1_000_000, + top_provider: { max_completion_tokens: 65_536 }, + supported_parameters: ["tools", "structured_outputs"], + expiration_date: null, + ...extra, +}); + +test("listBatchModels keeps only :batch variants and sorts cheapest first", async () => { + const dir = await mkdtemp(join(tmpdir(), "broadside-models-")); + try { + const config = { model: BROADSIDE_MODEL, apiKey: "", defaultLenses: ["architecture"], maxCost: 0, pricing: null }; + const fetcher = async () => + fakeResponse( + 200, + catalogWith( + BATCH_MODEL_SHAPE("openai/gpt-5.2-pro:batch", 0.00000375, 0.000084), + BATCH_MODEL_SHAPE("google/gemini-3.7-flash:batch", 0.0000001875, 0.0000009375), + { id: "openai/gpt-5.2-pro", pricing: { prompt: "0.00001", completion: "0.0001" } }, // non-batch, must be excluded + BATCH_MODEL_SHAPE("deepseek/deepseek-v4-pro:batch", 0.000000481, 0.000000963), + ), + ); + const { entries } = await listBatchModels(dir, config, "sk-fake", { fetcher }); + assert.equal(entries.length, 3); + assert.ok(!entries.some((e) => !e.id.endsWith(":batch"))); + assert.equal(entries[0].id, "google/gemini-3.7-flash:batch", "cheapest first"); + assert.equal(entries[2].id, "openai/gpt-5.2-pro:batch", "most expensive last"); + } finally { + await rm(dir, { recursive: true, force: true }); + } +}); + +test("submit refuses models that do not advertise structured outputs", async () => { + const dir = await mkdtemp(join(tmpdir(), "broadside-cap-")); + try { + await writeFile(join(dir, "go.mod"), "module x\n"); + await writeFile(join(dir, "main.go"), "package main\n"); + const fetcher = async (url, init) => { + if (init.method === "POST") { + return fakeResponse(202, { id: "batch-x", status: "validating" }); + } + return fakeResponse( + 200, + catalogWith(BATCH_MODEL_SHAPE("vendor/no-structured:batch", 0.0000001, 0.0000002, { supported_parameters: ["tools"] })), + ); + }; + await assert.rejects( + () => runBroadsideSubmit(dir, "sk-fake", { lenses: ["architecture"], fetcher, model: "vendor/no-structured:batch" }), + /does not advertise structured-output support/, + ); + } finally { + await rm(dir, { recursive: true, force: true }); + } +}); + +test("submit clamps lens max_tokens to the provider completion ceiling", async () => { + const dir = await mkdtemp(join(tmpdir(), "broadside-clamp-")); + try { + await writeFile(join(dir, "go.mod"), "module x\n"); + await writeFile(join(dir, "main.go"), "package main\n"); + let payload; + const fetcher = async (url, init) => { + if (init.method === "POST") { + payload = JSON.parse(init.body); + return fakeResponse(202, { id: "batch-c", status: "validating" }); + } + return fakeResponse( + 200, + catalogWith(BATCH_MODEL_SHAPE("vendor/tiny-out:batch", 0.0000001, 0.0000002, { top_provider: { max_completion_tokens: 1000 } })), + ); + }; + await runBroadsideSubmit(dir, "sk-fake", { lenses: ["architecture"], fetcher, model: "vendor/tiny-out:batch" }); + assert.equal(payload.requests[0].body.max_tokens, 1000, "8000-token lens must clamp to the 1000-token ceiling"); + } finally { + await rm(dir, { recursive: true, force: true }); + } +}); + +test("fetchCodingBenchmarks maps indices by base slug (batch suffix stripped)", async () => { + const fetcher = async () => + fakeResponse( + 200, + { + data: [{ model_permaslug: "google/gemini-3.7-flash", coding_index: 62.4, intelligence_index: 58.1 }], + meta: { as_of: "2026-08-23", source_url: "https://example.com" }, + }, + ); + const benchmarks = await fetchCodingBenchmarks("sk-fake", fetcher); + assert.equal(benchmarks.byBaseSlug["google/gemini-3.7-flash"].codingIndex, 62.4); + assert.ok("google/gemini-3.7-flash:batch".indexOf(":") >= 0, "the batch variant resolves through its base slug"); + assert.equal(benchmarks.meta.as_of, "2026-08-23"); +}); + +test("modelsText renders pricing, caps, support, and benchmark columns", () => { + const entries = [ + { + id: "google/gemini-3.7-flash:batch", + name: "Google: Gemini 3.7 Flash (batch)", + inputPerM: 0.1875, + outputPerM: 0.9375, + contextLength: 1_048_576, + maxCompletionTokens: 65_536, + supportedParameters: ["tools", "structured_outputs"], + expirationDate: null, + }, + ]; + const text = modelsText(entries, { + benchmarks: { byBaseSlug: { "google/gemini-3.7-flash": { codingIndex: 62.4 } }, meta: { as_of: "2026-08-23" } }, + defaultModel: "google/gemini-3.7-flash:batch", + }); + assert.match(text, /0\.188/); + assert.match(text, /64k/); + assert.match(text, /62\.4/); + assert.match(text, /\(default\)/); +}); + +// ---------- batch client with a fake fetcher ---------- + +function fakeResponse(status, body) { + return { + status, + ok: status >= 200 && status < 300, + json: async () => body, + }; +} + +test("submitBatch orders endpoint and model before requests in the payload", async () => { + let captured; + const fetcher = async (url, init) => { + captured = { url, payload: JSON.parse(init.body) }; + return fakeResponse(202, { id: "batch-test-1", status: "validating" }); + }; + const result = await submitBatch( + [ + { + custom_id: "req-1", + body: { + model: BROADSIDE_MODEL, + messages: [{ role: "user", content: "hi" }], + response_format: { type: "json_schema", json_schema: { name: "x", strict: true, schema: {} } }, + max_tokens: 100, + }, + }, + ], + "sk-fake", + fetcher, + ); + assert.equal(result.batchId, "batch-test-1"); + const keys = Object.keys(captured.payload); + assert.deepEqual(keys, ["endpoint", "model", "requests"], "the API stream-parses and rejects requests-first bodies"); + assert.equal(captured.payload.endpoint, "/v1/chat/completions"); + assert.equal(captured.payload.model, BROADSIDE_MODEL); + assert.equal(captured.payload.requests.length, 1); +}); + +test("submitBatch surfaces non-202 rejection bodies", async () => { + const fetcher = async () => fakeResponse(400, { error: { message: "no" } }); + const result = await submitBatch([], "sk-fake", fetcher); + assert.equal(result.status, "rejected"); + assert.ok(result.error); +}); + +test("runBroadsideSubmit records a run and fires one batch per lens", async () => { + const dir = await makeFixture(); + try { + const seenBatches = []; + const fetcher = async (url, init) => { + if (init.method === "POST") { + const payload = JSON.parse(init.body); + seenBatches.push(payload); + return fakeResponse(202, { id: `batch-${seenBatches.length}`, status: "validating" }); + } + return fakeResponse(200, { id: "x", status: "in_progress" }); + }; + const result = await runBroadsideSubmit(dir, "sk-fake", { lenses: ["architecture", "security"], fetcher }); + assert.equal(Object.keys(result.batches).length, 2); + assert.ok(result.estimatedTotalCost > 0); + assert.equal(seenBatches.length, 2); + for (const payload of seenBatches) { + assert.ok(payload.requests.length >= 1); + } + const { state } = await runBroadsideStatus(dir); + assert.equal(state.runs.length, 1); + assert.equal(state.runs[0].status, "in-flight"); + assert.equal(state.runs[0].batches.architecture.status, "validating"); + } finally { + await rm(dir, { recursive: true, force: true }); + } +}); + +// ---------- collect with fake fetcher ---------- + +test("runBroadsideCollect polls, saves results, and runs synthesis + triage", async () => { + const dir = await makeFixture(); + try { + const lensPayload = { + results: [ + { + custom_id: "architecture-root", + response: { + status_code: 200, + body: { + choices: [{ message: { role: "assistant", content: JSON.stringify({ tech_stack: { language: "Go", build_system: "go modules" }, module_architecture: [], data_flow: "x", entry_points: ["main.go"] }) } }], + }, + }, + error: null, + }, + ], + usage: { cost: 0.001 }, + request_counts: { total: 1, completed: 1, failed: 0 }, + }; + const synthPayload = { + results: [ + { + custom_id: "synthesis", + response: { + status_code: 200, + body: { + choices: [{ message: { role: "assistant", content: JSON.stringify({ executive_summary: "ok", severity_summary: { critical: 0, high: 1, medium: 2, low: 3 }, top_findings: [{ title: "lead", severity: "high", source_lens: "architecture", summary: "a lead" }] }) } }], + }, + }, + error: null, + }, + ], + usage: { cost: 0.002 }, + request_counts: { total: 1, completed: 1, failed: 0 }, + }; + const triagePayload = { + results: [ + { + custom_id: "triage", + response: { + status_code: 200, + body: { + choices: [{ message: { role: "assistant", content: JSON.stringify({ summary: "work order", items: [{ title: "fix the thing", severity: "high", module: "server", impact: "high", difficulty: "low", priority: "P0", effort_estimate: "2h", rationale: "obvious" }] }) } }], + }, + }, + error: null, + }, + ], + usage: { cost: 0.003 }, + request_counts: { total: 1, completed: 1, failed: 0 }, + }; + let postCount = 0; + const fetcher = async (url, init) => { + if (init.method === "POST") { + postCount += 1; + const payload = JSON.parse(init.body); + const id = payload.requests[0].custom_id === "synthesis" ? "batch-synth" : payload.requests[0].custom_id === "triage" ? "batch-triage" : "batch-lens"; + return fakeResponse(202, { id, status: "validating" }); + } + if (String(url).includes("batch-lens")) { + return fakeResponse(200, { id: "batch-lens", status: "completed", ...lensPayload }); + } + if (String(url).includes("batch-triage")) { + return fakeResponse(200, { id: "batch-triage", status: "completed", ...triagePayload }); + } + return fakeResponse(200, { id: "batch-synth", status: "completed", ...synthPayload }); + }; + + await runBroadsideSubmit(dir, "sk-fake", { lenses: ["architecture"], fetcher }); + const collect = await runBroadsideCollect(dir, "sk-fake", { fetcher }); + assert.equal(collect.status, "completed"); + assert.equal(collect.resultCount, 1); + assert.ok(collect.totalCost > 0); + assert.equal(collect.synthesis.status, "completed"); + assert.equal(collect.topFindings.length, 1); + assert.equal(collect.topFindings[0].title, "lead"); + assert.equal(collect.triage.status, "completed"); + assert.equal(collect.topTriageItems.length, 1); + assert.equal(collect.topTriageItems[0].title, "fix the thing"); + assert.equal(collect.topTriageItems[0].priority, "P0"); + assert.equal(postCount, 3, "lens + synthesis + triage batches"); + } finally { + await rm(dir, { recursive: true, force: true }); + } +}); + +test("runBroadsideCollect skips triage when include_triage is false", async () => { + const dir = await makeFixture(); + try { + const lensPayload = { + results: [ + { + custom_id: "architecture-root", + response: { + status_code: 200, + body: { choices: [{ message: { content: JSON.stringify({ tech_stack: { language: "Go", build_system: "x" }, module_architecture: [], data_flow: "x", entry_points: [] }) } }] }, + }, + error: null, + }, + ], + usage: { cost: 0.001 }, + request_counts: { total: 1, completed: 1, failed: 0 }, + }; + const synthPayload = { + results: [ + { + custom_id: "synthesis", + response: { + status_code: 200, + body: { choices: [{ message: { content: JSON.stringify({ executive_summary: "ok", severity_summary: { critical: 0, high: 0, medium: 0, low: 0 }, top_findings: [] }) } }] }, + }, + error: null, + }, + ], + usage: { cost: 0.002 }, + request_counts: { total: 1, completed: 1, failed: 0 }, + }; + const seenPosts = []; + const fetcher = async (url, init) => { + if (init.method === "POST") { + const payload = JSON.parse(init.body); + seenPosts.push(payload.requests[0].custom_id); + return fakeResponse(202, { id: `batch-${payload.requests[0].custom_id}`, status: "validating" }); + } + if (String(url).includes("batch-synthesis")) { + return fakeResponse(200, { id: "batch-synthesis", status: "completed", ...synthPayload }); + } + return fakeResponse(200, { id: "batch-architecture-root", status: "completed", ...lensPayload }); + }; + + await runBroadsideSubmit(dir, "sk-fake", { lenses: ["architecture"], fetcher }); + const collect = await runBroadsideCollect(dir, "sk-fake", { fetcher, includeTriage: false }); + assert.deepEqual(seenPosts, ["architecture-root", "synthesis"], "no triage batch may be submitted"); + assert.equal(collect.triage.status, "pending"); + } finally { + await rm(dir, { recursive: true, force: true }); + } +}); + +// ---------- markdown rendering & fence-tolerant parsing ---------- + +test("renderFindingsMarkdown turns parsed JSON into readable text", () => { + const md = renderFindingsMarkdown(JSON.stringify({ title: "T", severity: "high", nested: { a: "b" }, list: [{ title: "x" }] })); + assert.match(md, /\*\*title\*\*: T/); + assert.match(md, /\*\*severity\*\*: high/); + assert.match(md, /\*\*nested\*\*:/); + assert.match(md, /1\. x/); +}); + +test("renderFindingsMarkdown passes invalid JSON through untouched", () => { + assert.equal(renderFindingsMarkdown("not json"), "not json"); +}); + +test("parseLensJson strips markdown code fences", () => { + const fenced = '```json\n{"title": "F", "severity": "low"}\n```'; + const parsed = core.parseLensJson(fenced); + assert.deepEqual(parsed, { title: "F", severity: "low" }); + const withoutLang = '```\n{"title": "F"}\n```'; + assert.deepEqual(core.parseLensJson(withoutLang), { title: "F" }); +}); + +test("parseLensJson returns null for truncated or non-JSON content", () => { + assert.equal(core.parseLensJson('```json\n{"title": "unterminated\n```'), null); + assert.equal(core.parseLensJson("The findings are numerous."), null); +}); + +test("saveLensResults marks truncated content and writes parsed JSON cleanly", async () => { + const dir = await mkdtemp(join(tmpdir(), "broadside-trunc-")); + try { + const batch = { + results: [ + { + custom_id: "defect-core-1", + response: { + status_code: 200, + body: { choices: [{ message: { content: '```json\n{"module": "core", "findings": []}\n```' } }] }, + }, + error: null, + }, + { + custom_id: "defect-core-2", + response: { + status_code: 200, + body: { choices: [{ message: { content: '{"module": "core-2", "findin' } }] }, + }, + error: null, + }, + ], + }; + const stored = await core.saveLensResults(dir, "defect", batch); + assert.equal(stored.length, 2); + assert.equal(stored[0].truncated, false); + assert.equal(stored[1].truncated, true); + + const written = JSON.parse(await readFile(join(dir, "defect-core-1.json"), "utf8")); + assert.equal(written.module, "core", "fenced JSON must be saved parsed, not verbatim"); + } finally { + await rm(dir, { recursive: true, force: true }); + } +});