diff --git a/CHANGELOG.md b/CHANGELOG.md index 485ea2c..ba8b2c8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ All notable changes to this project are documented here. The format is based on ### Added +- **Broad-Side: incremental re-scouting** (#142). Submit records the git HEAD (and dirty flag) of each run, and `incremental: true` diffs against the previous run's HEAD to scan only the modules whose files changed — unchanged modules are skipped, so recurring scouting costs O(delta) instead of O(repo). Falls back to a full scan on a dirty tree, a non-git tree, or when no prior run exists. Repo-info lenses (architecture) always run. - **Broad-Side: zero-config slicing** (#140). The defect, conventions, and porting lenses now slice by `auto` instead of always per-directory: a repo whose matching files fit within the lens's char cap collapses to a single whole-repo slice (one request instead of one per module), while a repo too large for one slice still splits by top-level directory. Small repos stop paying for per-module request overhead; large repos keep full coverage. The decision is deterministic from file sizes and recorded implicitly in the slice layout. - **Broad-Side: truncated slices re-submit automatically** (#133). Submit now persists every request body to `.codecarto/broadside//requests.json`, and collect re-submits each truncated result once with a doubled output cap (bounded by the model's completion ceiling) — recovering coverage lost to a `max_tokens` cutoff instead of leaving the module silently unscouted. Recovered slices rewrite their JSON/markdown, clear their truncation flag, and report in the collect summary (`↻ N recovered`); anything still truncated after the retry stays flagged. Opt out with `retry_truncated: false` on collect. - **Broad-Side: per-language lens prompts** (#137). The defect and conventions lenses now build their system prompts from a language profile (Go, Python, Rust, TypeScript/JavaScript, plus a neutral default) instead of hardcoding Go idioms — a Python scanner no longer hears "goroutines without ctx"; it hears bare-except and context-manager checks. Convention extraction names language-appropriate categories (crates/modules vs modules/packages) and idiom hints. Language detection gains a `.js` bucket so JavaScript repos resolve to the TypeScript profile. Schemas are unchanged, so synthesis is unaffected. diff --git a/ROADMAP.md b/ROADMAP.md index 431a4d8..bb6f85b 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -51,7 +51,7 @@ file only moves when a tier completes. | Item | Issue | Notes | |---|---|---| | **Multi-model** — DeepSeek/Anthropic batch endpoints behind the lens registry | [#141](https://github.com/HuginnIndustries/CodeCartographer/issues/141) | Partially shipped: catalog lookup, `models` action, pricing + capability pre-flight. Remaining: per-model prompt tweaks and a stronger default for semantic lenses | -| **Incremental re-scouting** — diff against previous run's HEAD, rescan changed modules only | [#142](https://github.com/HuginnIndustries/CodeCartographer/issues/142) | Makes recurring scans O(delta) | +| **Incremental re-scouting** — diff against previous run's HEAD, rescan changed modules only | [#142](https://github.com/HuginnIndustries/CodeCartographer/issues/142) | **Shipped**: `incremental: true` diffs against the prior run's HEAD; dirty tree falls back to full scan | | **CodeCartoShow pipeline stage** — BATCH-SCOUT between SELECT and the interactive run | [CodeCartoShow#1](https://github.com/HuginnIndustries/CodeCartoShow/issues/1) | `scripts/batch-analyze.py` proved it; evidence rules apply unchanged | ## Tier 4 — open questions, not commitments diff --git a/core/broadside.ts b/core/broadside.ts index 40cd467..b4c4f3d 100644 --- a/core/broadside.ts +++ b/core/broadside.ts @@ -136,6 +136,8 @@ export type FileSlice = { content: string; fileCount: number; chars: number; + /** Repo-relative paths of the files folded into this slice. */ + files: string[]; }; export type BatchRequest = { @@ -201,6 +203,12 @@ export type BroadsideRun = { maxCost?: number; /** The model's completion ceiling, recorded so collect can cap retries. */ outputCap?: number; + /** Git HEAD at submit time, for incremental re-scouting (#142). */ + sourceHead?: string | null; + /** Whether the working tree was dirty at submit time. */ + sourceDirty?: boolean; + /** When incremental, the previous run's HEAD this run diffs against. */ + baseHead?: string | null; }; export type BroadsideStateFile = { @@ -1035,6 +1043,43 @@ async function listRepoFiles(targetDir: string): Promise { } } +async function gitHead(targetDir: string): Promise { + try { + const { stdout } = await execFileAsync("git", ["-C", targetDir, "rev-parse", "HEAD"], { maxBuffer: 1024 * 1024 }); + return stdout.trim() || null; + } catch { + return null; + } +} + +async function gitDirty(targetDir: string): Promise { + try { + const { stdout } = await execFileAsync("git", ["-C", targetDir, "status", "--porcelain"], { maxBuffer: 1024 * 1024 }); + return stdout.trim().length > 0; + } catch { + return false; + } +} + +/** + * Repo-relative paths changed since `baseHead` (or all files when there is + * no base). Returns null when the diff cannot be computed (non-git tree, + * missing base commit) so callers fall back to a full scan. + */ +async function changedFilesSince(targetDir: string, baseHead: string | null): Promise | null> { + if (!baseHead) return null; + try { + const { stdout } = await execFileAsync( + "git", + ["-C", targetDir, "diff", "--name-only", baseHead, "HEAD"], + { maxBuffer: 64 * 1024 * 1024 }, + ); + return new Set(stdout.split("\n").filter(Boolean)); + } catch { + return null; + } +} + async function walkFiles( rootDir: string, dir: string, @@ -1269,6 +1314,7 @@ async function slurpFileList( let parts: string[] = []; let running = 0; let fileCount = 0; + let filePaths: string[] = []; const flush = () => { if (parts.length === 0) return; @@ -1277,10 +1323,12 @@ async function slurpFileList( content: parts.join("\n"), fileCount, chars: running, + files: filePaths, }); parts = []; running = 0; fileCount = 0; + filePaths = []; }; for (const file of files) { @@ -1306,6 +1354,7 @@ async function slurpFileList( parts.push(block); running += block.length; fileCount += 1; + filePaths.push(file.relPath); } flush(); return slices; @@ -1314,7 +1363,7 @@ async function slurpFileList( export async function gatherSlices(targetDir: string, lens: LensDefinition, info: RepoInfo): Promise { if (lens.sliceBy === "none" && lens.globsFor(info).length === 0) { // Repo-info lens (architecture): the prompt is built from info alone. - return [{ moduleName: "root", content: "", fileCount: 0, chars: 0 }]; + return [{ moduleName: "root", content: "", fileCount: 0, chars: 0, files: [] }]; } const allFiles = await listRepoFiles(targetDir); const files = collectLensFiles(allFiles, lens, info); @@ -1792,6 +1841,8 @@ export async function runBroadsideSubmit( maxCost?: number; /** Submit even when the estimate exceeds maxCost. */ force?: boolean; + /** Diff against the previous run's HEAD and scan only changed modules (#142). */ + incremental?: boolean; } = {}, ): Promise { const info = await collectRepoInfo(cwd); @@ -1829,6 +1880,24 @@ export async function runBroadsideSubmit( // output than the model can produce fails the whole batch. const outputCap = entry.maxCompletionTokens; + // Incremental re-scouting (#142): diff against the previous run's HEAD + // and scan only the modules whose files changed. Falls back to a full + // scan when there is no prior run, the tree is dirty, or the diff fails. + const sourceHead = await gitHead(cwd); + const sourceDirty = await gitDirty(cwd); + let baseHead: string | null = null; + let changed: Set | null = null; + if (opts.incremental) { + const state = await loadBroadsideState(broadsideDir); + // The baseline is the most recent run that recorded a HEAD — a + // submit-only run (never collected) is still a valid committed base. + const previous = [...state.runs].reverse().find((r) => r.sourceHead); + if (previous?.sourceHead && !sourceDirty) { + baseHead = previous.sourceHead; + changed = await changedFilesSince(cwd, baseHead); + } + } + // Slice offline first so the estimate covers every request we would send. const slicesByLens = new Map(); let estimatedInputTokens = 0; @@ -1837,7 +1906,12 @@ export async function runBroadsideSubmit( const perLensEstimate: Array<{ lens: LensDefinition; cost: number; maxTokens: number }> = []; for (const lensId of lensIds) { const lens = getLens(lensId); - const slices = await gatherSlices(cwd, lens, info); + let slices = await gatherSlices(cwd, lens, info); + if (changed) { + // Repo-info slices (empty files, e.g. architecture) always run; + // file-backed slices run only when one of their files changed. + slices = slices.filter((s) => s.files.length === 0 || s.files.some((f) => changed!.has(f))); + } slicesByLens.set(lensId, slices); const maxTokens = outputCap ? Math.min(lens.maxTokens, outputCap) : lens.maxTokens; const estimate = estimateCost(lens, slices, pricing, maxTokens); @@ -1873,6 +1947,9 @@ export async function runBroadsideSubmit( pricing, maxCost: limit > 0 ? limit : undefined, outputCap, + sourceHead, + sourceDirty, + baseHead, }; state.runs.push(run); await saveBroadsideState(broadsideDir, state); diff --git a/mcp-server/server.ts b/mcp-server/server.ts index f505d16..1e557e5 100644 --- a/mcp-server/server.ts +++ b/mcp-server/server.ts @@ -1006,6 +1006,7 @@ export async function handleBroadside(args: { max_cost?: number; force?: boolean; include_benchmarks?: boolean; + incremental?: boolean; }) { const cwd = await validateCwd(args.cwd); const action = args.action ?? "submit"; @@ -1055,6 +1056,7 @@ export async function handleBroadside(args: { model: config.model, maxCost, force: args.force === true, + incremental: args.incremental === true, }).catch((error) => { throw new McpError(ErrorCode.InvalidRequest, error instanceof Error ? error.message : String(error)); }); @@ -1448,6 +1450,11 @@ const TOOLS = [ type: "boolean", description: "Submit even when the cost estimate exceeds max_cost (default false).", }, + incremental: { + type: "boolean", + description: + "Diff against the previous run's git HEAD and scan only the modules whose files changed (falls back to a full scan on a dirty tree or when no prior run exists). Default false.", + }, include_benchmarks: { type: "boolean", description: "For action 'models': annotate each model with its Artificial Analysis coding index (extra API call; default false).", diff --git a/tests/broadside.test.mjs b/tests/broadside.test.mjs index 3dde27e..67f51cc 100644 --- a/tests/broadside.test.mjs +++ b/tests/broadside.test.mjs @@ -799,6 +799,96 @@ test("collect polls multiple in-flight lenses concurrently", async () => { } }); +// ---------- incremental re-scouting (#142) ---------- + +import { execFile } from "node:child_process"; +import { promisify } from "node:util"; +const execFileAsync = promisify(execFile); + +async function git(dir, ...args) { + await execFileAsync("git", ["-C", dir, ...args], { maxBuffer: 16 * 1024 * 1024 }); +} + +async function makeGitRepo() { + const dir = await mkdtemp(join(tmpdir(), "broadside-git-")); + await git(dir, "init", "-q"); + await git(dir, "config", "user.email", "test@example.com"); + await git(dir, "config", "user.name", "Test"); + await mkdir(join(dir, "server")); + await mkdir(join(dir, "model")); + await writeFile(join(dir, "go.mod"), "module x\n"); + await writeFile(join(dir, "main.go"), "package main\n"); + for (let i = 0; i < 40; i++) { + await writeFile(join(dir, "server", `s${i}.go`), "package server\n" + `// ${"x".repeat(2000)}\n`); + await writeFile(join(dir, "model", `m${i}.go`), "package model\n" + `// ${"y".repeat(2000)}\n`); + } + await git(dir, "add", "-A"); + await git(dir, "commit", "-q", "-m", "initial"); + return dir; +} + +function submittedCustomIds(payloads) { + return payloads.flatMap((p) => p.requests.map((r) => r.custom_id)); +} + +test("incremental submit scans only modules whose files changed", async () => { + const dir = await makeGitRepo(); + try { + const payloads = []; + const fetcher = async (url, init) => { + if (init.method === "POST") { + payloads.push(JSON.parse(init.body)); + return fakeResponse(202, { id: `batch-${payloads.length}`, status: "validating" }); + } + return fakeResponse(200, { id: "x", status: "in_progress" }); + }; + + await runBroadsideSubmit(dir, "sk-fake", { lenses: ["defect"], fetcher }); + const firstIds = submittedCustomIds(payloads); + assert.ok(firstIds.some((id) => id.startsWith("defect-server")), "baseline must scan server"); + assert.ok(firstIds.some((id) => id.startsWith("defect-model")), "baseline must scan model"); + + // Change one server file, commit, then re-scout incrementally. + await writeFile(join(dir, "server", "s0.go"), "package server\n// changed\n"); + await git(dir, "add", "-A"); + await git(dir, "commit", "-q", "-m", "change server"); + + payloads.length = 0; + await runBroadsideSubmit(dir, "sk-fake", { lenses: ["defect"], fetcher, incremental: true }); + const secondIds = submittedCustomIds(payloads); + assert.ok(secondIds.some((id) => id.startsWith("defect-server")), "changed module must be re-scanned"); + assert.ok(!secondIds.some((id) => id.startsWith("defect-model")), "unchanged module must be skipped"); + } finally { + await rm(dir, { recursive: true, force: true }); + } +}); + +test("incremental submit falls back to a full scan when the tree is dirty", async () => { + const dir = await makeGitRepo(); + try { + const payloads = []; + const fetcher = async (url, init) => { + if (init.method === "POST") { + payloads.push(JSON.parse(init.body)); + return fakeResponse(202, { id: `batch-${payloads.length}`, status: "validating" }); + } + return fakeResponse(200, { id: "x", status: "in_progress" }); + }; + + await runBroadsideSubmit(dir, "sk-fake", { lenses: ["defect"], fetcher }); + // Uncommitted change → the diff is unreliable, so scan everything. + await writeFile(join(dir, "model", "m0.go"), "package model\n// dirty\n"); + + payloads.length = 0; + await runBroadsideSubmit(dir, "sk-fake", { lenses: ["defect"], fetcher, incremental: true }); + const ids = submittedCustomIds(payloads); + assert.ok(ids.some((id) => id.startsWith("defect-server")), "dirty tree must still scan server"); + assert.ok(ids.some((id) => id.startsWith("defect-model")), "dirty tree must still scan model"); + } finally { + await rm(dir, { recursive: true, force: true }); + } +}); + // ---------- batch client with a fake fetcher ---------- function fakeResponse(status, body) {