feat: Broad-Side incremental re-scouting (#142)

CHANGELOG.md

@@ -6,6 +6,7 @@ All notable changes to this project are documented here. The format is based on
 
 ### Added
 
+- **Broad-Side: incremental re-scouting** (#142). Submit records the git HEAD (and dirty flag) of each run, and `incremental: true` diffs against the previous run's HEAD to scan only the modules whose files changed — unchanged modules are skipped, so recurring scouting costs O(delta) instead of O(repo). Falls back to a full scan on a dirty tree, a non-git tree, or when no prior run exists. Repo-info lenses (architecture) always run.
 - **Broad-Side: zero-config slicing** (#140). The defect, conventions, and porting lenses now slice by `auto` instead of always per-directory: a repo whose matching files fit within the lens's char cap collapses to a single whole-repo slice (one request instead of one per module), while a repo too large for one slice still splits by top-level directory. Small repos stop paying for per-module request overhead; large repos keep full coverage. The decision is deterministic from file sizes and recorded implicitly in the slice layout.
 - **Broad-Side: truncated slices re-submit automatically** (#133). Submit now persists every request body to `.codecarto/broadside/<run>/requests.json`, and collect re-submits each truncated result once with a doubled output cap (bounded by the model's completion ceiling) — recovering coverage lost to a `max_tokens` cutoff instead of leaving the module silently unscouted. Recovered slices rewrite their JSON/markdown, clear their truncation flag, and report in the collect summary (`↻ N recovered`); anything still truncated after the retry stays flagged. Opt out with `retry_truncated: false` on collect.
 - **Broad-Side: per-language lens prompts** (#137). The defect and conventions lenses now build their system prompts from a language profile (Go, Python, Rust, TypeScript/JavaScript, plus a neutral default) instead of hardcoding Go idioms — a Python scanner no longer hears "goroutines without ctx"; it hears bare-except and context-manager checks. Convention extraction names language-appropriate categories (crates/modules vs modules/packages) and idiom hints. Language detection gains a `.js` bucket so JavaScript repos resolve to the TypeScript profile. Schemas are unchanged, so synthesis is unaffected.

ROADMAP.md

@@ -51,7 +51,7 @@ file only moves when a tier completes.
 | Item | Issue | Notes |
 |---|---|---|
 | **Multi-model** — DeepSeek/Anthropic batch endpoints behind the lens registry | [#141](https://github.com/HuginnIndustries/CodeCartographer/issues/141) | Partially shipped: catalog lookup, `models` action, pricing + capability pre-flight. Remaining: per-model prompt tweaks and a stronger default for semantic lenses |
-| **Incremental re-scouting** — diff against previous run's HEAD, rescan changed modules only | [#142](https://github.com/HuginnIndustries/CodeCartographer/issues/142) | Makes recurring scans O(delta) |
+| **Incremental re-scouting** — diff against previous run's HEAD, rescan changed modules only | [#142](https://github.com/HuginnIndustries/CodeCartographer/issues/142) | **Shipped**: `incremental: true` diffs against the prior run's HEAD; dirty tree falls back to full scan |
 | **CodeCartoShow pipeline stage** — BATCH-SCOUT between SELECT and the interactive run | [CodeCartoShow#1](https://github.com/HuginnIndustries/CodeCartoShow/issues/1) | `scripts/batch-analyze.py` proved it; evidence rules apply unchanged |
 
 ## Tier 4 — open questions, not commitments

core/broadside.ts

@@ -136,6 +136,8 @@ export type FileSlice = {
 	content: string;
 	fileCount: number;
 	chars: number;
+	/** Repo-relative paths of the files folded into this slice. */
+	files: string[];
 };
 
 export type BatchRequest = {
@@ -201,6 +203,12 @@ export type BroadsideRun = {
 	maxCost?: number;
 	/** The model's completion ceiling, recorded so collect can cap retries. */
 	outputCap?: number;
+	/** Git HEAD at submit time, for incremental re-scouting (#142). */
+	sourceHead?: string | null;
+	/** Whether the working tree was dirty at submit time. */
+	sourceDirty?: boolean;
+	/** When incremental, the previous run's HEAD this run diffs against. */
+	baseHead?: string | null;
 };
 
 export type BroadsideStateFile = {
@@ -1035,6 +1043,43 @@ async function listRepoFiles(targetDir: string): Promise<string[]> {
 	}
 }
 
+async function gitHead(targetDir: string): Promise<string | null> {
+	try {
+		const { stdout } = await execFileAsync("git", ["-C", targetDir, "rev-parse", "HEAD"], { maxBuffer: 1024 * 1024 });
+		return stdout.trim() || null;
+	} catch {
+		return null;
+	}
+}
+
+async function gitDirty(targetDir: string): Promise<boolean> {
+	try {
+		const { stdout } = await execFileAsync("git", ["-C", targetDir, "status", "--porcelain"], { maxBuffer: 1024 * 1024 });
+		return stdout.trim().length > 0;
+	} catch {
+		return false;
+	}
+}
+
+/**
+ * Repo-relative paths changed since `baseHead` (or all files when there is
+ * no base). Returns null when the diff cannot be computed (non-git tree,
+ * missing base commit) so callers fall back to a full scan.
+ */
+async function changedFilesSince(targetDir: string, baseHead: string | null): Promise<Set<string> | null> {
+	if (!baseHead) return null;
+	try {
+		const { stdout } = await execFileAsync(
+			"git",
+			["-C", targetDir, "diff", "--name-only", baseHead, "HEAD"],
+			{ maxBuffer: 64 * 1024 * 1024 },
+		);
+		return new Set(stdout.split("\n").filter(Boolean));
+	} catch {
+		return null;
+	}
+}
+
 async function walkFiles(
 	rootDir: string,
 	dir: string,
@@ -1269,6 +1314,7 @@ async function slurpFileList(
 	let parts: string[] = [];
 	let running = 0;
 	let fileCount = 0;
+	let filePaths: string[] = [];
 
 	const flush = () => {
 		if (parts.length === 0) return;
@@ -1277,10 +1323,12 @@ async function slurpFileList(
 			content: parts.join("\n"),
 			fileCount,
 			chars: running,
+			files: filePaths,
 		});
 		parts = [];
 		running = 0;
 		fileCount = 0;
+		filePaths = [];
 	};
 
 	for (const file of files) {
@@ -1306,6 +1354,7 @@ async function slurpFileList(
 		parts.push(block);
 		running += block.length;
 		fileCount += 1;
+		filePaths.push(file.relPath);
 	}
 	flush();
 	return slices;
@@ -1314,7 +1363,7 @@ async function slurpFileList(
 export async function gatherSlices(targetDir: string, lens: LensDefinition, info: RepoInfo): Promise<FileSlice[]> {
 	if (lens.sliceBy === "none" && lens.globsFor(info).length === 0) {
 		// Repo-info lens (architecture): the prompt is built from info alone.
-		return [{ moduleName: "root", content: "", fileCount: 0, chars: 0 }];
+		return [{ moduleName: "root", content: "", fileCount: 0, chars: 0, files: [] }];
 	}
 	const allFiles = await listRepoFiles(targetDir);
 	const files = collectLensFiles(allFiles, lens, info);
@@ -1792,6 +1841,8 @@ export async function runBroadsideSubmit(
 		maxCost?: number;
 		/** Submit even when the estimate exceeds maxCost. */
 		force?: boolean;
+		/** Diff against the previous run's HEAD and scan only changed modules (#142). */
+		incremental?: boolean;
 	} = {},
 ): Promise<BroadsideSubmitResult> {
 	const info = await collectRepoInfo(cwd);
@@ -1829,6 +1880,24 @@ export async function runBroadsideSubmit(
 	// output than the model can produce fails the whole batch.
 	const outputCap = entry.maxCompletionTokens;
 
+	// Incremental re-scouting (#142): diff against the previous run's HEAD
+	// and scan only the modules whose files changed. Falls back to a full
+	// scan when there is no prior run, the tree is dirty, or the diff fails.
+	const sourceHead = await gitHead(cwd);
+	const sourceDirty = await gitDirty(cwd);
+	let baseHead: string | null = null;
+	let changed: Set<string> | null = null;
+	if (opts.incremental) {
+		const state = await loadBroadsideState(broadsideDir);
+		// The baseline is the most recent run that recorded a HEAD — a
+		// submit-only run (never collected) is still a valid committed base.
+		const previous = [...state.runs].reverse().find((r) => r.sourceHead);
+		if (previous?.sourceHead && !sourceDirty) {
+			baseHead = previous.sourceHead;
+			changed = await changedFilesSince(cwd, baseHead);
+		}
+	}
+
 	// Slice offline first so the estimate covers every request we would send.
 	const slicesByLens = new Map<BroadsideLensId, FileSlice[]>();
 	let estimatedInputTokens = 0;
@@ -1837,7 +1906,12 @@ export async function runBroadsideSubmit(
 	const perLensEstimate: Array<{ lens: LensDefinition; cost: number; maxTokens: number }> = [];
 	for (const lensId of lensIds) {
 		const lens = getLens(lensId);
-		const slices = await gatherSlices(cwd, lens, info);
+		let slices = await gatherSlices(cwd, lens, info);
+		if (changed) {
+			// Repo-info slices (empty files, e.g. architecture) always run;
+			// file-backed slices run only when one of their files changed.
+			slices = slices.filter((s) => s.files.length === 0 || s.files.some((f) => changed!.has(f)));
+		}
 		slicesByLens.set(lensId, slices);
 		const maxTokens = outputCap ? Math.min(lens.maxTokens, outputCap) : lens.maxTokens;
 		const estimate = estimateCost(lens, slices, pricing, maxTokens);
@@ -1873,6 +1947,9 @@ export async function runBroadsideSubmit(
 		pricing,
 		maxCost: limit > 0 ? limit : undefined,
 		outputCap,
+		sourceHead,
+		sourceDirty,
+		baseHead,
 	};
 	state.runs.push(run);
 	await saveBroadsideState(broadsideDir, state);

mcp-server/server.ts

@@ -1006,6 +1006,7 @@ export async function handleBroadside(args: {
 	max_cost?: number;
 	force?: boolean;
 	include_benchmarks?: boolean;
+	incremental?: boolean;
 }) {
 	const cwd = await validateCwd(args.cwd);
 	const action = args.action ?? "submit";
@@ -1055,6 +1056,7 @@ export async function handleBroadside(args: {
 			model: config.model,
 			maxCost,
 			force: args.force === true,
+			incremental: args.incremental === true,
 		}).catch((error) => {
 			throw new McpError(ErrorCode.InvalidRequest, error instanceof Error ? error.message : String(error));
 		});
@@ -1448,6 +1450,11 @@ const TOOLS = [
 					type: "boolean",
 					description: "Submit even when the cost estimate exceeds max_cost (default false).",
 				},
+				incremental: {
+					type: "boolean",
+					description:
+						"Diff against the previous run's git HEAD and scan only the modules whose files changed (falls back to a full scan on a dirty tree or when no prior run exists). Default false.",
+				},
 				include_benchmarks: {
 					type: "boolean",
 					description: "For action 'models': annotate each model with its Artificial Analysis coding index (extra API call; default false).",

tests/broadside.test.mjs

@@ -799,6 +799,96 @@ test("collect polls multiple in-flight lenses concurrently", async () => {
 	}
 });
 
+// ---------- incremental re-scouting (#142) ----------
+
+import { execFile } from "node:child_process";
+import { promisify } from "node:util";
+const execFileAsync = promisify(execFile);
+
+async function git(dir, ...args) {
+	await execFileAsync("git", ["-C", dir, ...args], { maxBuffer: 16 * 1024 * 1024 });
+}
+
+async function makeGitRepo() {
+	const dir = await mkdtemp(join(tmpdir(), "broadside-git-"));
+	await git(dir, "init", "-q");
+	await git(dir, "config", "user.email", "test@example.com");
+	await git(dir, "config", "user.name", "Test");
+	await mkdir(join(dir, "server"));
+	await mkdir(join(dir, "model"));
+	await writeFile(join(dir, "go.mod"), "module x\n");
+	await writeFile(join(dir, "main.go"), "package main\n");
+	for (let i = 0; i < 40; i++) {
+		await writeFile(join(dir, "server", `s${i}.go`), "package server\n" + `// ${"x".repeat(2000)}\n`);
+		await writeFile(join(dir, "model", `m${i}.go`), "package model\n" + `// ${"y".repeat(2000)}\n`);
+	}
+	await git(dir, "add", "-A");
+	await git(dir, "commit", "-q", "-m", "initial");
+	return dir;
+}
+
+function submittedCustomIds(payloads) {
+	return payloads.flatMap((p) => p.requests.map((r) => r.custom_id));
+}
+
+test("incremental submit scans only modules whose files changed", async () => {
+	const dir = await makeGitRepo();
+	try {
+		const payloads = [];
+		const fetcher = async (url, init) => {
+			if (init.method === "POST") {
+				payloads.push(JSON.parse(init.body));
+				return fakeResponse(202, { id: `batch-${payloads.length}`, status: "validating" });
+			}
+			return fakeResponse(200, { id: "x", status: "in_progress" });
+		};
+
+		await runBroadsideSubmit(dir, "sk-fake", { lenses: ["defect"], fetcher });
+		const firstIds = submittedCustomIds(payloads);
+		assert.ok(firstIds.some((id) => id.startsWith("defect-server")), "baseline must scan server");
+		assert.ok(firstIds.some((id) => id.startsWith("defect-model")), "baseline must scan model");
+
+		// Change one server file, commit, then re-scout incrementally.
+		await writeFile(join(dir, "server", "s0.go"), "package server\n// changed\n");
+		await git(dir, "add", "-A");
+		await git(dir, "commit", "-q", "-m", "change server");
+
+		payloads.length = 0;
+		await runBroadsideSubmit(dir, "sk-fake", { lenses: ["defect"], fetcher, incremental: true });
+		const secondIds = submittedCustomIds(payloads);
+		assert.ok(secondIds.some((id) => id.startsWith("defect-server")), "changed module must be re-scanned");
+		assert.ok(!secondIds.some((id) => id.startsWith("defect-model")), "unchanged module must be skipped");
+	} finally {
+		await rm(dir, { recursive: true, force: true });
+	}
+});
+
+test("incremental submit falls back to a full scan when the tree is dirty", async () => {
+	const dir = await makeGitRepo();
+	try {
+		const payloads = [];
+		const fetcher = async (url, init) => {
+			if (init.method === "POST") {
+				payloads.push(JSON.parse(init.body));
+				return fakeResponse(202, { id: `batch-${payloads.length}`, status: "validating" });
+			}
+			return fakeResponse(200, { id: "x", status: "in_progress" });
+		};
+
+		await runBroadsideSubmit(dir, "sk-fake", { lenses: ["defect"], fetcher });
+		// Uncommitted change → the diff is unreliable, so scan everything.
+		await writeFile(join(dir, "model", "m0.go"), "package model\n// dirty\n");
+
+		payloads.length = 0;
+		await runBroadsideSubmit(dir, "sk-fake", { lenses: ["defect"], fetcher, incremental: true });
+		const ids = submittedCustomIds(payloads);
+		assert.ok(ids.some((id) => id.startsWith("defect-server")), "dirty tree must still scan server");
+		assert.ok(ids.some((id) => id.startsWith("defect-model")), "dirty tree must still scan model");
+	} finally {
+		await rm(dir, { recursive: true, force: true });
+	}
+});
+
 // ---------- batch client with a fake fetcher ----------
 
 function fakeResponse(status, body) {