feat: Broad-Side re-submits truncated slices automatically (#133)

CHANGELOG.md

@@ -6,6 +6,7 @@ All notable changes to this project are documented here. The format is based on
 
 ### Added
 
+- **Broad-Side: truncated slices re-submit automatically** (#133). Submit now persists every request body to `.codecarto/broadside/<run>/requests.json`, and collect re-submits each truncated result once with a doubled output cap (bounded by the model's completion ceiling) — recovering coverage lost to a `max_tokens` cutoff instead of leaving the module silently unscouted. Recovered slices rewrite their JSON/markdown, clear their truncation flag, and report in the collect summary (`↻ N recovered`); anything still truncated after the retry stays flagged. Opt out with `retry_truncated: false` on collect.
 - **Broad-Side: per-language lens prompts** (#137). The defect and conventions lenses now build their system prompts from a language profile (Go, Python, Rust, TypeScript/JavaScript, plus a neutral default) instead of hardcoding Go idioms — a Python scanner no longer hears "goroutines without ctx"; it hears bare-except and context-manager checks. Convention extraction names language-appropriate categories (crates/modules vs modules/packages) and idiom hints. Language detection gains a `.js` bucket so JavaScript repos resolve to the TypeScript profile. Schemas are unchanged, so synthesis is unaffected.
 - **Broad-Side: lens batches poll concurrently** (#136). Collect previously polled one lens at a time to completion, so the slowest lens serialized the wall clock for lenses that had already finished server-side. In-flight batches now poll in parallel against one shared deadline via the new `pollBatchesConcurrently` helper, with progress callbacks tagged per lens; results still save in deterministic lens order. The poll interval is now injectable, which the new peak-concurrency regression tests use to prove the parallelism without timing flakiness.
 - **Broad-Side: batch reconnaissance over the OpenRouter Batch API** (#144). New `codecarto_broadside` MCP tool (actions: `submit`, `collect`, `status`) fires six single-turn analysis lenses — architecture, API surface, security, mechanical defect scan, convention extraction, porting — at any git repository as asynchronous batch jobs on a cheap batch model (~50% of sync pricing, unattended, 24h window), slices large modules by top-level directory, saves JSON plus rendered markdown to `.codecarto/broadside/<run>/`, and optionally synthesizes a cross-lens executive report. Works without an initialized workspace; needs an OpenRouter key via the `api_key` parameter, `OPENROUTER_API_KEY`, or `.codecarto/broadside/config.yaml`. Broad-Side findings are explicitly unverified scouting signals — file:line leads for the interactive pipeline to confirm, never evidence themselves. `codecarto_init` tolerates a `.codecarto/` that holds only `broadside/` (no force/backup needed), and scaffold refresh never touches broadside state, config, or results.

ROADMAP.md

@@ -34,7 +34,7 @@ file only moves when a tier completes.
 | Item | Issue | Notes |
 |---|---|---|
 | **Triage lens** — prioritized fix queue (impact × difficulty, grouped by module) | [#135](https://github.com/HuginnIndustries/CodeCartographer/issues/135) | **Shipped**: triage pass runs on collect alongside synthesis (`include_triage` to skip) |
-| **Truncation repair** — detect max_tokens-cutoff JSON, resubmit slices, report truncation in summaries | [#133](https://github.com/HuginnIndustries/CodeCartographer/issues/133) | Partially shipped: fence-tolerant parsing + `truncated` flagging in collect, meta, and synthesis. Remaining: automatic resubmit of truncated slices |
+| **Truncation repair** — detect max_tokens-cutoff JSON, resubmit slices, report truncation in summaries | [#133](https://github.com/HuginnIndustries/CodeCartographer/issues/133) | **Shipped**: fence-tolerant parsing + `truncated` flags + automatic re-submit of truncated slices with a doubled output cap |
 | **Concurrent polling** — poll all in-flight batches round-robin against one deadline | [#136](https://github.com/HuginnIndustries/CodeCartographer/issues/136) | **Shipped**: `pollBatchesConcurrently` polls in parallel with per-lens progress tags |
 | **Per-language prompts** — Go/Python/Rust/TS lens prompts; globs already adapt | [#137](https://github.com/HuginnIndustries/CodeCartographer/issues/137) | **Shipped**: language profiles drive defect/conventions prompts; schemas unchanged |
 

core/broadside.ts

@@ -199,6 +199,8 @@ export type BroadsideRun = {
 	totalCost?: number;
 	pricing?: ModelPricing;
 	maxCost?: number;
+	/** The model's completion ceiling, recorded so collect can cap retries. */
+	outputCap?: number;
 };
 
 export type BroadsideStateFile = {
@@ -241,6 +243,8 @@ export type BroadsideCollectResult = {
 	/** Results whose JSON did not parse even after fence stripping —
 	 * the signature of an output cut off at max_tokens. */
 	truncatedCount: number;
+	/** Truncated slices recovered by the automatic re-submit pass (#133). */
+	retriedCount: number;
 	lensOutcomes: Partial<
 		Record<BroadsideLensId, { status: string; cost?: number; resultCount?: number; truncated?: number }>
 	>;
@@ -1833,16 +1837,19 @@ export async function runBroadsideSubmit(
 		triage: { status: "pending" },
 		pricing,
 		maxCost: limit > 0 ? limit : undefined,
+		outputCap,
 	};
 	state.runs.push(run);
 	await saveBroadsideState(broadsideDir, state);
 
+	const requestsByCustomId: Record<string, BatchRequest> = {};
 	const submissions: Promise<void>[] = [];
 	for (const lensId of lensIds) {
 		const lens = getLens(lensId);
 		const slices = slicesByLens.get(lensId) ?? [];
 		const maxTokens = outputCap ? Math.min(lens.maxTokens, outputCap) : lens.maxTokens;
 		const requests = slices.map((s, i) => buildBatchRequest(lens, info, s, i, slices.length, model, maxTokens));
+		for (const request of requests) requestsByCustomId[request.custom_id] = request;
 		const estimate = estimateCost(lens, slices, pricing, maxTokens);
 
 		const entry: BroadsideBatchEntry = {
@@ -1881,6 +1888,14 @@ export async function runBroadsideSubmit(
 	await Promise.allSettled(submissions);
 	await saveBroadsideState(broadsideDir, state);
 
+	// Persist the exact request bodies so collect can re-submit a truncated
+	// slice (bumped output cap) without re-walking the repo (#133). The run
+	// dir is created here rather than waiting for collect so a crash between
+	// submit and collect still leaves the retry input on disk.
+	const runDir = join(broadsideDir, runId);
+	await mkdir(runDir, { recursive: true });
+	await writeFile(join(runDir, "requests.json"), `${JSON.stringify(requestsByCustomId, null, "\t")}\n`, "utf8");
+
 	return {
 		runId,
 		outputDir: join(".codecarto", BROADSIDE_DIR, runId),
@@ -1976,6 +1991,17 @@ export async function saveLensResults(
 	return out;
 }
 
+async function loadStoredRequests(runDir: string): Promise<Record<string, BatchRequest>> {
+	const path = join(runDir, "requests.json");
+	if (!(await pathExists(path))) return {};
+	try {
+		const parsed = JSON.parse(await readFile(path, "utf8")) as Record<string, BatchRequest>;
+		return parsed && typeof parsed === "object" ? parsed : {};
+	} catch {
+		return {};
+	}
+}
+
 // ---------- post-lens passes: synthesis + triage ----------
 
 function buildSynthesisRequest(findingsText: string, truncatedNote: string, model: string): BatchRequest {
@@ -2075,6 +2101,8 @@ export async function runBroadsideCollect(
 		waitMs?: number;
 		includeSynthesis?: boolean;
 		includeTriage?: boolean;
+		/** Re-submit truncated slices once with a doubled output cap (#133). */
+		retryTruncated?: boolean;
 		onStatus?: (lensId: string, status: string, counts: Record<string, unknown>) => void;
 		fetcher?: FetchLike;
 	} = {},
@@ -2154,6 +2182,61 @@ export async function runBroadsideCollect(
 		await saveBroadsideState(broadsideDir, state);
 	}
 
+	// #133: re-submit truncated slices once with a bumped output cap. Batch
+	// requests are pure, so re-running is always safe; the aim is to recover
+	// coverage the first pass lost to a max_tokens cutoff, not to loop forever.
+	let retriedCount = 0;
+	if (opts.retryTruncated !== false && truncatedCount > 0) {
+		const requestsByCustomId = await loadStoredRequests(runDir);
+		for (const stored of allLensResults) {
+			if (!stored.truncated) continue;
+			const original = requestsByCustomId[stored.customId];
+			if (!original) continue;
+			const previousMax = original.body.max_tokens ?? getLens(stored.lensId).maxTokens;
+			const bumpedMax = run.outputCap ? Math.min(previousMax * 2, run.outputCap) : previousMax * 2;
+			if (bumpedMax <= previousMax) continue; // already at the ceiling
+
+			const bumped: BatchRequest = {
+				...original,
+				body: { ...original.body, max_tokens: bumpedMax },
+			};
+			try {
+				const { batchId, error } = await submitBatch([bumped], apiKey, opts.fetcher, run.model);
+				if (error) continue;
+				const batch = await pollBatchUntilTerminal(batchId, apiKey, {
+					deadlineMs: BROADSIDE_DEFAULT_POLL_BUDGET_MS,
+					onStatus: (status, counts) => opts.onStatus?.(`${stored.lensId}:retry`, status, counts),
+					fetcher: opts.fetcher,
+				});
+				if (batch.status !== "completed") continue;
+				const results = Array.isArray(batch.results) ? (batch.results as Array<Record<string, unknown>>) : [];
+				const content = results.length > 0 ? extractContent(results[0]) : null;
+				if (content === null || parseLensJson(content) === null) continue; // still no good
+
+				const usage = (batch.usage ?? {}) as Record<string, unknown>;
+				totalCost += typeof usage.cost === "number" ? usage.cost : 0;
+
+				const parsed = parseLensJson(content);
+				await writeFile(join(runDir, `${sanitizeId(stored.customId)}.json`), `${JSON.stringify(parsed, null, "\t")}\n`, "utf8");
+				await writeFile(join(runDir, `${sanitizeId(stored.customId)}.md`), renderFindingsMarkdown(content), "utf8");
+
+				stored.content = content;
+				stored.truncated = false;
+				retriedCount += 1;
+			} catch {
+				// A retry that fails to submit/poll leaves the original
+				// truncated result in place — nothing is lost.
+			}
+		}
+		truncatedCount = allLensResults.filter((s) => s.truncated).length;
+		for (const [lensId, outcome] of Object.entries(lensOutcomes)) {
+			if (outcome.truncated !== undefined) {
+				outcome.truncated = allLensResults.filter((s) => s.lensId === lensId && s.truncated).length;
+			}
+		}
+		await saveBroadsideState(broadsideDir, state);
+	}
+
 	// Synthesis + triage: cross-lens post-passes, only after every lens batch
 	// is terminal. Triage turns the leads into a prioritized work order.
 	run.triage ??= { status: "pending" };
@@ -2274,6 +2357,7 @@ export async function runBroadsideCollect(
 				total_cost: totalCost,
 				result_count: resultCount,
 				truncated_count: truncatedCount,
+				retried_count: retriedCount,
 				synthesis: run.synthesis,
 				triage: run.triage,
 				lenses: run.lenses,
@@ -2293,6 +2377,7 @@ export async function runBroadsideCollect(
 		totalCost,
 		resultCount,
 		truncatedCount,
+		retriedCount,
 		lensOutcomes,
 		synthesis: run.synthesis,
 		triage: run.triage,
@@ -2455,9 +2540,12 @@ export function collectResultText(result: BroadsideCollectResult): string {
 				truncation,
 		);
 	}
+	if (result.retriedCount > 0) {
+		lines.push(`  ↻ ${result.retriedCount} truncated result(s) recovered by re-submission with a doubled output cap.`);
+	}
 	if (result.truncatedCount > 0) {
 		lines.push(
-			`  ⚠ ${result.truncatedCount} result(s) truncated at the output limit — their modules are unscouted, not clean.`,
+			`  ⚠ ${result.truncatedCount} result(s) still truncated after retry — their modules are unscouted, not clean.`,
 		);
 	}
 	if (result.synthesis.status === "completed") {

mcp-server/server.ts

@@ -1002,6 +1002,7 @@ export async function handleBroadside(args: {
 	wait_seconds?: number;
 	include_synthesis?: boolean;
 	include_triage?: boolean;
+	retry_truncated?: boolean;
 	max_cost?: number;
 	force?: boolean;
 	include_benchmarks?: boolean;
@@ -1065,6 +1066,7 @@ export async function handleBroadside(args: {
 				waitMs,
 				includeSynthesis: args.include_synthesis !== false,
 				includeTriage: args.include_triage !== false,
+				retryTruncated: args.retry_truncated !== false,
 				onStatus: (lensId, status, counts) =>
 					lines.push(`  ${lensId}: ${status} (${counts.completed ?? 0}/${counts.total ?? "?"})`),
 			});
@@ -1085,6 +1087,7 @@ export async function handleBroadside(args: {
 		waitMs,
 		includeSynthesis: args.include_synthesis !== false,
 		includeTriage: args.include_triage !== false,
+		retryTruncated: args.retry_truncated !== false,
 	}).catch((error) => {
 		throw new McpError(ErrorCode.InvalidRequest, error instanceof Error ? error.message : String(error));
 	});
@@ -1094,6 +1097,7 @@ export async function handleBroadside(args: {
 		totalCost: collect.totalCost,
 		resultCount: collect.resultCount,
 		truncatedCount: collect.truncatedCount,
+		retriedCount: collect.retriedCount,
 		lensOutcomes: collect.lensOutcomes,
 		synthesis: collect.synthesis,
 		triage: collect.triage,
@@ -1430,6 +1434,11 @@ const TOOLS = [
 					description:
 						"Run the triage pass once all lens batches complete: turns the findings into a prioritized work order (impact × difficulty, P0-P3, effort estimates). Default true.",
 				},
+				retry_truncated: {
+					type: "boolean",
+					description:
+						"Re-submit lens results that came back truncated at the output token limit, once, with a doubled output cap. Default true.",
+				},
 				max_cost: {
 					type: "number",
 					description:

tests/broadside.test.mjs

@@ -563,6 +563,90 @@ test("modelsText renders pricing, caps, support, and benchmark columns", () => {
 	assert.match(text, /\(default\)/);
 });
 
+// ---------- truncated-slice resubmit (#133) ----------
+
+test("submit persists request bodies for truncated-slice recovery", async () => {
+	const dir = await makeFixture();
+	try {
+		const fetcher = async (url, init) =>
+			init.method === "POST" ? fakeResponse(202, { id: "batch-x", status: "validating" }) : fakeResponse(200, { id: "x", status: "in_progress" });
+		const result = await runBroadsideSubmit(dir, "sk-fake", { lenses: ["architecture"], fetcher });
+		const runDir = join(dir, ".codecarto", "broadside", result.outputDir.split("/").pop());
+		const requests = JSON.parse(await readFile(join(runDir, "requests.json"), "utf8"));
+		assert.ok(requests["architecture-root"], "architecture request must be persisted");
+		assert.equal(requests["architecture-root"].body.model, BROADSIDE_MODEL);
+	} finally {
+		await rm(dir, { recursive: true, force: true });
+	}
+});
+
+test("collect re-submits truncated slices once with a doubled output cap", async () => {
+	const dir = await makeFixture();
+	try {
+		const truncated = '{"module": "server", "findings": [';
+		const recovered = JSON.stringify({ module: "server", findings: [], patterns_checked: [], files_scanned: 0 });
+		const retryPayloads = [];
+		const fetcher = async (url, init) => {
+			if (init.method === "POST") {
+				const payload = JSON.parse(init.body);
+				const isRetry = retryPayloads.length > 0;
+				retryPayloads.push(payload);
+				return fakeResponse(202, { id: isRetry ? "batch-retry" : "batch-lens", status: "validating" });
+			}
+			if (String(url).includes("batch-lens")) {
+				return fakeResponse(200, {
+					id: "batch-lens",
+					status: "completed",
+					results: [{ custom_id: "architecture-root", response: { status_code: 200, body: { choices: [{ message: { content: truncated } }] } }, error: null }],
+					usage: { cost: 0.001 },
+				});
+			}
+			return fakeResponse(200, {
+				id: "batch-retry",
+				status: "completed",
+				results: [{ custom_id: "architecture-root", response: { status_code: 200, body: { choices: [{ message: { content: recovered } }] } }, error: null }],
+				usage: { cost: 0.002 },
+			});
+		};
+
+		await runBroadsideSubmit(dir, "sk-fake", { lenses: ["architecture"], fetcher });
+		const collect = await runBroadsideCollect(dir, "sk-fake", { fetcher, includeSynthesis: false, includeTriage: false });
+
+		assert.equal(collect.truncatedCount, 0, "recovered slice must clear the truncation count");
+		assert.equal(collect.retriedCount, 1, "one slice recovered by resubmission");
+		assert.equal(retryPayloads.length, 2, "one original submit + one retry");
+		assert.equal(retryPayloads[1].requests[0].body.max_tokens, retryPayloads[0].requests[0].body.max_tokens * 2, "retry must double the output cap");
+	} finally {
+		await rm(dir, { recursive: true, force: true });
+	}
+});
+
+test("collect leaves truncated slices alone when retry_truncated is false", async () => {
+	const dir = await makeFixture();
+	try {
+		const truncated = '{"module": "server", "findings": [';
+		const fetcher = async (url, init) => {
+			if (init.method === "POST") {
+				return fakeResponse(202, { id: "batch-lens", status: "validating" });
+			}
+			return fakeResponse(200, {
+				id: "batch-lens",
+				status: "completed",
+				results: [{ custom_id: "architecture-root", response: { status_code: 200, body: { choices: [{ message: { content: truncated } }] } }, error: null }],
+				usage: { cost: 0.001 },
+			});
+		};
+
+		await runBroadsideSubmit(dir, "sk-fake", { lenses: ["architecture"], fetcher });
+		const collect = await runBroadsideCollect(dir, "sk-fake", { fetcher, includeSynthesis: false, includeTriage: false, retryTruncated: false });
+
+		assert.equal(collect.truncatedCount, 1, "truncation must remain reported");
+		assert.equal(collect.retriedCount, 0, "no resubmission when retry_truncated is false");
+	} finally {
+		await rm(dir, { recursive: true, force: true });
+	}
+});
+
 // ---------- per-language prompts (#137) ----------
 
 test("defect lens prompt speaks the detected language, not Go", () => {