diff --git a/CHANGELOG.md b/CHANGELOG.md index 011a71c..1409487 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ All notable changes to this project are documented here. The format is based on ### Added +- **Broad-Side: truncated slices re-submit automatically** (#133). Submit now persists every request body to `.codecarto/broadside//requests.json`, and collect re-submits each truncated result once with a doubled output cap (bounded by the model's completion ceiling) — recovering coverage lost to a `max_tokens` cutoff instead of leaving the module silently unscouted. Recovered slices rewrite their JSON/markdown, clear their truncation flag, and report in the collect summary (`↻ N recovered`); anything still truncated after the retry stays flagged. Opt out with `retry_truncated: false` on collect. - **Broad-Side: per-language lens prompts** (#137). The defect and conventions lenses now build their system prompts from a language profile (Go, Python, Rust, TypeScript/JavaScript, plus a neutral default) instead of hardcoding Go idioms — a Python scanner no longer hears "goroutines without ctx"; it hears bare-except and context-manager checks. Convention extraction names language-appropriate categories (crates/modules vs modules/packages) and idiom hints. Language detection gains a `.js` bucket so JavaScript repos resolve to the TypeScript profile. Schemas are unchanged, so synthesis is unaffected. - **Broad-Side: lens batches poll concurrently** (#136). Collect previously polled one lens at a time to completion, so the slowest lens serialized the wall clock for lenses that had already finished server-side. In-flight batches now poll in parallel against one shared deadline via the new `pollBatchesConcurrently` helper, with progress callbacks tagged per lens; results still save in deterministic lens order. The poll interval is now injectable, which the new peak-concurrency regression tests use to prove the parallelism without timing flakiness. - **Broad-Side: batch reconnaissance over the OpenRouter Batch API** (#144). New `codecarto_broadside` MCP tool (actions: `submit`, `collect`, `status`) fires six single-turn analysis lenses — architecture, API surface, security, mechanical defect scan, convention extraction, porting — at any git repository as asynchronous batch jobs on a cheap batch model (~50% of sync pricing, unattended, 24h window), slices large modules by top-level directory, saves JSON plus rendered markdown to `.codecarto/broadside//`, and optionally synthesizes a cross-lens executive report. Works without an initialized workspace; needs an OpenRouter key via the `api_key` parameter, `OPENROUTER_API_KEY`, or `.codecarto/broadside/config.yaml`. Broad-Side findings are explicitly unverified scouting signals — file:line leads for the interactive pipeline to confirm, never evidence themselves. `codecarto_init` tolerates a `.codecarto/` that holds only `broadside/` (no force/backup needed), and scaffold refresh never touches broadside state, config, or results. diff --git a/ROADMAP.md b/ROADMAP.md index 4c9372d..d977348 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -34,7 +34,7 @@ file only moves when a tier completes. | Item | Issue | Notes | |---|---|---| | **Triage lens** — prioritized fix queue (impact × difficulty, grouped by module) | [#135](https://github.com/HuginnIndustries/CodeCartographer/issues/135) | **Shipped**: triage pass runs on collect alongside synthesis (`include_triage` to skip) | -| **Truncation repair** — detect max_tokens-cutoff JSON, resubmit slices, report truncation in summaries | [#133](https://github.com/HuginnIndustries/CodeCartographer/issues/133) | Partially shipped: fence-tolerant parsing + `truncated` flagging in collect, meta, and synthesis. Remaining: automatic resubmit of truncated slices | +| **Truncation repair** — detect max_tokens-cutoff JSON, resubmit slices, report truncation in summaries | [#133](https://github.com/HuginnIndustries/CodeCartographer/issues/133) | **Shipped**: fence-tolerant parsing + `truncated` flags + automatic re-submit of truncated slices with a doubled output cap | | **Concurrent polling** — poll all in-flight batches round-robin against one deadline | [#136](https://github.com/HuginnIndustries/CodeCartographer/issues/136) | **Shipped**: `pollBatchesConcurrently` polls in parallel with per-lens progress tags | | **Per-language prompts** — Go/Python/Rust/TS lens prompts; globs already adapt | [#137](https://github.com/HuginnIndustries/CodeCartographer/issues/137) | **Shipped**: language profiles drive defect/conventions prompts; schemas unchanged | diff --git a/core/broadside.ts b/core/broadside.ts index 18c87b2..bcbf5e9 100644 --- a/core/broadside.ts +++ b/core/broadside.ts @@ -199,6 +199,8 @@ export type BroadsideRun = { totalCost?: number; pricing?: ModelPricing; maxCost?: number; + /** The model's completion ceiling, recorded so collect can cap retries. */ + outputCap?: number; }; export type BroadsideStateFile = { @@ -241,6 +243,8 @@ export type BroadsideCollectResult = { /** Results whose JSON did not parse even after fence stripping — * the signature of an output cut off at max_tokens. */ truncatedCount: number; + /** Truncated slices recovered by the automatic re-submit pass (#133). */ + retriedCount: number; lensOutcomes: Partial< Record >; @@ -1833,16 +1837,19 @@ export async function runBroadsideSubmit( triage: { status: "pending" }, pricing, maxCost: limit > 0 ? limit : undefined, + outputCap, }; state.runs.push(run); await saveBroadsideState(broadsideDir, state); + const requestsByCustomId: Record = {}; const submissions: Promise[] = []; for (const lensId of lensIds) { const lens = getLens(lensId); const slices = slicesByLens.get(lensId) ?? []; const maxTokens = outputCap ? Math.min(lens.maxTokens, outputCap) : lens.maxTokens; const requests = slices.map((s, i) => buildBatchRequest(lens, info, s, i, slices.length, model, maxTokens)); + for (const request of requests) requestsByCustomId[request.custom_id] = request; const estimate = estimateCost(lens, slices, pricing, maxTokens); const entry: BroadsideBatchEntry = { @@ -1881,6 +1888,14 @@ export async function runBroadsideSubmit( await Promise.allSettled(submissions); await saveBroadsideState(broadsideDir, state); + // Persist the exact request bodies so collect can re-submit a truncated + // slice (bumped output cap) without re-walking the repo (#133). The run + // dir is created here rather than waiting for collect so a crash between + // submit and collect still leaves the retry input on disk. + const runDir = join(broadsideDir, runId); + await mkdir(runDir, { recursive: true }); + await writeFile(join(runDir, "requests.json"), `${JSON.stringify(requestsByCustomId, null, "\t")}\n`, "utf8"); + return { runId, outputDir: join(".codecarto", BROADSIDE_DIR, runId), @@ -1976,6 +1991,17 @@ export async function saveLensResults( return out; } +async function loadStoredRequests(runDir: string): Promise> { + const path = join(runDir, "requests.json"); + if (!(await pathExists(path))) return {}; + try { + const parsed = JSON.parse(await readFile(path, "utf8")) as Record; + return parsed && typeof parsed === "object" ? parsed : {}; + } catch { + return {}; + } +} + // ---------- post-lens passes: synthesis + triage ---------- function buildSynthesisRequest(findingsText: string, truncatedNote: string, model: string): BatchRequest { @@ -2075,6 +2101,8 @@ export async function runBroadsideCollect( waitMs?: number; includeSynthesis?: boolean; includeTriage?: boolean; + /** Re-submit truncated slices once with a doubled output cap (#133). */ + retryTruncated?: boolean; onStatus?: (lensId: string, status: string, counts: Record) => void; fetcher?: FetchLike; } = {}, @@ -2154,6 +2182,61 @@ export async function runBroadsideCollect( await saveBroadsideState(broadsideDir, state); } + // #133: re-submit truncated slices once with a bumped output cap. Batch + // requests are pure, so re-running is always safe; the aim is to recover + // coverage the first pass lost to a max_tokens cutoff, not to loop forever. + let retriedCount = 0; + if (opts.retryTruncated !== false && truncatedCount > 0) { + const requestsByCustomId = await loadStoredRequests(runDir); + for (const stored of allLensResults) { + if (!stored.truncated) continue; + const original = requestsByCustomId[stored.customId]; + if (!original) continue; + const previousMax = original.body.max_tokens ?? getLens(stored.lensId).maxTokens; + const bumpedMax = run.outputCap ? Math.min(previousMax * 2, run.outputCap) : previousMax * 2; + if (bumpedMax <= previousMax) continue; // already at the ceiling + + const bumped: BatchRequest = { + ...original, + body: { ...original.body, max_tokens: bumpedMax }, + }; + try { + const { batchId, error } = await submitBatch([bumped], apiKey, opts.fetcher, run.model); + if (error) continue; + const batch = await pollBatchUntilTerminal(batchId, apiKey, { + deadlineMs: BROADSIDE_DEFAULT_POLL_BUDGET_MS, + onStatus: (status, counts) => opts.onStatus?.(`${stored.lensId}:retry`, status, counts), + fetcher: opts.fetcher, + }); + if (batch.status !== "completed") continue; + const results = Array.isArray(batch.results) ? (batch.results as Array>) : []; + const content = results.length > 0 ? extractContent(results[0]) : null; + if (content === null || parseLensJson(content) === null) continue; // still no good + + const usage = (batch.usage ?? {}) as Record; + totalCost += typeof usage.cost === "number" ? usage.cost : 0; + + const parsed = parseLensJson(content); + await writeFile(join(runDir, `${sanitizeId(stored.customId)}.json`), `${JSON.stringify(parsed, null, "\t")}\n`, "utf8"); + await writeFile(join(runDir, `${sanitizeId(stored.customId)}.md`), renderFindingsMarkdown(content), "utf8"); + + stored.content = content; + stored.truncated = false; + retriedCount += 1; + } catch { + // A retry that fails to submit/poll leaves the original + // truncated result in place — nothing is lost. + } + } + truncatedCount = allLensResults.filter((s) => s.truncated).length; + for (const [lensId, outcome] of Object.entries(lensOutcomes)) { + if (outcome.truncated !== undefined) { + outcome.truncated = allLensResults.filter((s) => s.lensId === lensId && s.truncated).length; + } + } + await saveBroadsideState(broadsideDir, state); + } + // Synthesis + triage: cross-lens post-passes, only after every lens batch // is terminal. Triage turns the leads into a prioritized work order. run.triage ??= { status: "pending" }; @@ -2274,6 +2357,7 @@ export async function runBroadsideCollect( total_cost: totalCost, result_count: resultCount, truncated_count: truncatedCount, + retried_count: retriedCount, synthesis: run.synthesis, triage: run.triage, lenses: run.lenses, @@ -2293,6 +2377,7 @@ export async function runBroadsideCollect( totalCost, resultCount, truncatedCount, + retriedCount, lensOutcomes, synthesis: run.synthesis, triage: run.triage, @@ -2455,9 +2540,12 @@ export function collectResultText(result: BroadsideCollectResult): string { truncation, ); } + if (result.retriedCount > 0) { + lines.push(` ↻ ${result.retriedCount} truncated result(s) recovered by re-submission with a doubled output cap.`); + } if (result.truncatedCount > 0) { lines.push( - ` ⚠ ${result.truncatedCount} result(s) truncated at the output limit — their modules are unscouted, not clean.`, + ` ⚠ ${result.truncatedCount} result(s) still truncated after retry — their modules are unscouted, not clean.`, ); } if (result.synthesis.status === "completed") { diff --git a/mcp-server/server.ts b/mcp-server/server.ts index 64f9aab..f505d16 100644 --- a/mcp-server/server.ts +++ b/mcp-server/server.ts @@ -1002,6 +1002,7 @@ export async function handleBroadside(args: { wait_seconds?: number; include_synthesis?: boolean; include_triage?: boolean; + retry_truncated?: boolean; max_cost?: number; force?: boolean; include_benchmarks?: boolean; @@ -1065,6 +1066,7 @@ export async function handleBroadside(args: { waitMs, includeSynthesis: args.include_synthesis !== false, includeTriage: args.include_triage !== false, + retryTruncated: args.retry_truncated !== false, onStatus: (lensId, status, counts) => lines.push(` ${lensId}: ${status} (${counts.completed ?? 0}/${counts.total ?? "?"})`), }); @@ -1085,6 +1087,7 @@ export async function handleBroadside(args: { waitMs, includeSynthesis: args.include_synthesis !== false, includeTriage: args.include_triage !== false, + retryTruncated: args.retry_truncated !== false, }).catch((error) => { throw new McpError(ErrorCode.InvalidRequest, error instanceof Error ? error.message : String(error)); }); @@ -1094,6 +1097,7 @@ export async function handleBroadside(args: { totalCost: collect.totalCost, resultCount: collect.resultCount, truncatedCount: collect.truncatedCount, + retriedCount: collect.retriedCount, lensOutcomes: collect.lensOutcomes, synthesis: collect.synthesis, triage: collect.triage, @@ -1430,6 +1434,11 @@ const TOOLS = [ description: "Run the triage pass once all lens batches complete: turns the findings into a prioritized work order (impact × difficulty, P0-P3, effort estimates). Default true.", }, + retry_truncated: { + type: "boolean", + description: + "Re-submit lens results that came back truncated at the output token limit, once, with a doubled output cap. Default true.", + }, max_cost: { type: "number", description: diff --git a/tests/broadside.test.mjs b/tests/broadside.test.mjs index 3f5c813..3b6971a 100644 --- a/tests/broadside.test.mjs +++ b/tests/broadside.test.mjs @@ -563,6 +563,90 @@ test("modelsText renders pricing, caps, support, and benchmark columns", () => { assert.match(text, /\(default\)/); }); +// ---------- truncated-slice resubmit (#133) ---------- + +test("submit persists request bodies for truncated-slice recovery", async () => { + const dir = await makeFixture(); + try { + const fetcher = async (url, init) => + init.method === "POST" ? fakeResponse(202, { id: "batch-x", status: "validating" }) : fakeResponse(200, { id: "x", status: "in_progress" }); + const result = await runBroadsideSubmit(dir, "sk-fake", { lenses: ["architecture"], fetcher }); + const runDir = join(dir, ".codecarto", "broadside", result.outputDir.split("/").pop()); + const requests = JSON.parse(await readFile(join(runDir, "requests.json"), "utf8")); + assert.ok(requests["architecture-root"], "architecture request must be persisted"); + assert.equal(requests["architecture-root"].body.model, BROADSIDE_MODEL); + } finally { + await rm(dir, { recursive: true, force: true }); + } +}); + +test("collect re-submits truncated slices once with a doubled output cap", async () => { + const dir = await makeFixture(); + try { + const truncated = '{"module": "server", "findings": ['; + const recovered = JSON.stringify({ module: "server", findings: [], patterns_checked: [], files_scanned: 0 }); + const retryPayloads = []; + const fetcher = async (url, init) => { + if (init.method === "POST") { + const payload = JSON.parse(init.body); + const isRetry = retryPayloads.length > 0; + retryPayloads.push(payload); + return fakeResponse(202, { id: isRetry ? "batch-retry" : "batch-lens", status: "validating" }); + } + if (String(url).includes("batch-lens")) { + return fakeResponse(200, { + id: "batch-lens", + status: "completed", + results: [{ custom_id: "architecture-root", response: { status_code: 200, body: { choices: [{ message: { content: truncated } }] } }, error: null }], + usage: { cost: 0.001 }, + }); + } + return fakeResponse(200, { + id: "batch-retry", + status: "completed", + results: [{ custom_id: "architecture-root", response: { status_code: 200, body: { choices: [{ message: { content: recovered } }] } }, error: null }], + usage: { cost: 0.002 }, + }); + }; + + await runBroadsideSubmit(dir, "sk-fake", { lenses: ["architecture"], fetcher }); + const collect = await runBroadsideCollect(dir, "sk-fake", { fetcher, includeSynthesis: false, includeTriage: false }); + + assert.equal(collect.truncatedCount, 0, "recovered slice must clear the truncation count"); + assert.equal(collect.retriedCount, 1, "one slice recovered by resubmission"); + assert.equal(retryPayloads.length, 2, "one original submit + one retry"); + assert.equal(retryPayloads[1].requests[0].body.max_tokens, retryPayloads[0].requests[0].body.max_tokens * 2, "retry must double the output cap"); + } finally { + await rm(dir, { recursive: true, force: true }); + } +}); + +test("collect leaves truncated slices alone when retry_truncated is false", async () => { + const dir = await makeFixture(); + try { + const truncated = '{"module": "server", "findings": ['; + const fetcher = async (url, init) => { + if (init.method === "POST") { + return fakeResponse(202, { id: "batch-lens", status: "validating" }); + } + return fakeResponse(200, { + id: "batch-lens", + status: "completed", + results: [{ custom_id: "architecture-root", response: { status_code: 200, body: { choices: [{ message: { content: truncated } }] } }, error: null }], + usage: { cost: 0.001 }, + }); + }; + + await runBroadsideSubmit(dir, "sk-fake", { lenses: ["architecture"], fetcher }); + const collect = await runBroadsideCollect(dir, "sk-fake", { fetcher, includeSynthesis: false, includeTriage: false, retryTruncated: false }); + + assert.equal(collect.truncatedCount, 1, "truncation must remain reported"); + assert.equal(collect.retriedCount, 0, "no resubmission when retry_truncated is false"); + } finally { + await rm(dir, { recursive: true, force: true }); + } +}); + // ---------- per-language prompts (#137) ---------- test("defect lens prompt speaks the detected language, not Go", () => {