diff --git a/CHANGELOG.md b/CHANGELOG.md index 1409487..485ea2c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ All notable changes to this project are documented here. The format is based on ### Added +- **Broad-Side: zero-config slicing** (#140). The defect, conventions, and porting lenses now slice by `auto` instead of always per-directory: a repo whose matching files fit within the lens's char cap collapses to a single whole-repo slice (one request instead of one per module), while a repo too large for one slice still splits by top-level directory. Small repos stop paying for per-module request overhead; large repos keep full coverage. The decision is deterministic from file sizes and recorded implicitly in the slice layout. - **Broad-Side: truncated slices re-submit automatically** (#133). Submit now persists every request body to `.codecarto/broadside//requests.json`, and collect re-submits each truncated result once with a doubled output cap (bounded by the model's completion ceiling) — recovering coverage lost to a `max_tokens` cutoff instead of leaving the module silently unscouted. Recovered slices rewrite their JSON/markdown, clear their truncation flag, and report in the collect summary (`↻ N recovered`); anything still truncated after the retry stays flagged. Opt out with `retry_truncated: false` on collect. - **Broad-Side: per-language lens prompts** (#137). The defect and conventions lenses now build their system prompts from a language profile (Go, Python, Rust, TypeScript/JavaScript, plus a neutral default) instead of hardcoding Go idioms — a Python scanner no longer hears "goroutines without ctx"; it hears bare-except and context-manager checks. Convention extraction names language-appropriate categories (crates/modules vs modules/packages) and idiom hints. Language detection gains a `.js` bucket so JavaScript repos resolve to the TypeScript profile. Schemas are unchanged, so synthesis is unaffected. - **Broad-Side: lens batches poll concurrently** (#136). Collect previously polled one lens at a time to completion, so the slowest lens serialized the wall clock for lenses that had already finished server-side. In-flight batches now poll in parallel against one shared deadline via the new `pollBatchesConcurrently` helper, with progress callbacks tagged per lens; results still save in deterministic lens order. The poll interval is now injectable, which the new peak-concurrency regression tests use to prove the parallelism without timing flakiness. diff --git a/ROADMAP.md b/ROADMAP.md index d977348..431a4d8 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -44,7 +44,7 @@ file only moves when a tier completes. |---|---|---| | **Pi extension** — `/codecarto-broadside` command with lens picker and live progress | [#138](https://github.com/HuginnIndustries/CodeCartographer/issues/138) | Agreed order: MCP first (shipped), Pi second | | **Pipeline phase** — `broadside-scout` phase feeding later phases via `required_reads` | [#139](https://github.com/HuginnIndustries/CodeCartographer/issues/139) | SKILL.md contract stays: leads, never evidence | -| **Zero-config executive** — meta-pass picks lenses and slicing resolution from repo shape | [#140](https://github.com/HuginnIndustries/CodeCartographer/issues/140) | Decision recorded in `run-meta.json` for reproducibility | +| **Zero-config executive** — meta-pass picks lenses and slicing resolution from repo shape | [#140](https://github.com/HuginnIndustries/CodeCartographer/issues/140) | **Shipped**: `auto` slicing collapses small repos to one slice, directory-splits large ones | ## Tier 3 — cost and coverage economics diff --git a/core/broadside.ts b/core/broadside.ts index bcbf5e9..40cd467 100644 --- a/core/broadside.ts +++ b/core/broadside.ts @@ -30,7 +30,7 @@ // Deliberately not in .codecarto/ template prose: Broad-Side requires runtime // code, so it lives on the executable surfaces (MCP today, Pi on the roadmap). -import { mkdir, readFile, readdir, writeFile } from "node:fs/promises"; +import { mkdir, readFile, readdir, stat, writeFile } from "node:fs/promises"; import { execFile } from "node:child_process"; import { promisify } from "node:util"; import { dirname, join } from "node:path"; @@ -726,7 +726,10 @@ type LensDefinition = { name: string; description: string; schemaName: string; - sliceBy: "none" | "directory"; + // "none" = one slice for the whole repo; "directory" = one slice per + // top-level module; "auto" = directory for large repos, none for small + // ones (see resolveSliceMode). + sliceBy: "none" | "directory" | "auto"; maxChars: number; maxTokens: number; // Test files rarely carry the surface a lens audits — they bulk up the @@ -835,7 +838,7 @@ const LENSES: Record = { name: "Mechanical defect scan", description: "Nil derefs, error gaps, leaks, races, panics — pattern-based, sliced per module.", schemaName: "defect_mechanical", - sliceBy: "directory", + sliceBy: "auto", maxChars: 60_000, maxTokens: 6000, globsFor: (info) => [info.sourceGlob], @@ -865,7 +868,7 @@ const LENSES: Record = { name: "Convention extraction", description: "Naming, error handling, idioms, inconsistencies, promotable conventions.", schemaName: "conventions", - sliceBy: "directory", + sliceBy: "auto", maxChars: 60_000, maxTokens: 6000, globsFor: (info) => [info.sourceGlob], @@ -899,7 +902,7 @@ const LENSES: Record = { name: "Porting surface assessment", description: "Platform coupling, external deps, build complexity, porting risk areas.", schemaName: "porting", - sliceBy: "directory", + sliceBy: "auto", maxChars: 60_000, maxTokens: 6000, skipTestFiles: true, @@ -1231,6 +1234,18 @@ function isTestFile(relPath: string): boolean { return /[._](test|spec)\.[a-z]+$/i.test(base) || base.includes("_test."); } +/** + * "auto" slicing: directory-slice when the repo is large enough that a + * single whole-repo slice would overflow the lens's char cap, otherwise a + * single slice. The threshold is the lens's own cap — a repo whose matching + * files fit in one slice gains nothing from per-module splitting, and a + * small repo pays for it in extra requests. + */ +function resolveSliceMode(lens: LensDefinition, files: CollectedFile[], totalChars: number): "none" | "directory" { + if (lens.sliceBy !== "auto") return lens.sliceBy; + return totalChars > lens.maxChars ? "directory" : "none"; +} + function collectLensFiles(allFiles: string[], lens: LensDefinition, info: RepoInfo): CollectedFile[] { const globs = lens.globsFor(info); if (globs.length === 0) return []; @@ -1239,7 +1254,7 @@ function collectLensFiles(allFiles: string[], lens: LensDefinition, info: RepoIn if (!isSlurpable(f)) continue; if (lens.skipTestFiles && isTestFile(f)) continue; if (!matchesAnyGlob(f, globs)) continue; - out.push({ relPath: f, moduleName: lens.sliceBy === "directory" ? topLevelModule(f) : info.name }); + out.push({ relPath: f, moduleName: topLevelModule(f) }); } return out; } @@ -1303,9 +1318,29 @@ export async function gatherSlices(targetDir: string, lens: LensDefinition, info } const allFiles = await listRepoFiles(targetDir); const files = collectLensFiles(allFiles, lens, info); + const totalChars = await sumFileSizes(targetDir, files); + const mode = resolveSliceMode(lens, files, totalChars); + if (mode === "none") { + // Whole-repo slice: one module named after the repo, so a small + // repo produces a single request instead of one per directory. + const single = files.map((f) => ({ ...f, moduleName: info.name })); + return slurpFileList(targetDir, single, lens.maxChars); + } return slurpFileList(targetDir, files, lens.maxChars); } +async function sumFileSizes(targetDir: string, files: CollectedFile[]): Promise { + let total = 0; + for (const f of files) { + try { + total += (await stat(join(targetDir, f.relPath))).size; + } catch { + // Unreadable file — slurpFileList substitutes a placeholder. + } + } + return total; +} + // ---------- request building ---------- export function buildBatchRequest( diff --git a/tests/broadside.test.mjs b/tests/broadside.test.mjs index 3b6971a..3dde27e 100644 --- a/tests/broadside.test.mjs +++ b/tests/broadside.test.mjs @@ -88,9 +88,33 @@ test("collectRepoInfo detects Go and gathers manifest, tree, and counts", async } }); -test("directory slicing puts nested files under their top-level module", async () => { +test("auto slicing collapses a small repo to a single whole-repo slice", async () => { const dir = await makeFixture(); try { + const info = await collectRepoInfo(dir); + const lens = getLens("defect"); + const slices = await gatherSlices(dir, lens, info); + assert.equal(slices.length, 1, "a repo that fits one slice must not be split per directory"); + assert.equal(slices[0].moduleName, info.name); + assert.match(slices[0].content, /server\/routes\.go/); + assert.match(slices[0].content, /model\/deep\/nested\.go/); + assert.match(slices[0].content, /main\.go/); + } finally { + await rm(dir, { recursive: true, force: true }); + } +}); + +test("auto slicing directory-splits a repo too large for one slice", async () => { + const dir = await mkdtemp(join(tmpdir(), "broadside-auto-")); + try { + await mkdir(join(dir, "server")); + await mkdir(join(dir, "model")); + await writeFile(join(dir, "go.mod"), "module x\n"); + await writeFile(join(dir, "main.go"), "package main\n"); + for (let i = 0; i < 40; i++) { + await writeFile(join(dir, "server", `s${i}.go`), "package server\n" + `// ${"x".repeat(2000)}\n`); + await writeFile(join(dir, "model", `m${i}.go`), "package model\n" + `// ${"y".repeat(2000)}\n`); + } const info = await collectRepoInfo(dir); const lens = getLens("defect"); const slices = await gatherSlices(dir, lens, info); @@ -98,9 +122,6 @@ test("directory slicing puts nested files under their top-level module", async ( assert.ok(byModule.server, "server/ must be its own slice"); assert.ok(byModule.model, "model/ must be its own slice"); assert.ok(byModule.root, "top-level main.go must land in the root slice"); - assert.match(byModule.server.content, /server\/routes\.go/); - assert.match(byModule.model.content, /model\/deep\/nested\.go/, "nested files belong to the top-level module"); - assert.match(byModule.root.content, /main\.go/); for (const slice of slices) { assert.ok(slice.chars <= lens.maxChars); }