From c06c0233a600e1775ab807e7f78ba6c375b0dd26 Mon Sep 17 00:00:00 2001 From: James Sesler Date: Sun, 23 Aug 2026 13:41:16 -0400 Subject: [PATCH] feat: broadside per-language lens prompts (#137) The defect and conventions lenses hardcoded Go idioms; a Python scanner was told to look for 'goroutines without ctx'. Language profiles (Go, Python, Rust, TypeScript/JavaScript, plus a neutral default) now drive the defect pattern list and convention vocabulary per detected language, and convention prompts name language-appropriate idiom hints (dunder methods, error wrapping with %w, Result handling with ?). Language detection gains a .js bucket so JavaScript repos resolve to the TypeScript profile; unknown languages fall back to the neutral default. JSON schemas are unchanged, so synthesis is unaffected. Also corrects the changelog/roadmap attribution: the Broad-Side PR is --- CHANGELOG.md | 1 + ROADMAP.md | 2 +- core/broadside.ts | 189 +++++++++++++++++++++++++++++++++------ tests/broadside.test.mjs | 35 ++++++++ 4 files changed, 201 insertions(+), 26 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 83e6b4d..0d5e97d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ All notable changes to this project are documented here. The format is based on ### Added +- **Broad-Side: per-language lens prompts** (#137). The defect and conventions lenses now build their system prompts from a language profile (Go, Python, Rust, TypeScript/JavaScript, plus a neutral default) instead of hardcoding Go idioms — a Python scanner no longer hears "goroutines without ctx"; it hears bare-except and context-manager checks. Convention extraction names language-appropriate categories (crates/modules vs modules/packages) and idiom hints. Language detection gains a `.js` bucket so JavaScript repos resolve to the TypeScript profile. Schemas are unchanged, so synthesis is unaffected. - **Broad-Side: batch reconnaissance over the OpenRouter Batch API** (#144). New `codecarto_broadside` MCP tool (actions: `submit`, `collect`, `status`) fires six single-turn analysis lenses — architecture, API surface, security, mechanical defect scan, convention extraction, porting — at any git repository as asynchronous batch jobs on a cheap batch model (~50% of sync pricing, unattended, 24h window), slices large modules by top-level directory, saves JSON plus rendered markdown to `.codecarto/broadside//`, and optionally synthesizes a cross-lens executive report. Works without an initialized workspace; needs an OpenRouter key via the `api_key` parameter, `OPENROUTER_API_KEY`, or `.codecarto/broadside/config.yaml`. Broad-Side findings are explicitly unverified scouting signals — file:line leads for the interactive pipeline to confirm, never evidence themselves. `codecarto_init` tolerates a `.codecarto/` that holds only `broadside/` (no force/backup needed), and scaffold refresh never touches broadside state, config, or results. - **Broad-Side: expense guardrails and live per-model pricing** (#144). `config.yaml` now accepts `model`, `max_cost`, and `pricing.input_per_m`/`output_per_m` overrides, and the MCP tool accepts `max_cost` and `force` parameters. Before submitting, Broad-Side estimates the run cost from collected file sizes (≈4 chars/token) against the configured model's per-token pricing — looked up live from OpenRouter's model catalog (cached 24h), so models like `openai/gpt-5.2-pro:batch` at ~$84/M output are priced correctly, not at the default model's rates. A submit whose estimate exceeds `max_cost` refuses with a per-lens breakdown and creates no run entry unless `force: true`. The submit response now reports the pricing used and its source (built-in/config/live/cache). - **Broad-Side: model catalog action and capability pre-flight** (#144). New `models` action lists every `:batch` model on OpenRouter — pricing per million tokens, context window, completion ceiling, structured-output support, and optional Artificial Analysis coding indices (via `GET /api/v1/benchmarks`, attribution preserved) — cheapest first with the configured model marked. Submits now pre-flight the chosen model against that catalog: lens `max_tokens` clamps to the provider's completion ceiling, deprecated models are flagged, and models that do not advertise structured-output support are refused outright, since every lens depends on `json_schema` response_format. The catalog cache is shared between the `models` action and submit-time pricing resolution. diff --git a/ROADMAP.md b/ROADMAP.md index 509a757..98a6308 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -36,7 +36,7 @@ file only moves when a tier completes. | **Triage lens** — prioritized fix queue (impact × difficulty, grouped by module) | [#135](https://github.com/HuginnIndustries/CodeCartographer/issues/135) | **Shipped**: triage pass runs on collect alongside synthesis (`include_triage` to skip) | | **Truncation repair** — detect max_tokens-cutoff JSON, resubmit slices, report truncation in summaries | [#133](https://github.com/HuginnIndustries/CodeCartographer/issues/133) | Partially shipped: fence-tolerant parsing + `truncated` flagging in collect, meta, and synthesis. Remaining: automatic resubmit of truncated slices | | **Concurrent polling** — poll all in-flight batches round-robin against one deadline | [#136](https://github.com/HuginnIndustries/CodeCartographer/issues/136) | Submissions already parallel; polling is sequential today | -| **Per-language prompts** — Go/Python/Rust/TS lens prompts; globs already adapt | [#137](https://github.com/HuginnIndustries/CodeCartographer/issues/137) | Schemas stay shared so synthesis is unaffected | +| **Per-language prompts** — Go/Python/Rust/TS lens prompts; globs already adapt | [#137](https://github.com/HuginnIndustries/CodeCartographer/issues/137) | **Shipped**: language profiles drive defect/conventions prompts; schemas unchanged | ## Tier 2 — integration depth diff --git a/core/broadside.ts b/core/broadside.ts index 92abb0d..0bd7b9a 100644 --- a/core/broadside.ts +++ b/core/broadside.ts @@ -592,6 +592,131 @@ const SCHEMAS: Record = { // ---------- lens definitions ---------- +// Lenses share their JSON schemas across languages, but prompts must speak +// the language's idioms: "goroutines without ctx" is noise to a Python +// scanner. Profiles supply per-language defect patterns and convention +// vocabulary; unknown languages get the neutral default. +type LanguageProfile = { + defectPatterns: string[]; + conventionCategories: Array<{ key: string; label: string }>; + idiomHints: string[]; +}; + +const TS_PROFILE: LanguageProfile = { + defectPatterns: [ + "Null/undefined dereference risks (unchecked optional access)", + "Error handling gaps (unhandled promise rejections, swallowed catches)", + "Resource leaks (unclosed handles, missing cleanup, dangling timers/listeners)", + "Race conditions (shared mutable state, async interleavings without guards)", + "Integer/precision assumptions in arithmetic", + "Unsafe type assumptions (as-casts, any leaks, non-null assertions)", + "Panic-prone code (out-of-bounds access, runtime TypeError paths)", + "Timezone/locale assumptions", + ], + conventionCategories: [ + { key: "packages", label: "modules and imports" }, + { key: "types", label: "interfaces and type aliases" }, + { key: "functions", label: "functions (camelCase), components (PascalCase)" }, + { key: "variables", label: "variables and constants (camelCase)" }, + { key: "files", label: "file naming (kebab vs camel) and folder organization" }, + { key: "tests", label: "test files (*.test.ts, describe/it patterns)" }, + ], + idiomHints: ["strict null checks usage", "async/await vs promise chains", "dependency injection patterns"], +}; + +const LANGUAGE_PROFILES: Record = { + go: { + defectPatterns: [ + "Nil pointer dereference risks (unchecked returns, missing nil guards)", + "Error handling gaps (ignored errors, deferred errors unchecked)", + "Resource leaks (unclosed files, connections, goroutines without ctx)", + "Race conditions (shared state without sync, channel misuse)", + "Integer overflow/underflow in arithmetic or bounds", + "Unsafe type assertions without ok check", + "Panic-prone code (slice out of bounds, map access without ok)", + "Timezone/locale assumptions", + ], + conventionCategories: [ + { key: "packages", label: "packages" }, + { key: "types", label: "types and interfaces" }, + { key: "functions", label: "functions and methods" }, + { key: "variables", label: "variables and fields" }, + { key: "files", label: "file and directory organization" }, + { key: "tests", label: "test files and table-driven tests" }, + ], + idiomHints: ["error wrapping with %w", "zero-value construction"], + }, + python: { + defectPatterns: [ + "None dereference risks (unchecked optional returns, AttributeError paths)", + "Exception handling gaps (bare except, swallowed exceptions, broad catch-all)", + "Resource leaks (unclosed files, sockets, connections, context managers)", + "Race conditions (shared mutable state, threading without locks, async pitfalls)", + "Integer/float precision assumptions in arithmetic", + "Unsafe type assumptions (unpacking mismatches, isinstance without fallback)", + "Panic-prone code (IndexError/KeyError paths, unbounded slicing)", + "Timezone/locale assumptions (naive datetimes)", + ], + conventionCategories: [ + { key: "packages", label: "modules and packages" }, + { key: "types", label: "classes and type hints" }, + { key: "functions", label: "functions and methods (snake_case vs camelCase)" }, + { key: "variables", label: "variables and constants" }, + { key: "files", label: "file and module organization" }, + { key: "tests", label: "test files (pytest fixtures, naming)" }, + ], + idiomHints: ["dunder method usage", "context manager idioms", "dataclass/pydantic models"], + }, + rust: { + defectPatterns: [ + "Unwrap/expect panics on fallible paths", + "Error handling gaps (swallowed Results, lossy conversions)", + "Resource leaks (unclosed handles, drop order assumptions)", + "Data races and Send/Sync violations (unsafe blocks, interior mutability misuse)", + "Integer overflow/underflow (arithmetic, casting)", + "Unsafe type assumptions (transmute/casts without invariants)", + "Panic-prone code (indexing, slicing, unreachable! in library paths)", + "Timezone/locale assumptions", + ], + conventionCategories: [ + { key: "packages", label: "crates and modules" }, + { key: "types", label: "structs, enums, and traits" }, + { key: "functions", label: "functions and methods (snake_case)" }, + { key: "variables", label: "variables and constants (SCREAMING_SNAKE)" }, + { key: "files", label: "module file organization" }, + { key: "tests", label: "test modules and #[cfg(test)] patterns" }, + ], + idiomHints: ["Result/Option handling with ?", "builder patterns", "trait-based extension"], + }, + typescript: TS_PROFILE, + javascript: TS_PROFILE, + default: { + defectPatterns: [ + "Null/undefined dereference risks (unchecked optional access)", + "Error handling gaps (ignored or swallowed errors)", + "Resource leaks (unclosed files, connections, handles)", + "Race conditions (shared mutable state without synchronization)", + "Integer overflow/underflow in arithmetic or bounds", + "Unsafe type assumptions and unchecked casts", + "Panic-prone code (out-of-bounds access, missing keys)", + "Timezone/locale assumptions", + ], + conventionCategories: [ + { key: "packages", label: "modules, packages, or namespaces" }, + { key: "types", label: "types, classes, and interfaces" }, + { key: "functions", label: "functions and methods" }, + { key: "variables", label: "variables and constants" }, + { key: "files", label: "file and directory organization" }, + { key: "tests", label: "test files and test organization" }, + ], + idiomHints: [], + }, +}; + +function languageProfile(language: string): LanguageProfile { + return LANGUAGE_PROFILES[language] ?? LANGUAGE_PROFILES.default; +} + type LensDefinition = { id: BroadsideLensId; name: string; @@ -710,21 +835,20 @@ const LENSES: Record = { maxChars: 60_000, maxTokens: 6000, globsFor: (info) => [info.sourceGlob], - systemPrompt: (info) => - `You are a senior code reviewer performing an automated defect scan on ${info.language} ` + - "source files. Look for these specific patterns:\n" + - " 1. Nil/null pointer dereference risks (unchecked returns, missing guards)\n" + - " 2. Error handling gaps (ignored errors, deferred errors unchecked)\n" + - " 3. Resource leaks (unclosed files, connections, goroutines without ctx)\n" + - " 4. Race conditions (shared state without sync, channel misuse)\n" + - " 5. Integer overflow/underflow in arithmetic or bounds\n" + - " 6. Unsafe type assertions without ok check\n" + - " 7. Panic-prone code (slice out of bounds, map access without ok)\n" + - " 8. Timezone/locale assumptions\n\n" + - "Return a JSON object following the defect_scan_report schema. " + - "Cite file:line for every finding. List which patterns you checked. " + - "If the code looks clean for a pattern, say so rather than staying silent. " + - "Prefer precision over volume — 3 solid findings beat 15 vague ones.", + systemPrompt: (info) => { + const profile = languageProfile(info.language); + const patterns = profile.defectPatterns.map((p, i) => ` ${i + 1}. ${p}`).join("\n"); + return ( + `You are a senior code reviewer performing an automated defect scan on ${info.language} ` + + "source files. Look for these specific patterns:\n" + + patterns + + "\n\n" + + "Return a JSON object following the defect_scan_report schema. " + + "Cite file:line for every finding. List which patterns you checked. " + + "If the code looks clean for a pattern, say so rather than staying silent. " + + "Prefer precision over volume — 3 solid findings beat 15 vague ones." + ); + }, userPrompt: (info, source, moduleName) => `Scan this ${info.language} module for mechanical defects.\n\n` + `Module: ${moduleName}\n\n` + @@ -741,15 +865,24 @@ const LENSES: Record = { maxChars: 60_000, maxTokens: 6000, globsFor: (info) => [info.sourceGlob], - systemPrompt: () => - "You are a code style analyst extracting conventions from source files. " + - "Catalog: naming conventions per category (packages, types, functions, variables, " + - "source files, test files), the dominant error-handling pattern, logging approach, " + - "test organization patterns, file/package organization rules, and recurring idioms. " + - "Also flag inconsistencies — places where the same convention is violated. " + - "If you find well-established conventions worth formalizing, list them as " + - "promotable_conventions with a title, rule, and evidence from the code. " + - "Return a JSON object following the conventions_report schema.", + systemPrompt: (info) => { + const profile = languageProfile(info.language); + const categories = profile.conventionCategories.map((c) => `${c.key} (${c.label})`).join(", "); + const idiomHint = + profile.idiomHints.length > 0 + ? ` Keep an eye out for ${info.language} idioms such as ${profile.idiomHints.join(", ")}.` + : ""; + return ( + `You are a code style analyst extracting conventions from ${info.language} source files. ` + + "Catalog naming conventions per category — " + categories + " — plus the dominant " + + "error-handling pattern, logging approach, test organization patterns, file/package " + + "organization rules, and recurring idioms." + idiomHint + + " Also flag inconsistencies — places where the same convention is violated. " + + "If you find well-established conventions worth formalizing, list them as " + + "promotable_conventions with a title, rule, and evidence from the code. " + + "Return a JSON object following the conventions_report schema." + ); + }, userPrompt: (info, source, moduleName) => "Extract coding conventions from this module.\n\n" + `Module: ${moduleName}\n\n` + @@ -930,7 +1063,13 @@ function detectLanguage(fileCounts: Record, manifestPath: string if (manifestPath === candidate) return lang; } } - const counts: Record = { go: fileCounts[".go"] ?? 0, python: fileCounts[".py"] ?? 0, rust: fileCounts[".rs"] ?? 0, typescript: (fileCounts[".ts"] ?? 0) + (fileCounts[".tsx"] ?? 0) }; + const counts: Record = { + go: fileCounts[".go"] ?? 0, + python: fileCounts[".py"] ?? 0, + rust: fileCounts[".rs"] ?? 0, + typescript: (fileCounts[".ts"] ?? 0) + (fileCounts[".tsx"] ?? 0), + javascript: fileCounts[".js"] ?? 0, + }; const best = Object.entries(counts).sort((a, b) => b[1] - a[1])[0]; return best && best[1] > 0 ? best[0] : "unknown"; } diff --git a/tests/broadside.test.mjs b/tests/broadside.test.mjs index 62e12a0..f6357a9 100644 --- a/tests/broadside.test.mjs +++ b/tests/broadside.test.mjs @@ -562,6 +562,41 @@ test("modelsText renders pricing, caps, support, and benchmark columns", () => { assert.match(text, /\(default\)/); }); +// ---------- per-language prompts (#137) ---------- + +test("defect lens prompt speaks the detected language, not Go", () => { + const python = getLens("defect").systemPrompt({ language: "python" }); + assert.match(python, /bare except/); + assert.ok(!python.includes("goroutines"), "Go idioms must not leak into Python prompts"); + + const go = getLens("defect").systemPrompt({ language: "go" }); + assert.match(go, /goroutines without ctx/); + + const rust = getLens("defect").systemPrompt({ language: "rust" }); + assert.match(rust, /Unwrap\/expect panics/); + + const ts = getLens("defect").systemPrompt({ language: "typescript" }); + assert.match(ts, /non-null assertions/); + + const js = getLens("defect").systemPrompt({ language: "javascript" }); + assert.match(js, /unhandled promise rejections/, "javascript rides the TS profile"); + + const unknown = getLens("defect").systemPrompt({ language: "whitespace-esque" }); + assert.match(unknown, /unchecked casts/, "unknown languages get the neutral default profile"); +}); + +test("conventions lens prompt names language-appropriate categories and idioms", () => { + const rust = getLens("conventions").systemPrompt({ language: "rust" }); + assert.match(rust, /crates and modules/); + + const python = getLens("conventions").systemPrompt({ language: "python" }); + assert.match(python, /dunder method usage/); + + const go = getLens("conventions").systemPrompt({ language: "go" }); + assert.match(go, /error wrapping with %w/); + assert.ok(!go.includes("dunder"), "python idiom hints must not leak into Go prompts"); +}); + // ---------- batch client with a fake fetcher ---------- function fakeResponse(status, body) {