diff --git a/scripts/intervention-forensics.ts b/scripts/intervention-forensics.ts index 7c518f3ee..7539974b6 100644 --- a/scripts/intervention-forensics.ts +++ b/scripts/intervention-forensics.ts @@ -60,6 +60,10 @@ function percentile(sorted: readonly number[], p: number): number { interface Bucket { count: number; byFamily: Map; + // Exact model id (CL-6775) — coarser than byFamily, which groups e.g. every + // grok model under "grok". Answers "which model loops most", not just + // "which family". + byModel: Map; values: number[]; thresholds: Set; editedWork: number; @@ -70,6 +74,7 @@ function emptyBucket(): Bucket { return { count: 0, byFamily: new Map(), + byModel: new Map(), values: [], thresholds: new Set(), editedWork: 0, @@ -121,6 +126,8 @@ for (const file of files) { bucket.count++; const family = record.family ?? record.model ?? "unknown"; bucket.byFamily.set(family, (bucket.byFamily.get(family) ?? 0) + 1); + const model = record.model ?? "unknown"; + bucket.byModel.set(model, (bucket.byModel.get(model) ?? 0) + 1); if (record.measurement !== undefined) { bucket.values.push(record.measurement.value); if (record.measurement.threshold !== undefined) { @@ -169,6 +176,35 @@ for (const [key, bucket] of rows) { console.log(`${key.padEnd(33)} ${families}`); } +// CL-6775: streamed degenerate-repetition aborts (mid-stream, not a turn-level +// stop) get their own model breakdown — "repetition-" ids, one row +// per model, so "which model loops most" reads off directly. This is a count, +// not a rate normalized by dispatch volume: the log's outcome records (total +// completed dispatches) are written from the parent side without a model tag, +// so a per-model denominator is not yet tracked — see the PR description. +const repetitionRows = rows.filter(([key]) => key.includes("/repetition-")); +if (repetitionRows.length > 0) { + const totalsByModel = new Map(); + for (const [, bucket] of repetitionRows) { + for (const [model, count] of bucket.byModel) { + totalsByModel.set(model, (totalsByModel.get(model) ?? 0) + count); + } + } + console.log("\nrepetition aborts by model (mid-stream degenerate-repetition, all detectors)"); + const modelRows = [...totalsByModel.entries()].sort((a, b) => b[1] - a[1]); + for (const [model, count] of modelRows) { + console.log(`${model.padEnd(33)} ${count}`); + } + console.log("\nrepetition aborts by model, per detector"); + for (const [key, bucket] of repetitionRows) { + const models = [...bucket.byModel.entries()] + .sort((a, b) => b[1] - a[1]) + .map(([model, count]) => `${model}=${count}`) + .join(" "); + console.log(`${key.padEnd(33)} ${models}`); + } +} + console.log( "\nedited = stops on runs that had already edited files; early = stops before half the turn budget (context, not a false-positive rate).", ); diff --git a/src/subagent/index.test.ts b/src/subagent/index.test.ts index 4e356be55..5e3e002d3 100644 --- a/src/subagent/index.test.ts +++ b/src/subagent/index.test.ts @@ -815,12 +815,16 @@ describe("sub-agent stop helpers", () => { expect(stopReasonFromReport("## Summary\nDone.\n\n## Findings\nx")).toBe(null); }); - test("repetitionStopDetail formats the looped window snippet and repeat count", () => { - expect(repetitionStopDetail({ window: "Groaning. ", repeats: 1363 })).toBe( - 'window "Groaning. " × 1363', + test("repetitionStopDetail reports period length and repeat count, never the looped text", () => { + expect(repetitionStopDetail({ window: "Groaning. ", repeats: 1363 }, null)).toBe( + "period 10ch × 1363", ); - const long = repetitionStopDetail({ window: "x".repeat(500), repeats: 7 }); - expect(long).toBe(`window "${"x".repeat(80)}" × 7`); + expect( + repetitionStopDetail( + { window: "x".repeat(500), repeats: 7 }, + { windowMinChars: 8, repeatThreshold: 16, probeChars: 8192 }, + ), + ).toBe("period 500ch × 7 (threshold 16)"); }); test("createSubAgentRunController aborts on an explicit deadline and reports deadlineHit", async () => { diff --git a/src/subagent/repetition.ts b/src/subagent/repetition.ts index 104f8d083..b1795b5e2 100644 --- a/src/subagent/repetition.ts +++ b/src/subagent/repetition.ts @@ -234,8 +234,11 @@ function visibleLength(token: string): number { /** * Fold one streamed token into the contentless-growth window. Returns the - * next state and whether the just-completed window was contentless: raw text - * grew by a full window while visible content grew less than the epsilon. + * next state, whether the just-completed window was contentless (raw text + * grew by a full window while visible content grew less than the epsilon), + * and the window's own raw/visible counts (`measured`) — reported alongside + * `state`, which resets to zero on completion, so a caller that wants to log + * what tripped the guard can read it before the reset erases it. * Pure reducer — the caller owns the state across deltas; the window resets * on completion either way, so one visible-rich window re-arms the guard. */ @@ -243,15 +246,17 @@ export function trackContentlessGrowth( state: ContentlessGrowthState, token: string, config: ContentlessGrowthConfig = DEFAULT_CONTENTLESS_GROWTH_CONFIG, -): { state: ContentlessGrowthState; hit: boolean } { +): { state: ContentlessGrowthState; hit: boolean; measured: ContentlessGrowthState } { const rawChars = state.rawChars + token.length; const visibleChars = state.visibleChars + visibleLength(token); + const measured = { rawChars, visibleChars }; if (rawChars < config.rawWindowChars) { - return { state: { rawChars, visibleChars }, hit: false }; + return { state: measured, hit: false, measured }; } return { state: INITIAL_CONTENTLESS_GROWTH_STATE, hit: visibleChars < config.minVisibleChars, + measured, }; } diff --git a/src/subagent/run.ts b/src/subagent/run.ts index 4f0c265f4..7db756b67 100644 --- a/src/subagent/run.ts +++ b/src/subagent/run.ts @@ -72,9 +72,12 @@ import { INITIAL_CONTENTLESS_GROWTH_STATE, trackContentlessGrowth, type ContentlessGrowthState, + DEFAULT_CONTENTLESS_GROWTH_CONFIG, + DEFAULT_REPETITION_CONFIG, DEFAULT_TEXT_FOLDED_REPETITION_CONFIG, DEFAULT_THINKING_REPETITION_CONFIG, REPETITION_CHECK_INTERVAL_CHARS, + type RepetitionConfig, type RepetitionHit, } from "./repetition.js"; import { refreshInferenceSourceBundle } from "./refresh-inference-source.js"; @@ -243,9 +246,28 @@ export function createSubAgentRunController( }; } -/** Stopped-line detail for a repetition abort: the looped window plus repeat count. */ -export function repetitionStopDetail(hit: RepetitionHit): string { - return `window ${JSON.stringify(hit.window.slice(0, 80))} × ${hit.repeats}`; +/** + * Stopped-line / log detail for a repetition abort. Reports the looped + * window's length and repeat count against the detector's threshold, never + * the window text itself (CL-6775) — the looped text is model output, and + * the parent-facing report carries a capped sample separately via `partial`. + */ +export function repetitionStopDetail(hit: RepetitionHit, config: RepetitionConfig | null): string { + const threshold = config?.repeatThreshold; + return `period ${hit.window.length}ch × ${hit.repeats}${threshold !== undefined ? ` (threshold ${threshold})` : ""}`; +} + +/** Stopped-line / log detail for a contentless/zero-width growth abort (CL-6775). */ +export function contentlessGrowthDetail( + measured: ContentlessGrowthState | null, + stream: string | null, +): string { + if (measured === null) return "contentless/zero-width flood"; + return ( + `contentless/zero-width flood: ${stream ?? "stream"} ` + + `${measured.rawChars}raw/${measured.visibleChars}visible ` + + `(min ${DEFAULT_CONTENTLESS_GROWTH_CONFIG.minVisibleChars}visible per ${DEFAULT_CONTENTLESS_GROWTH_CONFIG.rawWindowChars}raw)` + ); } /** String form of an abort signal's reason (cancel detail), or undefined. */ @@ -640,9 +662,24 @@ export async function runSubAgent(params: RunSubAgentParams): Promise { // Holder object rather than a let: the value is written inside the stream // sink closure, and flow analysis would otherwise narrow a let to null at // the later catch-site reads. - const repetition: { hit: RepetitionHit | null; contentless: boolean } = { + // `detector` names which of the three checks fired (CL-6775): raw-text + // periodicity, digit-folded thinking, or the contentless/zero-width growth + // guard — recorded so the intervention log can attribute aborts to a + // specific detector, not just "repetition" in general. + const repetition: { + hit: RepetitionHit | null; + contentless: boolean; + detector: "raw-text-periodicity" | "digit-folded-thinking" | "contentless-growth" | null; + config: RepetitionConfig | null; + contentlessMeasured: ContentlessGrowthState | null; + contentlessStream: string | null; + } = { hit: null, contentless: false, + detector: null, + config: null, + contentlessMeasured: null, + contentlessStream: null, }; let charsSinceRepetitionCheck = 0; let charsSinceThinkingRepetitionCheck = 0; @@ -660,6 +697,9 @@ export async function runSubAgent(params: RunSubAgentParams): Promise { const next = trackContentlessGrowth(state, token); if (next.hit) { repetition.contentless = true; + repetition.detector = "contentless-growth"; + repetition.contentlessMeasured = next.measured; + repetition.contentlessStream = stream; runController.abort( new Error( `sub-agent ${stream} output grew with only invisible/contentless characters (zero-width flood)`, @@ -689,13 +729,17 @@ export async function runSubAgent(params: RunSubAgentParams): Promise { // Two passes: digit-preserving for phrase loops, then the capped // folded pass for counter/timestamp/fence/emoji floods that are // never byte-periodic or fall under the plain window floor. + const rawHit = detectRepetition(cycleRecorder.text()); const hit = - detectRepetition(cycleRecorder.text()) ?? + rawHit ?? detectRepetition(cycleRecorder.text(), DEFAULT_TEXT_FOLDED_REPETITION_CONFIG, { normalizeDigits: true, }); if (hit !== null) { repetition.hit = hit; + repetition.detector = "raw-text-periodicity"; + repetition.config = + rawHit !== null ? DEFAULT_REPETITION_CONFIG : DEFAULT_TEXT_FOLDED_REPETITION_CONFIG; runController.abort( new Error(`sub-agent streamed output repeated the same window ${hit.repeats} times`), ); @@ -723,6 +767,8 @@ export async function runSubAgent(params: RunSubAgentParams): Promise { ); if (hit !== null) { repetition.hit = hit; + repetition.detector = "digit-folded-thinking"; + repetition.config = DEFAULT_THINKING_REPETITION_CONFIG; runController.abort( new Error(`sub-agent thinking output repeated the same window ${hit.repeats} times`), ); @@ -844,23 +890,43 @@ export async function runSubAgent(params: RunSubAgentParams): Promise { : tail; const detail = repetition.hit !== null - ? repetitionStopDetail(repetition.hit) + ? repetitionStopDetail(repetition.hit, repetition.config) : repetition.contentless - ? "contentless/zero-width flood" + ? contentlessGrowthDetail( + repetition.contentlessMeasured, + repetition.contentlessStream, + ) : reason === "deadline" && resolvedDeadlineMs !== undefined ? `${resolvedDeadlineMs}ms elapsed` : abortReasonText(runController.signal); interventions({ - id: reason, + // CL-6775: which detector fired is folded into the id (rather than + // a new field) so scripts/intervention-forensics.ts buckets each + // detector separately without any change to its aggregation logic. + id: + reason === "repetition" && repetition.detector !== null + ? `repetition-${repetition.detector}` + : reason, class: "stop", ...(repetition.hit !== null ? { measurement: { metric: "repeats", value: repetition.hit.repeats, + ...(repetition.config !== null + ? { threshold: repetition.config.repeatThreshold } + : {}), }, } - : {}), + : repetition.contentless + ? { + measurement: { + metric: "visibleChars", + value: repetition.contentlessMeasured?.visibleChars ?? 0, + threshold: DEFAULT_CONTENTLESS_GROWTH_CONFIG.minVisibleChars, + }, + } + : {}), state: { totalToolCalls: toolNamesUsed.length }, ...(detail !== undefined ? { detail } : {}), });