From e0760e1f9832ad826187c2bb533db06d47b00623 Mon Sep 17 00:00:00 2001 From: Sawyer Cutler Date: Sun, 23 Aug 2026 14:19:40 -0700 Subject: [PATCH] Attribute mid-stream repetition aborts to model and detector Repetition aborts in the intervention log (CL-6938) were all filed under one generic "repetition" id with no way to tell which of the three detectors fired or see the measured value against its threshold. Fold the detector name into the intervention id (repetition-raw-text-periodicity, repetition-digit-folded-thinking, repetition-contentless-growth) and record the measured value with its threshold for each. The contentless growth guard previously logged no measurement at all. scripts/intervention-forensics.ts already buckets by class/id and splits by family; add a per-model breakdown (family groups multiple models together) and a dedicated repetition-aborts-by-model summary so "which model loops most" reads directly off the report. No stream content is logged: repetitionStopDetail now reports period length and repeat count instead of the looped window text. No threshold values changed. --- scripts/intervention-forensics.ts | 36 +++++++++++++ src/subagent/index.test.ts | 14 ++++-- src/subagent/repetition.ts | 13 +++-- src/subagent/run.ts | 84 +++++++++++++++++++++++++++---- 4 files changed, 129 insertions(+), 18 deletions(-) diff --git a/scripts/intervention-forensics.ts b/scripts/intervention-forensics.ts index 7c518f3ee..7539974b6 100644 --- a/scripts/intervention-forensics.ts +++ b/scripts/intervention-forensics.ts @@ -60,6 +60,10 @@ function percentile(sorted: readonly number[], p: number): number { interface Bucket { count: number; byFamily: Map; + // Exact model id (CL-6775) — coarser than byFamily, which groups e.g. every + // grok model under "grok". Answers "which model loops most", not just + // "which family". + byModel: Map; values: number[]; thresholds: Set; editedWork: number; @@ -70,6 +74,7 @@ function emptyBucket(): Bucket { return { count: 0, byFamily: new Map(), + byModel: new Map(), values: [], thresholds: new Set(), editedWork: 0, @@ -121,6 +126,8 @@ for (const file of files) { bucket.count++; const family = record.family ?? record.model ?? "unknown"; bucket.byFamily.set(family, (bucket.byFamily.get(family) ?? 0) + 1); + const model = record.model ?? "unknown"; + bucket.byModel.set(model, (bucket.byModel.get(model) ?? 0) + 1); if (record.measurement !== undefined) { bucket.values.push(record.measurement.value); if (record.measurement.threshold !== undefined) { @@ -169,6 +176,35 @@ for (const [key, bucket] of rows) { console.log(`${key.padEnd(33)} ${families}`); } +// CL-6775: streamed degenerate-repetition aborts (mid-stream, not a turn-level +// stop) get their own model breakdown — "repetition-" ids, one row +// per model, so "which model loops most" reads off directly. This is a count, +// not a rate normalized by dispatch volume: the log's outcome records (total +// completed dispatches) are written from the parent side without a model tag, +// so a per-model denominator is not yet tracked — see the PR description. +const repetitionRows = rows.filter(([key]) => key.includes("/repetition-")); +if (repetitionRows.length > 0) { + const totalsByModel = new Map(); + for (const [, bucket] of repetitionRows) { + for (const [model, count] of bucket.byModel) { + totalsByModel.set(model, (totalsByModel.get(model) ?? 0) + count); + } + } + console.log("\nrepetition aborts by model (mid-stream degenerate-repetition, all detectors)"); + const modelRows = [...totalsByModel.entries()].sort((a, b) => b[1] - a[1]); + for (const [model, count] of modelRows) { + console.log(`${model.padEnd(33)} ${count}`); + } + console.log("\nrepetition aborts by model, per detector"); + for (const [key, bucket] of repetitionRows) { + const models = [...bucket.byModel.entries()] + .sort((a, b) => b[1] - a[1]) + .map(([model, count]) => `${model}=${count}`) + .join(" "); + console.log(`${key.padEnd(33)} ${models}`); + } +} + console.log( "\nedited = stops on runs that had already edited files; early = stops before half the turn budget (context, not a false-positive rate).", ); diff --git a/src/subagent/index.test.ts b/src/subagent/index.test.ts index 4e356be55..5e3e002d3 100644 --- a/src/subagent/index.test.ts +++ b/src/subagent/index.test.ts @@ -815,12 +815,16 @@ describe("sub-agent stop helpers", () => { expect(stopReasonFromReport("## Summary\nDone.\n\n## Findings\nx")).toBe(null); }); - test("repetitionStopDetail formats the looped window snippet and repeat count", () => { - expect(repetitionStopDetail({ window: "Groaning. ", repeats: 1363 })).toBe( - 'window "Groaning. " × 1363', + test("repetitionStopDetail reports period length and repeat count, never the looped text", () => { + expect(repetitionStopDetail({ window: "Groaning. ", repeats: 1363 }, null)).toBe( + "period 10ch × 1363", ); - const long = repetitionStopDetail({ window: "x".repeat(500), repeats: 7 }); - expect(long).toBe(`window "${"x".repeat(80)}" × 7`); + expect( + repetitionStopDetail( + { window: "x".repeat(500), repeats: 7 }, + { windowMinChars: 8, repeatThreshold: 16, probeChars: 8192 }, + ), + ).toBe("period 500ch × 7 (threshold 16)"); }); test("createSubAgentRunController aborts on an explicit deadline and reports deadlineHit", async () => { diff --git a/src/subagent/repetition.ts b/src/subagent/repetition.ts index 104f8d083..b1795b5e2 100644 --- a/src/subagent/repetition.ts +++ b/src/subagent/repetition.ts @@ -234,8 +234,11 @@ function visibleLength(token: string): number { /** * Fold one streamed token into the contentless-growth window. Returns the - * next state and whether the just-completed window was contentless: raw text - * grew by a full window while visible content grew less than the epsilon. + * next state, whether the just-completed window was contentless (raw text + * grew by a full window while visible content grew less than the epsilon), + * and the window's own raw/visible counts (`measured`) — reported alongside + * `state`, which resets to zero on completion, so a caller that wants to log + * what tripped the guard can read it before the reset erases it. * Pure reducer — the caller owns the state across deltas; the window resets * on completion either way, so one visible-rich window re-arms the guard. */ @@ -243,15 +246,17 @@ export function trackContentlessGrowth( state: ContentlessGrowthState, token: string, config: ContentlessGrowthConfig = DEFAULT_CONTENTLESS_GROWTH_CONFIG, -): { state: ContentlessGrowthState; hit: boolean } { +): { state: ContentlessGrowthState; hit: boolean; measured: ContentlessGrowthState } { const rawChars = state.rawChars + token.length; const visibleChars = state.visibleChars + visibleLength(token); + const measured = { rawChars, visibleChars }; if (rawChars < config.rawWindowChars) { - return { state: { rawChars, visibleChars }, hit: false }; + return { state: measured, hit: false, measured }; } return { state: INITIAL_CONTENTLESS_GROWTH_STATE, hit: visibleChars < config.minVisibleChars, + measured, }; } diff --git a/src/subagent/run.ts b/src/subagent/run.ts index 4f0c265f4..7db756b67 100644 --- a/src/subagent/run.ts +++ b/src/subagent/run.ts @@ -72,9 +72,12 @@ import { INITIAL_CONTENTLESS_GROWTH_STATE, trackContentlessGrowth, type ContentlessGrowthState, + DEFAULT_CONTENTLESS_GROWTH_CONFIG, + DEFAULT_REPETITION_CONFIG, DEFAULT_TEXT_FOLDED_REPETITION_CONFIG, DEFAULT_THINKING_REPETITION_CONFIG, REPETITION_CHECK_INTERVAL_CHARS, + type RepetitionConfig, type RepetitionHit, } from "./repetition.js"; import { refreshInferenceSourceBundle } from "./refresh-inference-source.js"; @@ -243,9 +246,28 @@ export function createSubAgentRunController( }; } -/** Stopped-line detail for a repetition abort: the looped window plus repeat count. */ -export function repetitionStopDetail(hit: RepetitionHit): string { - return `window ${JSON.stringify(hit.window.slice(0, 80))} × ${hit.repeats}`; +/** + * Stopped-line / log detail for a repetition abort. Reports the looped + * window's length and repeat count against the detector's threshold, never + * the window text itself (CL-6775) — the looped text is model output, and + * the parent-facing report carries a capped sample separately via `partial`. + */ +export function repetitionStopDetail(hit: RepetitionHit, config: RepetitionConfig | null): string { + const threshold = config?.repeatThreshold; + return `period ${hit.window.length}ch × ${hit.repeats}${threshold !== undefined ? ` (threshold ${threshold})` : ""}`; +} + +/** Stopped-line / log detail for a contentless/zero-width growth abort (CL-6775). */ +export function contentlessGrowthDetail( + measured: ContentlessGrowthState | null, + stream: string | null, +): string { + if (measured === null) return "contentless/zero-width flood"; + return ( + `contentless/zero-width flood: ${stream ?? "stream"} ` + + `${measured.rawChars}raw/${measured.visibleChars}visible ` + + `(min ${DEFAULT_CONTENTLESS_GROWTH_CONFIG.minVisibleChars}visible per ${DEFAULT_CONTENTLESS_GROWTH_CONFIG.rawWindowChars}raw)` + ); } /** String form of an abort signal's reason (cancel detail), or undefined. */ @@ -640,9 +662,24 @@ export async function runSubAgent(params: RunSubAgentParams): Promise { // Holder object rather than a let: the value is written inside the stream // sink closure, and flow analysis would otherwise narrow a let to null at // the later catch-site reads. - const repetition: { hit: RepetitionHit | null; contentless: boolean } = { + // `detector` names which of the three checks fired (CL-6775): raw-text + // periodicity, digit-folded thinking, or the contentless/zero-width growth + // guard — recorded so the intervention log can attribute aborts to a + // specific detector, not just "repetition" in general. + const repetition: { + hit: RepetitionHit | null; + contentless: boolean; + detector: "raw-text-periodicity" | "digit-folded-thinking" | "contentless-growth" | null; + config: RepetitionConfig | null; + contentlessMeasured: ContentlessGrowthState | null; + contentlessStream: string | null; + } = { hit: null, contentless: false, + detector: null, + config: null, + contentlessMeasured: null, + contentlessStream: null, }; let charsSinceRepetitionCheck = 0; let charsSinceThinkingRepetitionCheck = 0; @@ -660,6 +697,9 @@ export async function runSubAgent(params: RunSubAgentParams): Promise { const next = trackContentlessGrowth(state, token); if (next.hit) { repetition.contentless = true; + repetition.detector = "contentless-growth"; + repetition.contentlessMeasured = next.measured; + repetition.contentlessStream = stream; runController.abort( new Error( `sub-agent ${stream} output grew with only invisible/contentless characters (zero-width flood)`, @@ -689,13 +729,17 @@ export async function runSubAgent(params: RunSubAgentParams): Promise { // Two passes: digit-preserving for phrase loops, then the capped // folded pass for counter/timestamp/fence/emoji floods that are // never byte-periodic or fall under the plain window floor. + const rawHit = detectRepetition(cycleRecorder.text()); const hit = - detectRepetition(cycleRecorder.text()) ?? + rawHit ?? detectRepetition(cycleRecorder.text(), DEFAULT_TEXT_FOLDED_REPETITION_CONFIG, { normalizeDigits: true, }); if (hit !== null) { repetition.hit = hit; + repetition.detector = "raw-text-periodicity"; + repetition.config = + rawHit !== null ? DEFAULT_REPETITION_CONFIG : DEFAULT_TEXT_FOLDED_REPETITION_CONFIG; runController.abort( new Error(`sub-agent streamed output repeated the same window ${hit.repeats} times`), ); @@ -723,6 +767,8 @@ export async function runSubAgent(params: RunSubAgentParams): Promise { ); if (hit !== null) { repetition.hit = hit; + repetition.detector = "digit-folded-thinking"; + repetition.config = DEFAULT_THINKING_REPETITION_CONFIG; runController.abort( new Error(`sub-agent thinking output repeated the same window ${hit.repeats} times`), ); @@ -844,23 +890,43 @@ export async function runSubAgent(params: RunSubAgentParams): Promise { : tail; const detail = repetition.hit !== null - ? repetitionStopDetail(repetition.hit) + ? repetitionStopDetail(repetition.hit, repetition.config) : repetition.contentless - ? "contentless/zero-width flood" + ? contentlessGrowthDetail( + repetition.contentlessMeasured, + repetition.contentlessStream, + ) : reason === "deadline" && resolvedDeadlineMs !== undefined ? `${resolvedDeadlineMs}ms elapsed` : abortReasonText(runController.signal); interventions({ - id: reason, + // CL-6775: which detector fired is folded into the id (rather than + // a new field) so scripts/intervention-forensics.ts buckets each + // detector separately without any change to its aggregation logic. + id: + reason === "repetition" && repetition.detector !== null + ? `repetition-${repetition.detector}` + : reason, class: "stop", ...(repetition.hit !== null ? { measurement: { metric: "repeats", value: repetition.hit.repeats, + ...(repetition.config !== null + ? { threshold: repetition.config.repeatThreshold } + : {}), }, } - : {}), + : repetition.contentless + ? { + measurement: { + metric: "visibleChars", + value: repetition.contentlessMeasured?.visibleChars ?? 0, + threshold: DEFAULT_CONTENTLESS_GROWTH_CONFIG.minVisibleChars, + }, + } + : {}), state: { totalToolCalls: toolNamesUsed.length }, ...(detail !== undefined ? { detail } : {}), });