Skip to content

Commit e0760e1

Browse files
committed
Attribute mid-stream repetition aborts to model and detector
Repetition aborts in the intervention log (CL-6938) were all filed under one generic "repetition" id with no way to tell which of the three detectors fired or see the measured value against its threshold. Fold the detector name into the intervention id (repetition-raw-text-periodicity, repetition-digit-folded-thinking, repetition-contentless-growth) and record the measured value with its threshold for each. The contentless growth guard previously logged no measurement at all. scripts/intervention-forensics.ts already buckets by class/id and splits by family; add a per-model breakdown (family groups multiple models together) and a dedicated repetition-aborts-by-model summary so "which model loops most" reads directly off the report. No stream content is logged: repetitionStopDetail now reports period length and repeat count instead of the looped window text. No threshold values changed.
1 parent 3be75cf commit e0760e1

4 files changed

Lines changed: 129 additions & 18 deletions

File tree

scripts/intervention-forensics.ts

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -60,6 +60,10 @@ function percentile(sorted: readonly number[], p: number): number {
6060
interface Bucket {
6161
count: number;
6262
byFamily: Map<string, number>;
63+
// Exact model id (CL-6775) — coarser than byFamily, which groups e.g. every
64+
// grok model under "grok". Answers "which model loops most", not just
65+
// "which family".
66+
byModel: Map<string, number>;
6367
values: number[];
6468
thresholds: Set<number>;
6569
editedWork: number;
@@ -70,6 +74,7 @@ function emptyBucket(): Bucket {
7074
return {
7175
count: 0,
7276
byFamily: new Map(),
77+
byModel: new Map(),
7378
values: [],
7479
thresholds: new Set(),
7580
editedWork: 0,
@@ -121,6 +126,8 @@ for (const file of files) {
121126
bucket.count++;
122127
const family = record.family ?? record.model ?? "unknown";
123128
bucket.byFamily.set(family, (bucket.byFamily.get(family) ?? 0) + 1);
129+
const model = record.model ?? "unknown";
130+
bucket.byModel.set(model, (bucket.byModel.get(model) ?? 0) + 1);
124131
if (record.measurement !== undefined) {
125132
bucket.values.push(record.measurement.value);
126133
if (record.measurement.threshold !== undefined) {
@@ -169,6 +176,35 @@ for (const [key, bucket] of rows) {
169176
console.log(`${key.padEnd(33)} ${families}`);
170177
}
171178

179+
// CL-6775: streamed degenerate-repetition aborts (mid-stream, not a turn-level
180+
// stop) get their own model breakdown — "repetition-<detector>" ids, one row
181+
// per model, so "which model loops most" reads off directly. This is a count,
182+
// not a rate normalized by dispatch volume: the log's outcome records (total
183+
// completed dispatches) are written from the parent side without a model tag,
184+
// so a per-model denominator is not yet tracked — see the PR description.
185+
const repetitionRows = rows.filter(([key]) => key.includes("/repetition-"));
186+
if (repetitionRows.length > 0) {
187+
const totalsByModel = new Map<string, number>();
188+
for (const [, bucket] of repetitionRows) {
189+
for (const [model, count] of bucket.byModel) {
190+
totalsByModel.set(model, (totalsByModel.get(model) ?? 0) + count);
191+
}
192+
}
193+
console.log("\nrepetition aborts by model (mid-stream degenerate-repetition, all detectors)");
194+
const modelRows = [...totalsByModel.entries()].sort((a, b) => b[1] - a[1]);
195+
for (const [model, count] of modelRows) {
196+
console.log(`${model.padEnd(33)} ${count}`);
197+
}
198+
console.log("\nrepetition aborts by model, per detector");
199+
for (const [key, bucket] of repetitionRows) {
200+
const models = [...bucket.byModel.entries()]
201+
.sort((a, b) => b[1] - a[1])
202+
.map(([model, count]) => `${model}=${count}`)
203+
.join(" ");
204+
console.log(`${key.padEnd(33)} ${models}`);
205+
}
206+
}
207+
172208
console.log(
173209
"\nedited = stops on runs that had already edited files; early = stops before half the turn budget (context, not a false-positive rate).",
174210
);

src/subagent/index.test.ts

Lines changed: 9 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -815,12 +815,16 @@ describe("sub-agent stop helpers", () => {
815815
expect(stopReasonFromReport("## Summary\nDone.\n\n## Findings\nx")).toBe(null);
816816
});
817817

818-
test("repetitionStopDetail formats the looped window snippet and repeat count", () => {
819-
expect(repetitionStopDetail({ window: "Groaning. ", repeats: 1363 })).toBe(
820-
'window "Groaning. " × 1363',
818+
test("repetitionStopDetail reports period length and repeat count, never the looped text", () => {
819+
expect(repetitionStopDetail({ window: "Groaning. ", repeats: 1363 }, null)).toBe(
820+
"period 10ch × 1363",
821821
);
822-
const long = repetitionStopDetail({ window: "x".repeat(500), repeats: 7 });
823-
expect(long).toBe(`window "${"x".repeat(80)}" × 7`);
822+
expect(
823+
repetitionStopDetail(
824+
{ window: "x".repeat(500), repeats: 7 },
825+
{ windowMinChars: 8, repeatThreshold: 16, probeChars: 8192 },
826+
),
827+
).toBe("period 500ch × 7 (threshold 16)");
824828
});
825829

826830
test("createSubAgentRunController aborts on an explicit deadline and reports deadlineHit", async () => {

src/subagent/repetition.ts

Lines changed: 9 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -234,24 +234,29 @@ function visibleLength(token: string): number {
234234

235235
/**
236236
* Fold one streamed token into the contentless-growth window. Returns the
237-
* next state and whether the just-completed window was contentless: raw text
238-
* grew by a full window while visible content grew less than the epsilon.
237+
* next state, whether the just-completed window was contentless (raw text
238+
* grew by a full window while visible content grew less than the epsilon),
239+
* and the window's own raw/visible counts (`measured`) — reported alongside
240+
* `state`, which resets to zero on completion, so a caller that wants to log
241+
* what tripped the guard can read it before the reset erases it.
239242
* Pure reducer — the caller owns the state across deltas; the window resets
240243
* on completion either way, so one visible-rich window re-arms the guard.
241244
*/
242245
export function trackContentlessGrowth(
243246
state: ContentlessGrowthState,
244247
token: string,
245248
config: ContentlessGrowthConfig = DEFAULT_CONTENTLESS_GROWTH_CONFIG,
246-
): { state: ContentlessGrowthState; hit: boolean } {
249+
): { state: ContentlessGrowthState; hit: boolean; measured: ContentlessGrowthState } {
247250
const rawChars = state.rawChars + token.length;
248251
const visibleChars = state.visibleChars + visibleLength(token);
252+
const measured = { rawChars, visibleChars };
249253
if (rawChars < config.rawWindowChars) {
250-
return { state: { rawChars, visibleChars }, hit: false };
254+
return { state: measured, hit: false, measured };
251255
}
252256
return {
253257
state: INITIAL_CONTENTLESS_GROWTH_STATE,
254258
hit: visibleChars < config.minVisibleChars,
259+
measured,
255260
};
256261
}
257262

src/subagent/run.ts

Lines changed: 75 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -72,9 +72,12 @@ import {
7272
INITIAL_CONTENTLESS_GROWTH_STATE,
7373
trackContentlessGrowth,
7474
type ContentlessGrowthState,
75+
DEFAULT_CONTENTLESS_GROWTH_CONFIG,
76+
DEFAULT_REPETITION_CONFIG,
7577
DEFAULT_TEXT_FOLDED_REPETITION_CONFIG,
7678
DEFAULT_THINKING_REPETITION_CONFIG,
7779
REPETITION_CHECK_INTERVAL_CHARS,
80+
type RepetitionConfig,
7881
type RepetitionHit,
7982
} from "./repetition.js";
8083
import { refreshInferenceSourceBundle } from "./refresh-inference-source.js";
@@ -243,9 +246,28 @@ export function createSubAgentRunController(
243246
};
244247
}
245248

246-
/** Stopped-line detail for a repetition abort: the looped window plus repeat count. */
247-
export function repetitionStopDetail(hit: RepetitionHit): string {
248-
return `window ${JSON.stringify(hit.window.slice(0, 80))} × ${hit.repeats}`;
249+
/**
250+
* Stopped-line / log detail for a repetition abort. Reports the looped
251+
* window's length and repeat count against the detector's threshold, never
252+
* the window text itself (CL-6775) — the looped text is model output, and
253+
* the parent-facing report carries a capped sample separately via `partial`.
254+
*/
255+
export function repetitionStopDetail(hit: RepetitionHit, config: RepetitionConfig | null): string {
256+
const threshold = config?.repeatThreshold;
257+
return `period ${hit.window.length}ch × ${hit.repeats}${threshold !== undefined ? ` (threshold ${threshold})` : ""}`;
258+
}
259+
260+
/** Stopped-line / log detail for a contentless/zero-width growth abort (CL-6775). */
261+
export function contentlessGrowthDetail(
262+
measured: ContentlessGrowthState | null,
263+
stream: string | null,
264+
): string {
265+
if (measured === null) return "contentless/zero-width flood";
266+
return (
267+
`contentless/zero-width flood: ${stream ?? "stream"} ` +
268+
`${measured.rawChars}raw/${measured.visibleChars}visible ` +
269+
`(min ${DEFAULT_CONTENTLESS_GROWTH_CONFIG.minVisibleChars}visible per ${DEFAULT_CONTENTLESS_GROWTH_CONFIG.rawWindowChars}raw)`
270+
);
249271
}
250272

251273
/** String form of an abort signal's reason (cancel detail), or undefined. */
@@ -640,9 +662,24 @@ export async function runSubAgent(params: RunSubAgentParams): Promise<string> {
640662
// Holder object rather than a let: the value is written inside the stream
641663
// sink closure, and flow analysis would otherwise narrow a let to null at
642664
// the later catch-site reads.
643-
const repetition: { hit: RepetitionHit | null; contentless: boolean } = {
665+
// `detector` names which of the three checks fired (CL-6775): raw-text
666+
// periodicity, digit-folded thinking, or the contentless/zero-width growth
667+
// guard — recorded so the intervention log can attribute aborts to a
668+
// specific detector, not just "repetition" in general.
669+
const repetition: {
670+
hit: RepetitionHit | null;
671+
contentless: boolean;
672+
detector: "raw-text-periodicity" | "digit-folded-thinking" | "contentless-growth" | null;
673+
config: RepetitionConfig | null;
674+
contentlessMeasured: ContentlessGrowthState | null;
675+
contentlessStream: string | null;
676+
} = {
644677
hit: null,
645678
contentless: false,
679+
detector: null,
680+
config: null,
681+
contentlessMeasured: null,
682+
contentlessStream: null,
646683
};
647684
let charsSinceRepetitionCheck = 0;
648685
let charsSinceThinkingRepetitionCheck = 0;
@@ -660,6 +697,9 @@ export async function runSubAgent(params: RunSubAgentParams): Promise<string> {
660697
const next = trackContentlessGrowth(state, token);
661698
if (next.hit) {
662699
repetition.contentless = true;
700+
repetition.detector = "contentless-growth";
701+
repetition.contentlessMeasured = next.measured;
702+
repetition.contentlessStream = stream;
663703
runController.abort(
664704
new Error(
665705
`sub-agent ${stream} output grew with only invisible/contentless characters (zero-width flood)`,
@@ -689,13 +729,17 @@ export async function runSubAgent(params: RunSubAgentParams): Promise<string> {
689729
// Two passes: digit-preserving for phrase loops, then the capped
690730
// folded pass for counter/timestamp/fence/emoji floods that are
691731
// never byte-periodic or fall under the plain window floor.
732+
const rawHit = detectRepetition(cycleRecorder.text());
692733
const hit =
693-
detectRepetition(cycleRecorder.text()) ??
734+
rawHit ??
694735
detectRepetition(cycleRecorder.text(), DEFAULT_TEXT_FOLDED_REPETITION_CONFIG, {
695736
normalizeDigits: true,
696737
});
697738
if (hit !== null) {
698739
repetition.hit = hit;
740+
repetition.detector = "raw-text-periodicity";
741+
repetition.config =
742+
rawHit !== null ? DEFAULT_REPETITION_CONFIG : DEFAULT_TEXT_FOLDED_REPETITION_CONFIG;
699743
runController.abort(
700744
new Error(`sub-agent streamed output repeated the same window ${hit.repeats} times`),
701745
);
@@ -723,6 +767,8 @@ export async function runSubAgent(params: RunSubAgentParams): Promise<string> {
723767
);
724768
if (hit !== null) {
725769
repetition.hit = hit;
770+
repetition.detector = "digit-folded-thinking";
771+
repetition.config = DEFAULT_THINKING_REPETITION_CONFIG;
726772
runController.abort(
727773
new Error(`sub-agent thinking output repeated the same window ${hit.repeats} times`),
728774
);
@@ -844,23 +890,43 @@ export async function runSubAgent(params: RunSubAgentParams): Promise<string> {
844890
: tail;
845891
const detail =
846892
repetition.hit !== null
847-
? repetitionStopDetail(repetition.hit)
893+
? repetitionStopDetail(repetition.hit, repetition.config)
848894
: repetition.contentless
849-
? "contentless/zero-width flood"
895+
? contentlessGrowthDetail(
896+
repetition.contentlessMeasured,
897+
repetition.contentlessStream,
898+
)
850899
: reason === "deadline" && resolvedDeadlineMs !== undefined
851900
? `${resolvedDeadlineMs}ms elapsed`
852901
: abortReasonText(runController.signal);
853902
interventions({
854-
id: reason,
903+
// CL-6775: which detector fired is folded into the id (rather than
904+
// a new field) so scripts/intervention-forensics.ts buckets each
905+
// detector separately without any change to its aggregation logic.
906+
id:
907+
reason === "repetition" && repetition.detector !== null
908+
? `repetition-${repetition.detector}`
909+
: reason,
855910
class: "stop",
856911
...(repetition.hit !== null
857912
? {
858913
measurement: {
859914
metric: "repeats",
860915
value: repetition.hit.repeats,
916+
...(repetition.config !== null
917+
? { threshold: repetition.config.repeatThreshold }
918+
: {}),
861919
},
862920
}
863-
: {}),
921+
: repetition.contentless
922+
? {
923+
measurement: {
924+
metric: "visibleChars",
925+
value: repetition.contentlessMeasured?.visibleChars ?? 0,
926+
threshold: DEFAULT_CONTENTLESS_GROWTH_CONFIG.minVisibleChars,
927+
},
928+
}
929+
: {}),
864930
state: { totalToolCalls: toolNamesUsed.length },
865931
...(detail !== undefined ? { detail } : {}),
866932
});

0 commit comments

Comments
 (0)