Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
36 changes: 36 additions & 0 deletions scripts/intervention-forensics.ts
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,10 @@ function percentile(sorted: readonly number[], p: number): number {
interface Bucket {
count: number;
byFamily: Map<string, number>;
// Exact model id (CL-6775) — coarser than byFamily, which groups e.g. every
// grok model under "grok". Answers "which model loops most", not just
// "which family".
byModel: Map<string, number>;
values: number[];
thresholds: Set<number>;
editedWork: number;
Expand All @@ -70,6 +74,7 @@ function emptyBucket(): Bucket {
return {
count: 0,
byFamily: new Map(),
byModel: new Map(),
values: [],
thresholds: new Set(),
editedWork: 0,
Expand Down Expand Up @@ -121,6 +126,8 @@ for (const file of files) {
bucket.count++;
const family = record.family ?? record.model ?? "unknown";
bucket.byFamily.set(family, (bucket.byFamily.get(family) ?? 0) + 1);
const model = record.model ?? "unknown";
bucket.byModel.set(model, (bucket.byModel.get(model) ?? 0) + 1);
if (record.measurement !== undefined) {
bucket.values.push(record.measurement.value);
if (record.measurement.threshold !== undefined) {
Expand Down Expand Up @@ -169,6 +176,35 @@ for (const [key, bucket] of rows) {
console.log(`${key.padEnd(33)} ${families}`);
}

// CL-6775: streamed degenerate-repetition aborts (mid-stream, not a turn-level
// stop) get their own model breakdown — "repetition-<detector>" ids, one row
// per model, so "which model loops most" reads off directly. This is a count,
// not a rate normalized by dispatch volume: the log's outcome records (total
// completed dispatches) are written from the parent side without a model tag,
// so a per-model denominator is not yet tracked — see the PR description.
const repetitionRows = rows.filter(([key]) => key.includes("/repetition-"));
if (repetitionRows.length > 0) {
const totalsByModel = new Map<string, number>();
for (const [, bucket] of repetitionRows) {
for (const [model, count] of bucket.byModel) {
totalsByModel.set(model, (totalsByModel.get(model) ?? 0) + count);
}
}
console.log("\nrepetition aborts by model (mid-stream degenerate-repetition, all detectors)");
const modelRows = [...totalsByModel.entries()].sort((a, b) => b[1] - a[1]);
for (const [model, count] of modelRows) {
console.log(`${model.padEnd(33)} ${count}`);
}
console.log("\nrepetition aborts by model, per detector");
for (const [key, bucket] of repetitionRows) {
const models = [...bucket.byModel.entries()]
.sort((a, b) => b[1] - a[1])
.map(([model, count]) => `${model}=${count}`)
.join(" ");
console.log(`${key.padEnd(33)} ${models}`);
}
}

console.log(
"\nedited = stops on runs that had already edited files; early = stops before half the turn budget (context, not a false-positive rate).",
);
Expand Down
14 changes: 9 additions & 5 deletions src/subagent/index.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -815,12 +815,16 @@ describe("sub-agent stop helpers", () => {
expect(stopReasonFromReport("## Summary\nDone.\n\n## Findings\nx")).toBe(null);
});

test("repetitionStopDetail formats the looped window snippet and repeat count", () => {
expect(repetitionStopDetail({ window: "Groaning. ", repeats: 1363 })).toBe(
'window "Groaning. " × 1363',
test("repetitionStopDetail reports period length and repeat count, never the looped text", () => {
expect(repetitionStopDetail({ window: "Groaning. ", repeats: 1363 }, null)).toBe(
"period 10ch × 1363",
);
const long = repetitionStopDetail({ window: "x".repeat(500), repeats: 7 });
expect(long).toBe(`window "${"x".repeat(80)}" × 7`);
expect(
repetitionStopDetail(
{ window: "x".repeat(500), repeats: 7 },
{ windowMinChars: 8, repeatThreshold: 16, probeChars: 8192 },
),
).toBe("period 500ch × 7 (threshold 16)");
});

test("createSubAgentRunController aborts on an explicit deadline and reports deadlineHit", async () => {
Expand Down
13 changes: 9 additions & 4 deletions src/subagent/repetition.ts
Original file line number Diff line number Diff line change
Expand Up @@ -234,24 +234,29 @@ function visibleLength(token: string): number {

/**
* Fold one streamed token into the contentless-growth window. Returns the
* next state and whether the just-completed window was contentless: raw text
* grew by a full window while visible content grew less than the epsilon.
* next state, whether the just-completed window was contentless (raw text
* grew by a full window while visible content grew less than the epsilon),
* and the window's own raw/visible counts (`measured`) — reported alongside
* `state`, which resets to zero on completion, so a caller that wants to log
* what tripped the guard can read it before the reset erases it.
* Pure reducer — the caller owns the state across deltas; the window resets
* on completion either way, so one visible-rich window re-arms the guard.
*/
export function trackContentlessGrowth(
state: ContentlessGrowthState,
token: string,
config: ContentlessGrowthConfig = DEFAULT_CONTENTLESS_GROWTH_CONFIG,
): { state: ContentlessGrowthState; hit: boolean } {
): { state: ContentlessGrowthState; hit: boolean; measured: ContentlessGrowthState } {
const rawChars = state.rawChars + token.length;
const visibleChars = state.visibleChars + visibleLength(token);
const measured = { rawChars, visibleChars };
if (rawChars < config.rawWindowChars) {
return { state: { rawChars, visibleChars }, hit: false };
return { state: measured, hit: false, measured };
}
return {
state: INITIAL_CONTENTLESS_GROWTH_STATE,
hit: visibleChars < config.minVisibleChars,
measured,
};
}

Expand Down
84 changes: 75 additions & 9 deletions src/subagent/run.ts
Original file line number Diff line number Diff line change
Expand Up @@ -72,9 +72,12 @@ import {
INITIAL_CONTENTLESS_GROWTH_STATE,
trackContentlessGrowth,
type ContentlessGrowthState,
DEFAULT_CONTENTLESS_GROWTH_CONFIG,
DEFAULT_REPETITION_CONFIG,
DEFAULT_TEXT_FOLDED_REPETITION_CONFIG,
DEFAULT_THINKING_REPETITION_CONFIG,
REPETITION_CHECK_INTERVAL_CHARS,
type RepetitionConfig,
type RepetitionHit,
} from "./repetition.js";
import { refreshInferenceSourceBundle } from "./refresh-inference-source.js";
Expand Down Expand Up @@ -243,9 +246,28 @@ export function createSubAgentRunController(
};
}

/** Stopped-line detail for a repetition abort: the looped window plus repeat count. */
export function repetitionStopDetail(hit: RepetitionHit): string {
return `window ${JSON.stringify(hit.window.slice(0, 80))} × ${hit.repeats}`;
/**
* Stopped-line / log detail for a repetition abort. Reports the looped
* window's length and repeat count against the detector's threshold, never
* the window text itself (CL-6775) — the looped text is model output, and
* the parent-facing report carries a capped sample separately via `partial`.
*/
export function repetitionStopDetail(hit: RepetitionHit, config: RepetitionConfig | null): string {
const threshold = config?.repeatThreshold;
return `period ${hit.window.length}ch × ${hit.repeats}${threshold !== undefined ? ` (threshold ${threshold})` : ""}`;
}

/** Stopped-line / log detail for a contentless/zero-width growth abort (CL-6775). */
export function contentlessGrowthDetail(
measured: ContentlessGrowthState | null,
stream: string | null,
): string {
if (measured === null) return "contentless/zero-width flood";
return (
`contentless/zero-width flood: ${stream ?? "stream"} ` +
`${measured.rawChars}raw/${measured.visibleChars}visible ` +
`(min ${DEFAULT_CONTENTLESS_GROWTH_CONFIG.minVisibleChars}visible per ${DEFAULT_CONTENTLESS_GROWTH_CONFIG.rawWindowChars}raw)`
);
}

/** String form of an abort signal's reason (cancel detail), or undefined. */
Expand Down Expand Up @@ -640,9 +662,24 @@ export async function runSubAgent(params: RunSubAgentParams): Promise<string> {
// Holder object rather than a let: the value is written inside the stream
// sink closure, and flow analysis would otherwise narrow a let to null at
// the later catch-site reads.
const repetition: { hit: RepetitionHit | null; contentless: boolean } = {
// `detector` names which of the three checks fired (CL-6775): raw-text
// periodicity, digit-folded thinking, or the contentless/zero-width growth
// guard — recorded so the intervention log can attribute aborts to a
// specific detector, not just "repetition" in general.
const repetition: {
hit: RepetitionHit | null;
contentless: boolean;
detector: "raw-text-periodicity" | "digit-folded-thinking" | "contentless-growth" | null;
config: RepetitionConfig | null;
contentlessMeasured: ContentlessGrowthState | null;
contentlessStream: string | null;
} = {
hit: null,
contentless: false,
detector: null,
config: null,
contentlessMeasured: null,
contentlessStream: null,
};
let charsSinceRepetitionCheck = 0;
let charsSinceThinkingRepetitionCheck = 0;
Expand All @@ -660,6 +697,9 @@ export async function runSubAgent(params: RunSubAgentParams): Promise<string> {
const next = trackContentlessGrowth(state, token);
if (next.hit) {
repetition.contentless = true;
repetition.detector = "contentless-growth";
repetition.contentlessMeasured = next.measured;
repetition.contentlessStream = stream;
runController.abort(
new Error(
`sub-agent ${stream} output grew with only invisible/contentless characters (zero-width flood)`,
Expand Down Expand Up @@ -689,13 +729,17 @@ export async function runSubAgent(params: RunSubAgentParams): Promise<string> {
// Two passes: digit-preserving for phrase loops, then the capped
// folded pass for counter/timestamp/fence/emoji floods that are
// never byte-periodic or fall under the plain window floor.
const rawHit = detectRepetition(cycleRecorder.text());
const hit =
detectRepetition(cycleRecorder.text()) ??
rawHit ??
detectRepetition(cycleRecorder.text(), DEFAULT_TEXT_FOLDED_REPETITION_CONFIG, {
normalizeDigits: true,
});
if (hit !== null) {
repetition.hit = hit;
repetition.detector = "raw-text-periodicity";
repetition.config =
rawHit !== null ? DEFAULT_REPETITION_CONFIG : DEFAULT_TEXT_FOLDED_REPETITION_CONFIG;
runController.abort(
new Error(`sub-agent streamed output repeated the same window ${hit.repeats} times`),
);
Expand Down Expand Up @@ -723,6 +767,8 @@ export async function runSubAgent(params: RunSubAgentParams): Promise<string> {
);
if (hit !== null) {
repetition.hit = hit;
repetition.detector = "digit-folded-thinking";
repetition.config = DEFAULT_THINKING_REPETITION_CONFIG;
runController.abort(
new Error(`sub-agent thinking output repeated the same window ${hit.repeats} times`),
);
Expand Down Expand Up @@ -844,23 +890,43 @@ export async function runSubAgent(params: RunSubAgentParams): Promise<string> {
: tail;
const detail =
repetition.hit !== null
? repetitionStopDetail(repetition.hit)
? repetitionStopDetail(repetition.hit, repetition.config)
: repetition.contentless
? "contentless/zero-width flood"
? contentlessGrowthDetail(
repetition.contentlessMeasured,
repetition.contentlessStream,
)
: reason === "deadline" && resolvedDeadlineMs !== undefined
? `${resolvedDeadlineMs}ms elapsed`
: abortReasonText(runController.signal);
interventions({
id: reason,
// CL-6775: which detector fired is folded into the id (rather than
// a new field) so scripts/intervention-forensics.ts buckets each
// detector separately without any change to its aggregation logic.
id:
reason === "repetition" && repetition.detector !== null
? `repetition-${repetition.detector}`
: reason,
class: "stop",
...(repetition.hit !== null
? {
measurement: {
metric: "repeats",
value: repetition.hit.repeats,
...(repetition.config !== null
? { threshold: repetition.config.repeatThreshold }
: {}),
},
}
: {}),
: repetition.contentless
? {
measurement: {
metric: "visibleChars",
value: repetition.contentlessMeasured?.visibleChars ?? 0,
threshold: DEFAULT_CONTENTLESS_GROWTH_CONFIG.minVisibleChars,
},
}
: {}),
state: { totalToolCalls: toolNamesUsed.length },
...(detail !== undefined ? { detail } : {}),
});
Expand Down
Loading