Merge branch 'main' into cl-6907-tui-fires-refreshcodexinstructions-un-awaited-request-prefix

Author: TheGreatAxios

src/subagent/repetition.test.ts

@@ -3,9 +3,14 @@ import { describe, expect, test } from "bun:test";
 import { appendCycleText, CYCLE_TEXT_CAP_CHARS } from "../session/stream-journal.js";
 import {
   detectRepetition,
+  DEFAULT_CONTENTLESS_GROWTH_CONFIG,
   DEFAULT_REPETITION_CONFIG,
+  DEFAULT_TEXT_FOLDED_REPETITION_CONFIG,
   DEFAULT_THINKING_REPETITION_CONFIG,
+  INITIAL_CONTENTLESS_GROWTH_STATE,
   REPETITION_CHECK_INTERVAL_CHARS,
+  trackContentlessGrowth,
+  type ContentlessGrowthState,
 } from "./repetition.js";
 
 // A monotonic counter that never repeats verbatim: each pair's numerator and
@@ -21,8 +26,9 @@ const LOOP_SENTENCE =
 describe("detectRepetition", () => {
   test("flags a looped status sentence with an oscillating counter", () => {
     // Counters that flip between values keep the raw text periodic — the
-    // period just spans one full oscillation (two sentences here).
-    const iterations = Array.from({ length: 20 }, (_, i) =>
+    // period just spans one full oscillation (two sentences here), so hitting
+    // the repeat threshold takes twice as many iterations.
+    const iterations = Array.from({ length: 40 }, (_, i) =>
       LOOP_SENTENCE.replace("0/1.0", `${i % 2}/1.0`),
     );
     const text = `some earlier legitimate prose about the task. ${iterations.join("")}`;
@@ -73,6 +79,26 @@ describe("detectRepetition", () => {
     expect(detectRepetition(looped)).not.toBeNull();
   });
 
+  test("flags a short-phrase loop (10-char unit, observed live)", () => {
+    // The second captured incident: "Groaning. " emitted ~1,363 times. The
+    // old 16-char window floor never saw a 10-char unit.
+    const text = "Groaning. ".repeat(1300);
+    const hit = detectRepetition(text);
+    expect(hit).not.toBeNull();
+    expect(hit?.window).toBe("Groaning. ");
+    expect(hit?.repeats).toBeGreaterThanOrEqual(DEFAULT_REPETITION_CONFIG.repeatThreshold);
+  });
+
+  test("does not flag a repeated markdown table separator row", () => {
+    const row = "| ---------------------- | ---------------------- |\n";
+    expect(detectRepetition(`| Left | Right |\n${row.repeat(6)}`)).toBeNull();
+  });
+
+  test("does not flag a few identical code lines", () => {
+    const line = "  const result = await fetchData(request, options, context)\n";
+    expect(detectRepetition(line.repeat(3))).toBeNull();
+  });
+
   test("returns null for text shorter than one full window set", () => {
     expect(detectRepetition("short")).toBeNull();
     expect(detectRepetition("")).toBeNull();
@@ -170,12 +196,135 @@ test("flags a loop that injects zero-width spaces between identical windows", ()
   // detector misses the loop (observed in live thrash fleets).
   const window = "I'll open the remaining source files and implement the activity preview. ";
   const zwsp = "\u200B";
-  const text = (window + zwsp).repeat(12);
+  const text = (window + zwsp).repeat(20);
   const hit = detectRepetition(text);
   expect(hit).not.toBeNull();
   expect(hit?.repeats).toBeGreaterThanOrEqual(DEFAULT_REPETITION_CONFIG.repeatThreshold);
 });
 
+describe("folded text pass (DEFAULT_TEXT_FOLDED_REPETITION_CONFIG)", () => {
+  // Mirrors run.ts: digit-preserving default first, capped folded pass second.
+  function detectText(text: string) {
+    return (
+      detectRepetition(text) ??
+      detectRepetition(text, DEFAULT_TEXT_FOLDED_REPETITION_CONFIG, { normalizeDigits: true })
+    );
+  }
+
+  test("flags an incrementing counter flood in visible text", () => {
+    // Observed live: "14279 14280 14281…" streamed to inference-error.
+    const text = Array.from({ length: 500 }, (_, i) => `${14279 + i} `).join("");
+    expect(detectRepetition(text)).toBeNull();
+    expect(detectText(text)).not.toBeNull();
+  });
+
+  test("flags an incrementing pair-counter flood", () => {
+    // Observed live: "5620/5620. 5621/5621. …"
+    const text = Array.from({ length: 300 }, (_, i) => `${5620 + i}/${5620 + i}. `).join("");
+    expect(detectText(text)).not.toBeNull();
+  });
+
+  test("flags a repeated-timestamp flood", () => {
+    // Observed live: "18:22:27." emitted hundreds of times, with drift.
+    const text = Array.from({ length: 300 }, (_, i) => `18:22:${27 + (i % 30)}. `).join("");
+    expect(detectText(text)).not.toBeNull();
+  });
+
+  test("flags a zero-percent flood", () => {
+    // "0% " is a 3-char unit — under the plain 8-char window floor.
+    const text = "0% ".repeat(200);
+    expect(detectRepetition(text)).toBeNull();
+    expect(detectText(text)).not.toBeNull();
+  });
+
+  test("flags fence and brace floods", () => {
+    expect(detectText("```\n".repeat(100))).not.toBeNull();
+    expect(detectText("}\n".repeat(200))).not.toBeNull();
+  });
+
+  test("flags an emoji flood (surrogate-pair unit)", () => {
+    // Observed live: "🤔 " ×~40K chars. The unit is 3 UTF-16 units; the
+    // code-point reversal must keep the pair intact for it to stay periodic.
+    const text = "🤔 ".repeat(2000);
+    expect(detectText(text)).not.toBeNull();
+  });
+
+  test("does not flag a numbered list under the folded text pass", () => {
+    // Folds to a ~47-char period — refused by maxFoldedPeriodChars.
+    const items = Array.from(
+      { length: 400 },
+      (_, i) => `${i + 1}. Ran batch ${i + 1} and verified ${i * 3} records migrated\n`,
+    ).join("");
+    expect(detectText(`Migration progress:\n${items}`)).toBeNull();
+  });
+
+  test("does not flag a digit-varying markdown table under the folded text pass", () => {
+    const rows = Array.from(
+      { length: 40 },
+      (_, i) => `| 202${i % 10} | ${i * 10} requests | ${i} errors |\n`,
+    ).join("");
+    expect(detectText(`| Year | Volume | Errors |\n|---|---|---|\n${rows}`)).toBeNull();
+  });
+
+  test("a short user-requested enumeration stays under the folded repeat bar", () => {
+    // "print 1..40" folds to "0 " ×40 — under repeatThreshold 64.
+    const text = Array.from({ length: 40 }, (_, i) => `${i + 1} `).join("");
+    expect(detectText(text)).toBeNull();
+  });
+});
+
+describe("trackContentlessGrowth", () => {
+  function feed(tokens: readonly string[]): boolean {
+    let state: ContentlessGrowthState = INITIAL_CONTENTLESS_GROWTH_STATE;
+    for (const token of tokens) {
+      const next = trackContentlessGrowth(state, token);
+      if (next.hit) return true;
+      state = next.state;
+    }
+    return false;
+  }
+
+  test("flags a zero-width flood (ZWNJ/ZWJ walls, observed live)", () => {
+    // Observed live: 500–53,000 U+200C/U+200D chars per stream.
+    // detectRepetition strips invisibles before checking, so it must not be
+    // the only line of defense.
+    const flood = Array.from({ length: 60 }, () => "‌‍".repeat(32));
+    expect(detectRepetition(flood.join(""))).toBeNull();
+    expect(feed(flood)).toBe(true);
+  });
+
+  test("flags a flood even when prefixed by healthy prose", () => {
+    const tokens = [
+      "Let me look at the config first. ".repeat(4),
+      ...Array.from({ length: 100 }, () => "‍".repeat(64)),
+    ];
+    expect(feed(tokens)).toBe(true);
+  });
+
+  test("does not flag ordinary prose or sparse code", () => {
+    const tokens = Array.from(
+      { length: 200 },
+      (_, i) => `  const value${i} = await compute(input${i});\n\n`,
+    );
+    expect(feed(tokens)).toBe(false);
+  });
+
+  test("a visible-rich window re-arms rather than latching", () => {
+    // Enough visible content inside every window keeps the guard quiet no
+    // matter how long the stream runs.
+    const tokens = Array.from(
+      { length: 50 },
+      () => `${"‌".repeat(100)} some genuinely visible sentence with plenty of characters. `,
+    );
+    expect(feed(tokens)).toBe(false);
+  });
+
+  test("whitespace does not count as visible content", () => {
+    const raw = " \n\t".repeat(DEFAULT_CONTENTLESS_GROWTH_CONFIG.rawWindowChars);
+    expect(feed([raw])).toBe(true);
+  });
+});
+
 describe("appendCycleText", () => {
   test("keeps only the tail past the cap", () => {
     const text = appendCycleText("a".repeat(10), "b".repeat(10), 15);

src/subagent/repetition.ts

@@ -29,11 +29,18 @@ export interface RepetitionConfig {
   maxFoldedPeriodChars?: number;
 }
 
-// windowMinChars * repeatThreshold = 128 chars of exactly periodic text —
-// far beyond anything legitimate prose or code produces by accident.
+// windowMinChars * repeatThreshold = 8 * 16 = 128 chars of exactly periodic
+// text — far beyond anything legitimate prose or code produces by accident.
+// windowMinChars sits at 8 because live loops repeat units as short as 10
+// chars ("Groaning. " emitted ~1,363 times), which a 16-char floor never sees;
+// the repeat threshold rises to 16 in compensation so the minimum periodic
+// span stays at 128 chars. Structural tics that legitimately repeat ("- item\n"
+// normalizes to 7 chars) still fall under the window floor, and longer healthy
+// repeats (a 6-row table separator, 3 identical code lines, a repeat(4)
+// paragraph) stay far below 16 consecutive repeats.
 export const DEFAULT_REPETITION_CONFIG: RepetitionConfig = {
-  windowMinChars: 16,
-  repeatThreshold: 8,
+  windowMinChars: 8,
+  repeatThreshold: 16,
   probeChars: 8192,
 };
 
@@ -65,6 +72,33 @@ export const DEFAULT_THINKING_REPETITION_CONFIG: RepetitionConfig = {
   maxFoldedPeriodChars: 16,
 };
 
+// Second, folded pass over *text* streams, for the flood shapes the default
+// (digit-preserving) config is structurally blind to. Live traces ending as
+// inference-error (670K wasted streamed chars) showed: incrementing counters
+// ("14279 14280 14281…", "5620/5620. 5621/5621…" — never byte-periodic),
+// repeated timestamps with drift ("18:22:27. 18:22:28."), "0% 0% 0%…",
+// repeated "```\n" fences and "}\n" braces, and emoji floods ("🤔 " ×~40K
+// chars). All fold (or already normalize) to a tiny 2–16 char period.
+//
+// The safety story for visible text is different from thinking, hence the
+// stricter numbers rather than reusing the thinking config:
+// - maxFoldedPeriodChars 16 refuses prose-shaped folds exactly as it does for
+//   thinking: a numbered-list or table row folds to a ~30–50 char period and
+//   never fires (see the normalize() rationale below).
+// - windowMinChars 2 (vs thinking's 4) reaches the shortest observed units:
+//   "0% " and "} " fold to 2–3 chars, below the thinking floor.
+// - repeatThreshold 64 (vs 32): the residual false-positive risk for text is
+//   a user-requested raw enumeration ("print 1..N"), which folds to "0 " —
+//   a legit dump of a few dozen numbers stays under 64 consecutive repeats,
+//   while the observed floods repeat thousands of times. Minimum folded
+//   periodic span: 2 * 64 = 128 chars.
+export const DEFAULT_TEXT_FOLDED_REPETITION_CONFIG: RepetitionConfig = {
+  windowMinChars: 2,
+  repeatThreshold: 64,
+  probeChars: 8192,
+  maxFoldedPeriodChars: 16,
+};
+
 export interface RepetitionHit {
   /** The normalized window that repeats. */
   window: string;
@@ -85,9 +119,11 @@ export interface RepetitionHit {
 // evade the detector. Observed thrash loops used U+200B between repeats.
 // `normalizeDigits` opts a caller into folding digit runs to one placeholder,
 // which collapses a monotonic counter's varying digits into a repeating unit.
-// Reserved for thinking streams (see DEFAULT_THINKING_REPETITION_CONFIG),
-// which are never rendered to the user and so carry none of the numbered-list
-// / table false-positive risk that keeps text normalization digit-preserving.
+// The digit-preserving default protects text streams' numbered lists and
+// tables; folded detection runs on them only as a second pass capped to tiny
+// periods (DEFAULT_TEXT_FOLDED_REPETITION_CONFIG), and uncapped-in-spirit on
+// thinking streams (DEFAULT_THINKING_REPETITION_CONFIG), which are never
+// rendered to the user and so carry less false-positive cost.
 function normalize(text: string, normalizeDigits: boolean): string {
   const stripped = text
     .replace(/[\u200B-\u200D\uFEFF\u00AD\u2060\u200E\u200F\u202A-\u202E\u2066-\u2069]/g, "")
@@ -121,6 +157,10 @@ export function detectRepetition(
   const tail = normalize(text.slice(-config.probeChars), opts.normalizeDigits ?? false);
   if (tail.length < config.windowMinChars * config.repeatThreshold) return null;
 
+  // Reverse by code point so surrogate pairs survive intact — an emoji flood
+  // ("🤔 " ×thousands) must stay byte-periodic after reversal. The prefix
+  // function and window extraction then both count plain UTF-16 units of the
+  // (pair-preserving) reversed string, so periods and slices stay consistent.
   const reversed = [...tail].reverse().join("");
   const pi = prefixFunction(reversed);
 
@@ -140,3 +180,68 @@ export function detectRepetition(
   }
   return best;
 }
+
+/** Tunables for the contentless-growth guard. */
+export interface ContentlessGrowthConfig {
+  /** Raw streamed chars per measurement window. */
+  rawWindowChars: number;
+  /** A window with fewer visible chars than this counts as contentless. */
+  minVisibleChars: number;
+}
+
+// detectRepetition can never see a zero-width flood: normalize() strips
+// invisibles *before* the periodicity check, so thousands of U+200C/U+200D
+// chars (observed live: 500–53,000 per stream) collapse to a short, healthy-
+// looking string. This guard watches the inverse signal — raw text keeps
+// growing while its visible content does not. The bar: 2048 raw chars with
+// fewer than 32 visible. Legitimate sparse output never approaches it — even
+// a heavily indented code block or a wide table row carries hundreds of
+// visible chars per 2048 raw, and a healthy stream would need 64:1
+// invisible-or-whitespace-to-content to trip it.
+export const DEFAULT_CONTENTLESS_GROWTH_CONFIG: ContentlessGrowthConfig = {
+  rawWindowChars: 2048,
+  minVisibleChars: 32,
+};
+
+export interface ContentlessGrowthState {
+  /** Raw chars accumulated in the current window. */
+  rawChars: number;
+  /** Visible (invisible-stripped, whitespace-removed) chars in the window. */
+  visibleChars: number;
+}
+
+export const INITIAL_CONTENTLESS_GROWTH_STATE: ContentlessGrowthState = {
+  rawChars: 0,
+  visibleChars: 0,
+};
+
+// Whitespace is removed rather than collapsed: a window of pure newlines is
+// as contentless as one of pure ZWJ, and counting collapsed runs would let a
+// space-interleaved flood (ZWJ, space, ZWJ, space, …) smuggle half its
+// length past the epsilon.
+function visibleLength(token: string): number {
+  return normalize(token, false).replace(/ /g, "").length;
+}
+
+/**
+ * Fold one streamed token into the contentless-growth window. Returns the
+ * next state and whether the just-completed window was contentless: raw text
+ * grew by a full window while visible content grew less than the epsilon.
+ * Pure reducer — the caller owns the state across deltas; the window resets
+ * on completion either way, so one visible-rich window re-arms the guard.
+ */
+export function trackContentlessGrowth(
+  state: ContentlessGrowthState,
+  token: string,
+  config: ContentlessGrowthConfig = DEFAULT_CONTENTLESS_GROWTH_CONFIG,
+): { state: ContentlessGrowthState; hit: boolean } {
+  const rawChars = state.rawChars + token.length;
+  const visibleChars = state.visibleChars + visibleLength(token);
+  if (rawChars < config.rawWindowChars) {
+    return { state: { rawChars, visibleChars }, hit: false };
+  }
+  return {
+    state: INITIAL_CONTENTLESS_GROWTH_STATE,
+    hit: visibleChars < config.minVisibleChars,
+  };
+}