Recover resume transcript when a turns segment has malformed JSON

src/session/optimized-context-store.test.ts

@@ -240,6 +240,39 @@ describe("loadRecentTurns", () => {
     const loaded = await loadRecentTurns(dir, 5);
     expect(loaded.map((t) => (t.content[0] as { text: string }).text)).toEqual(["a", "b"]);
   });
+
+  // Mid-file null holes (writer bug fixed under CL-5934) are not a torn final
+  // line — resume hydrate must skip them and still paint the surrounding turns.
+  test("skips mid-file null-byte holes and returns usable turns", async () => {
+    const dir = tempDir();
+    const goodA = JSON.stringify(turn("a"));
+    const goodB = JSON.stringify(turn("b"));
+    fs.writeFileSync(path.join(dir, TURNS_FILE), `${goodA}\n${"\0".repeat(64)}\n${goodB}\n`);
+    fs.writeFileSync(path.join(dir, segmentFileName(TURNS_FILE, 1)), jsonl([turn("c")]));
+
+    const loaded = await loadRecentTurns(dir, 10);
+    expect(loaded.map((t) => (t.content[0] as { text: string }).text)).toEqual(["a", "b", "c"]);
+  });
+
+  test("skips a non-tail malformed line in a sealed older segment", async () => {
+    const dir = tempDir();
+    fs.writeFileSync(path.join(dir, TURNS_FILE), jsonl([turn("a")]));
+    fs.writeFileSync(
+      path.join(dir, segmentFileName(TURNS_FILE, 1)),
+      `${JSON.stringify(turn("b"))}\nnot-json-at-all\n${JSON.stringify(turn("c"))}\n`,
+    );
+    fs.writeFileSync(path.join(dir, segmentFileName(TURNS_FILE, 2)), jsonl([turn("d")]));
+
+    const loaded = await loadRecentTurns(dir, 10);
+    expect(loaded.map((t) => (t.content[0] as { text: string }).text)).toEqual(["a", "b", "c", "d"]);
+  });
+
+  test("names the failing segment when every line is unrecoverable", async () => {
+    const dir = tempDir();
+    fs.writeFileSync(path.join(dir, TURNS_FILE), "\0\0\0\nnot-json\n");
+
+    await expect(loadRecentTurns(dir, 5)).rejects.toThrow(/turns\.jsonl/);
+  });
 });
 
 describe("createOptimizedContextStore checkpoint", () => {

src/session/optimized-context-store.ts

@@ -64,33 +64,83 @@ function sanitizeCallId(callId: string): string {
   return callId.replace(UNSAFE_FILENAME_CHARS, "_");
 }
 
+type ParseSegmentTurnsOpts = {
+  /** Drop a torn final line on the active (last) segment rather than aborting. */
+  tolerateTornTail?: boolean;
+  /**
+   * Display-only recovery: skip lines that fail JSON parse or schema validation
+   * instead of aborting the whole segment. Used by `loadRecentTurns` so resume
+   * hydrate can paint surrounding history past mid-file null holes / garbage.
+   * Reactor `ContextStore.load()` must not set this — it still hard-fails.
+   */
+  skipMalformedLines?: boolean;
+  /** Included in thrown error messages so operators can name the failing file. */
+  segmentName?: string;
+};
+
+type ParseSegmentTurnsResult = {
+  turns: ConversationTurn[];
+  skippedMalformed: number;
+};
+
+function segmentErrorLabel(segmentName: string | undefined): string {
+  return segmentName === undefined ? "turns segment" : `turns segment ${segmentName}`;
+}
+
 /**
  * Parse conversation turns out of one JSONL segment. A crash can tear the final
  * line of the active (last) segment mid-write; when `tolerateTornTail` is set a
  * final line that fails to parse is dropped rather than aborting the resume.
+ * When `skipMalformedLines` is set, non-tail (and non-torn) failures are skipped
+ * so display-only readers can recover usable history around mid-file corruption.
  */
-function parseSegmentTurns(text: string, tolerateTornTail: boolean): ConversationTurn[] {
-  if (text.length === 0) return [];
+function parseSegmentTurns(
+  text: string,
+  opts: ParseSegmentTurnsOpts = {},
+): ParseSegmentTurnsResult {
+  const tolerateTornTail = opts.tolerateTornTail === true;
+  const skipMalformedLines = opts.skipMalformedLines === true;
+  const label = segmentErrorLabel(opts.segmentName);
+
+  if (text.length === 0) return { turns: [], skippedMalformed: 0 };
   const lines = text.split("\n");
   if (lines[lines.length - 1] === "") lines.pop();
 
   const turns: ConversationTurn[] = [];
+  let skippedMalformed = 0;
   for (let i = 0; i < lines.length; i++) {
     const isLast = i === lines.length - 1;
     let raw: unknown;
     try {
       raw = JSON.parse(lines[i]!);
     } catch (cause) {
       if (tolerateTornTail && isLast) break;
-      throw new Error("turns segment has malformed JSON", { cause });
+      if (skipMalformedLines) {
+        skippedMalformed += 1;
+        log.warn("skipped malformed JSON in {segment} at line {line}", {
+          segment: opts.segmentName ?? "(unnamed)",
+          line: i + 1,
+        });
+        continue;
+      }
+      throw new Error(`${label} has malformed JSON`, { cause });
     }
     const result = ConversationTurnSchema(raw);
     if (result instanceof type.errors) {
-      throw new Error(`turns segment has unexpected structure: ${result.summary}`);
+      if (skipMalformedLines) {
+        skippedMalformed += 1;
+        log.warn("skipped unexpected structure in {segment} at line {line}: {summary}", {
+          segment: opts.segmentName ?? "(unnamed)",
+          line: i + 1,
+          summary: result.summary,
+        });
+        continue;
+      }
+      throw new Error(`${label} has unexpected structure: ${result.summary}`);
     }
     turns.push(result);
   }
-  return turns;
+  return { turns, skippedMalformed };
 }
 
 // Mirrors assertWellFormedToolSequence without throwing. Used to choose the
@@ -176,17 +226,39 @@ export async function loadRecentTurns(
 
   const collectedNewestFirst: ConversationTurn[][] = [];
   let total = 0;
+  // First segment that contributed only unrecoverable garbage (no good lines).
+  // Used only when the whole window yields zero turns — partial recovery wins.
+  let firstUnrecoverableSegment: string | undefined;
+
   for (let i = segments.length - 1; i >= 0; i--) {
-    const text = await fs.promises.readFile(path.join(dir, segments[i]!), "utf-8");
+    const segmentName = segments[i]!;
+    let text: string;
+    try {
+      text = await fs.promises.readFile(path.join(dir, segmentName), "utf-8");
+    } catch (cause) {
+      throw new Error(`turns segment ${segmentName} could not be read`, { cause });
+    }
     // Only the active (last) segment can be mid-write; sealed ones are complete.
-    const turns = parseSegmentTurns(text, i === segments.length - 1);
+    // Display-only path: skip mid-file null holes / malformed lines so resume
+    // hydrate paints usable history instead of aborting on one bad line.
+    const { turns, skippedMalformed } = parseSegmentTurns(text, {
+      tolerateTornTail: i === segments.length - 1,
+      skipMalformedLines: true,
+      segmentName,
+    });
+    if (skippedMalformed > 0 && turns.length === 0) {
+      firstUnrecoverableSegment ??= segmentName;
+    }
     collectedNewestFirst.push(turns);
     total += turns.length;
     if (total >= minTurns) break;
   }
 
   const turns: ConversationTurn[] = [];
   for (let i = collectedNewestFirst.length - 1; i >= 0; i--) turns.push(...collectedNewestFirst[i]!);
+  if (turns.length === 0 && firstUnrecoverableSegment !== undefined) {
+    throw new Error(`turns segment ${firstUnrecoverableSegment} has malformed JSON`);
+  }
   return turns;
 }
 
@@ -310,8 +382,9 @@ export async function createOptimizedContextStore(dir: string): Promise<ContextS
   ): Promise<ConversationTurn[]> {
     if (extraTexts.length === 0) return baseTurns;
 
-    const parsedExtras = extraTexts.map((text, index) =>
-      parseSegmentTurns(text, index === extraTexts.length - 1),
+    const parsedExtras = extraTexts.map(
+      (text, index) =>
+        parseSegmentTurns(text, { tolerateTornTail: index === extraTexts.length - 1 }).turns,
     );
     const keepExtras = longestWellFormedExtraCount(baseTurns, parsedExtras);
 
@@ -353,7 +426,7 @@ export async function createOptimizedContextStore(dir: string): Promise<ContextS
       const parsedExtras: ConversationTurn[][] = [];
       for (const name of extraNames) {
         const text = await runGit(dir, ["show", `${hash}:${name}`]);
-        parsedExtras.push(parseSegmentTurns(text, false));
+        parsedExtras.push(parseSegmentTurns(text, { tolerateTornTail: false }).turns);
       }
       const keepExtras = longestWellFormedExtraCount(baseTurns, parsedExtras);
       if (keepExtras === 0) return baseTurns;