Reuse unchanged large thread timelines

apps/server/src/routes/threads/data.ts

@@ -439,45 +439,47 @@ export function registerThreadDataRoutes(app: Hono, deps: AppDeps): void {
       includeDiagnosticOperations,
       completedTurnDisplay,
     };
-    const full = timelineCache.getOrBuild(
-      thread.id,
-      buildThreadTimelineCacheKey({ ...keyArgs, maxSeq }),
-      () => {
-        const { profile, response } = buildThreadTimelineWithProfile(
-          deps.db,
-          thread,
-          {
-            completedTurnDisplay,
-            eventBudget,
-            includeDiagnosticOperations,
-            includeNestedRows,
-            maxInlineOutputChars: DEFAULT_MAX_INLINE_OUTPUT_CHARS,
-            maxSeq,
-            page,
-            providerDisplayName,
-            planCommand: resolveProviderPlanCommand(
-              deps.providerRegistry,
-              thread.providerId,
-            ),
-            summaryOnly,
-          },
-        );
-        slowTimelineBuildLogger.log({ profile, threadId: thread.id });
-        const truncated = truncateTimelineResponseOutputs(
-          response,
-          DEFAULT_MAX_INLINE_OUTPUT_CHARS,
-        );
-        return includeNestedRows
-          ? truncated
-          : previewTimelineResponseOutputs(truncated);
-      },
-    );
+    const paramsKey = buildThreadTimelineParamsKey(keyArgs);
+    const full =
+      timelineLatestRowsCache.getResponse(thread.id, paramsKey, maxSeq) ??
+      timelineCache.getOrBuild(
+        thread.id,
+        buildThreadTimelineCacheKey({ ...keyArgs, maxSeq }),
+        () => {
+          const { profile, response } = buildThreadTimelineWithProfile(
+            deps.db,
+            thread,
+            {
+              completedTurnDisplay,
+              eventBudget,
+              includeDiagnosticOperations,
+              includeNestedRows,
+              maxInlineOutputChars: DEFAULT_MAX_INLINE_OUTPUT_CHARS,
+              maxSeq,
+              page,
+              providerDisplayName,
+              planCommand: resolveProviderPlanCommand(
+                deps.providerRegistry,
+                thread.providerId,
+              ),
+              summaryOnly,
+            },
+          );
+          slowTimelineBuildLogger.log({ profile, threadId: thread.id });
+          const truncated = truncateTimelineResponseOutputs(
+            response,
+            DEFAULT_MAX_INLINE_OUTPUT_CHARS,
+          );
+          return includeNestedRows
+            ? truncated
+            : previewTimelineResponseOutputs(truncated);
+        },
+      );
 
     const afterSequence = parseOptionalInteger(
       query.afterSequence,
       "afterSequence",
     );
-    const paramsKey = buildThreadTimelineParamsKey(keyArgs);
     const previous =
       afterSequence === undefined
         ? undefined
@@ -486,10 +488,7 @@ export function registerThreadDataRoutes(app: Hono, deps: AppDeps): void {
       previous === undefined
         ? undefined
         : computeTimelineRowDelta(previous.rows, full.rows);
-    timelineLatestRowsCache.set(thread.id, paramsKey, {
-      maxSeq,
-      rows: full.rows,
-    });
+    timelineLatestRowsCache.set(thread.id, paramsKey, full);
 
     return context.json({
       ...(delta === undefined ? full : { ...full, rows: [], delta }),

apps/server/src/services/threads/timeline-latest-rows-cache.ts

@@ -1,4 +1,4 @@
-import type { TimelineRow } from "@bb/server-contract";
+import type { ThreadTimelineResponse, TimelineRow } from "@bb/server-contract";
 
 const DEFAULT_MAX_ENTRIES = 64;
 const DEFAULT_RING_SIZE = 4;
@@ -9,17 +9,23 @@ interface TimelineLatestRows {
 }
 
 interface TimelineLatestRowsCache {
+  getResponse(
+    threadId: string,
+    paramsKey: string,
+    maxSeq: number,
+  ): ThreadTimelineResponse | undefined;
   get(
     threadId: string,
     paramsKey: string,
     maxSeq: number,
   ): TimelineLatestRows | undefined;
   invalidateThread(threadId: string): void;
-  set(threadId: string, paramsKey: string, value: TimelineLatestRows): void;
+  set(threadId: string, paramsKey: string, value: ThreadTimelineResponse): void;
   readonly size: number;
 }
 
 interface TimelineLatestRowsCacheEntry {
+  response: ThreadTimelineResponse;
   ring: TimelineLatestRows[];
   threadId: string;
 }
@@ -37,6 +43,18 @@ export function createTimelineLatestRowsCache(
   }
 
   return {
+    getResponse(threadId, paramsKey, maxSeq) {
+      const entry = entries.get(paramsKey);
+      if (
+        entry === undefined ||
+        entry.threadId !== threadId ||
+        entry.response.maxSeq !== maxSeq
+      ) {
+        return undefined;
+      }
+      touch(paramsKey, entry);
+      return entry.response;
+    },
     get(threadId, paramsKey, maxSeq) {
       const entry = entries.get(paramsKey);
       if (entry === undefined || entry.threadId !== threadId) {
@@ -55,15 +73,18 @@ export function createTimelineLatestRowsCache(
     set(threadId, paramsKey, value) {
       const cached = entries.get(paramsKey);
       const entry =
-        cached?.threadId === threadId ? cached : { ring: [], threadId };
+        cached?.threadId === threadId
+          ? cached
+          : { response: value, ring: [], threadId };
+      entry.response = value;
       const ring = entry.ring;
       const existingIndex = ring.findIndex(
         (entry) => entry.maxSeq === value.maxSeq,
       );
       if (existingIndex !== -1) {
         ring.splice(existingIndex, 1);
       }
-      ring.push(value);
+      ring.push({ maxSeq: value.maxSeq, rows: value.rows });
       while (ring.length > ringSize) {
         ring.shift();
       }

apps/server/test/public/public-thread-timeline-response-reuse.test.ts

@@ -0,0 +1,97 @@
+import { expect, it, vi } from "vitest";
+import { defaultAppSettings, threadScope, turnScope } from "@bb/domain";
+import { threadTimelineResponseSchema } from "@bb/server-contract";
+import * as timelineBuilder from "../../src/services/threads/timeline.js";
+import { readJson } from "../helpers/json.js";
+import { seedEvent, seedThreadFixture } from "../helpers/seed.js";
+import { withTestHarness } from "../helpers/test-app.js";
+
+it("reuses an unchanged large timeline and rebuilds after new events", async () => {
+  await withTestHarness(async (harness) => {
+    const { environment, thread } = seedThreadFixture(harness);
+    const settings = await harness.app.request("/api/v1/settings/general", {
+      method: "PUT",
+      headers: { "content-type": "application/json" },
+      body: JSON.stringify({
+        ...defaultAppSettings,
+        providerCompletedTurnDisplay: { [thread.providerId]: "flat" },
+      }),
+    });
+    expect(settings.status).toBe(200);
+    const scope = {
+      environmentId: environment.id,
+      threadId: thread.id,
+      providerThreadId: "response-reuse",
+      scope: turnScope("turn-1"),
+    };
+    let sequence = 1;
+    seedEvent(harness.deps, {
+      ...scope,
+      sequence: sequence++,
+      type: "turn/started",
+      data: {},
+    });
+    for (let index = 0; index < 110; index += 1) {
+      seedEvent(harness.deps, {
+        ...scope,
+        sequence: sequence++,
+        type: "item/completed",
+        data: {
+          item: {
+            id: `tool-${index}`,
+            type: "toolCall",
+            tool: "read",
+            status: "completed",
+            result: "file contents",
+          },
+        },
+      });
+      seedEvent(harness.deps, {
+        ...scope,
+        sequence: sequence++,
+        type: "item/completed",
+        data: {
+          item: {
+            id: `message-${index}`,
+            type: "agentMessage",
+            text: `Read file ${index}.`,
+          },
+        },
+      });
+    }
+    seedEvent(harness.deps, {
+      ...scope,
+      sequence: sequence++,
+      type: "turn/completed",
+      data: { status: "completed" },
+    });
+    const fetchTimeline = async () => {
+      const response = await harness.app.request(
+        `/api/v1/threads/${thread.id}/timeline`,
+      );
+      expect(response.status).toBe(200);
+      return threadTimelineResponseSchema.parse(await readJson(response));
+    };
+    const build = vi.spyOn(timelineBuilder, "buildThreadTimelineWithProfile");
+    try {
+      const first = await fetchTimeline();
+      expect(first.rows.length).toBeGreaterThan(200);
+      build.mockClear();
+      expect(await fetchTimeline()).toEqual(first);
+      expect(build).not.toHaveBeenCalled();
+      seedEvent(harness.deps, {
+        ...scope,
+        scope: threadScope(),
+        sequence,
+        type: "system/manager/user_message",
+        data: { text: "New user message" },
+      });
+      const updated = await fetchTimeline();
+      expect(updated.maxSeq).toBe(sequence);
+      expect(updated.rows).not.toEqual(first.rows);
+      expect(build).toHaveBeenCalledTimes(1);
+    } finally {
+      build.mockRestore();
+    }
+  });
+});

apps/server/test/services/threads/timeline-latest-rows-cache.test.ts

@@ -1,5 +1,5 @@
 import { describe, expect, it } from "vitest";
-import type { TimelineRow } from "@bb/server-contract";
+import type { ThreadTimelineResponse, TimelineRow } from "@bb/server-contract";
 import { createTimelineLatestRowsCache } from "../../../src/services/threads/timeline-latest-rows-cache.js";
 
 function rows(label: string): TimelineRow[] {
@@ -21,16 +21,54 @@ function rows(label: string): TimelineRow[] {
   ];
 }
 
+function response(maxSeq: number, label: string): ThreadTimelineResponse {
+  return {
+    rows: rows(label),
+    maxSeq,
+    contextBoundarySeq: null,
+    completedTurnDisplay: "collapse",
+    activePromptMode: null,
+    activeThinking: null,
+    activeWorkflows: [],
+    activeBackgroundCommands: [],
+    pendingTodos: null,
+    goal: null,
+    modelFallback: null,
+    timelinePage: {
+      kind: "latest",
+      segmentLimit: 20,
+      returnedSegmentCount: 0,
+      hasOlderRows: false,
+      olderCursor: null,
+    },
+  };
+}
+
 describe("createTimelineLatestRowsCache", () => {
+  it("reuses only the most recently stored response at its exact revision", () => {
+    const cache = createTimelineLatestRowsCache();
+    const first = response(1, "first");
+    const second = response(2, "second");
+    cache.set("thr_x", "k", first);
+    expect(cache.getResponse("thr_x", "k", 1)).toBe(first);
+    expect(cache.getResponse("thr_y", "k", 1)).toBeUndefined();
+    expect(cache.getResponse("thr_x", "other", 1)).toBeUndefined();
+    expect(cache.getResponse("thr_x", "k", 2)).toBeUndefined();
+    cache.set("thr_x", "k", second);
+    expect(cache.getResponse("thr_x", "k", 1)).toBeUndefined();
+    expect(cache.getResponse("thr_x", "k", 2)).toBe(second);
+    expect(cache.get("thr_x", "k", 1)?.rows).toBe(first.rows);
+  });
+
   it("keeps a ring of recent revisions per params key and evicts the oldest", () => {
     const cache = createTimelineLatestRowsCache({ ringSize: 3 });
     for (const maxSeq of [1, 2, 3]) {
-      cache.set("thr_x", "k", { maxSeq, rows: rows(`r${maxSeq}`) });
+      cache.set("thr_x", "k", response(maxSeq, `r${maxSeq}`));
     }
     expect(cache.get("thr_x", "k", 1)?.rows).toEqual(rows("r1"));
     expect(cache.get("thr_x", "k", 3)?.rows).toEqual(rows("r3"));
 
-    cache.set("thr_x", "k", { maxSeq: 4, rows: rows("r4") });
+    cache.set("thr_x", "k", response(4, "r4"));
     expect(cache.get("thr_x", "k", 1)).toBeUndefined();
     expect(cache.get("thr_x", "k", 2)?.rows).toEqual(rows("r2"));
     expect(cache.get("thr_x", "k", 4)?.rows).toEqual(rows("r4"));
@@ -40,38 +78,40 @@ describe("createTimelineLatestRowsCache", () => {
 
   it("a repeated set at the same revision refreshes recency without consuming a ring slot", () => {
     const cache = createTimelineLatestRowsCache({ ringSize: 2 });
-    cache.set("thr_x", "k", { maxSeq: 1, rows: rows("r1") });
-    cache.set("thr_x", "k", { maxSeq: 2, rows: rows("r2") });
-    cache.set("thr_x", "k", { maxSeq: 1, rows: rows("r1") });
-    cache.set("thr_x", "k", { maxSeq: 1, rows: rows("r1") });
+    cache.set("thr_x", "k", response(1, "r1"));
+    cache.set("thr_x", "k", response(2, "r2"));
+    cache.set("thr_x", "k", response(1, "r1"));
+    cache.set("thr_x", "k", response(1, "r1"));
     expect(cache.get("thr_x", "k", 1)?.rows).toEqual(rows("r1"));
     expect(cache.get("thr_x", "k", 2)?.rows).toEqual(rows("r2"));
-    cache.set("thr_x", "k", { maxSeq: 3, rows: rows("r3") });
+    cache.set("thr_x", "k", response(3, "r3"));
     expect(cache.get("thr_x", "k", 2)).toBeUndefined();
     expect(cache.get("thr_x", "k", 1)?.rows).toEqual(rows("r1"));
     expect(cache.get("thr_x", "k", 3)?.rows).toEqual(rows("r3"));
   });
 
   it("bounds params keys LRU-style; a lookup counts as use", () => {
     const cache = createTimelineLatestRowsCache({ maxEntries: 2 });
-    cache.set("thr_a", "a", { maxSeq: 1, rows: rows("a") });
-    cache.set("thr_b", "b", { maxSeq: 1, rows: rows("b") });
+    cache.set("thr_a", "a", response(1, "a"));
+    cache.set("thr_b", "b", response(1, "b"));
     expect(cache.get("thr_a", "a", 1)).toBeDefined();
-    cache.set("thr_c", "c", { maxSeq: 1, rows: rows("c") });
+    cache.set("thr_c", "c", response(1, "c"));
     expect(cache.size).toBe(2);
     expect(cache.get("thr_b", "b", 1)).toBeUndefined();
+    expect(cache.getResponse("thr_b", "b", 1)).toBeUndefined();
     expect(cache.get("thr_a", "a", 1)?.rows).toEqual(rows("a"));
     expect(cache.get("thr_c", "c", 1)?.rows).toEqual(rows("c"));
   });
 
   it("invalidates only revisions for the rewritten thread", () => {
     const cache = createTimelineLatestRowsCache();
-    cache.set("thr_x", "x", { maxSeq: 1, rows: rows("x") });
-    cache.set("thr_y", "y", { maxSeq: 1, rows: rows("y") });
+    cache.set("thr_x", "x", response(1, "x"));
+    cache.set("thr_y", "y", response(1, "y"));
 
     cache.invalidateThread("thr_x");
 
     expect(cache.get("thr_x", "x", 1)).toBeUndefined();
+    expect(cache.getResponse("thr_x", "x", 1)).toBeUndefined();
     expect(cache.get("thr_y", "y", 1)?.rows).toEqual(rows("y"));
     expect(cache.size).toBe(1);
   });