From ef2097cab26b7e5535a36890c86fa8a31c1b5534 Mon Sep 17 00:00:00 2001 From: Michael Yong Date: Wed, 16 Sep 2026 17:16:22 -0700 Subject: [PATCH 1/7] Compact settled completed items at their first lifecycle position --- .../services/threads/thread-fork-history.ts | 15 +- apps/server/src/services/threads/timeline.ts | 20 +- .../threads/timeline-event-budget.test.ts | 122 ++++++ apps/server/test/system/event-pruning.test.ts | 77 ++++ .../completed-item-compaction-verification.md | 98 +++++ docs/completed-item-compaction.md | 62 +++ packages/db/src/completed-item-history.ts | 223 ++++++++++ .../db/src/data/completed-item-compaction.ts | 395 ++++++++++++++++++ .../db/src/data/completed-item-history.ts | 64 +++ packages/db/src/data/events.ts | 2 +- packages/db/src/data/thread-pruning.ts | 16 + packages/db/src/index.ts | 2 + packages/db/src/schema.ts | 1 + .../data/completed-item-compaction.test.ts | 366 ++++++++++++++++ packages/db/test/data/thread-pruning.test.ts | 4 +- packages/db/test/migrate.test.ts | 10 + packages/plugin-api-map/src/surfaces.ts | 1 + .../src/templates/bb-guide-threads.md | 7 + .../bb-cli/references/thread-operation.md | 6 +- 19 files changed, 1478 insertions(+), 13 deletions(-) create mode 100644 docs/completed-item-compaction-verification.md create mode 100644 docs/completed-item-compaction.md create mode 100644 packages/db/src/completed-item-history.ts create mode 100644 packages/db/src/data/completed-item-compaction.ts create mode 100644 packages/db/src/data/completed-item-history.ts create mode 100644 packages/db/test/data/completed-item-compaction.test.ts diff --git a/apps/server/src/services/threads/thread-fork-history.ts b/apps/server/src/services/threads/thread-fork-history.ts index 8e3742a8996..13461088e84 100644 --- a/apps/server/src/services/threads/thread-fork-history.ts +++ b/apps/server/src/services/threads/thread-fork-history.ts @@ -1,3 +1,4 @@ +import { expandSelectedCompletedItemRows } from "@bb/db"; import { copyStoredThreadEventsInTransaction, findLastCompletedRootStoredTurn, @@ -262,11 +263,15 @@ function selectInheritedForkEventRows( deps: Pick, args: { historyEndSequence: number; sourceThreadId: string }, ): StoredEventRow[] { - const rows = listStoredEventRows(deps.db, { - beforeSequence: args.historyEndSequence + 1, - threadId: args.sourceThreadId, - types: INHERITED_EVENT_TYPES, - }); + const rows = expandSelectedCompletedItemRows( + deps.db, + listStoredEventRows(deps.db, { + beforeSequence: args.historyEndSequence + 1, + threadId: args.sourceThreadId, + types: INHERITED_EVENT_TYPES, + }), + args.historyEndSequence, + ).filter((row) => INHERITED_EVENT_TYPES.some((type) => type === row.type)); const completedTurnIds = new Set(); const acceptedClientRequestIds = new Set(); for (const row of rows) { diff --git a/apps/server/src/services/threads/timeline.ts b/apps/server/src/services/threads/timeline.ts index 46069a0d2a9..0c0fa69d231 100644 --- a/apps/server/src/services/threads/timeline.ts +++ b/apps/server/src/services/threads/timeline.ts @@ -1,3 +1,4 @@ +import { expandSelectedCompletedItemRows } from "@bb/db"; import { paginateTimelineContents } from "./timeline-content-pagination.js"; import { getTimelineGroupingContext, @@ -1357,7 +1358,11 @@ function buildThreadTimelineInternal( rows: hydrateRetainedEventOutputRows(db, storedEventSelection.rows), } : storedEventSelection; - const rawEventRows = eventSelection.rows; + const rawEventRows = expandSelectedCompletedItemRows( + db, + eventSelection.rows, + snapshot.maxSeq, + ); profile.eventDataBytes = byteLengthOfStoredEventRows(rawEventRows); profile.eventRowCount = rawEventRows.length; profile.selectionStrategy = eventSelection.strategy; @@ -1624,9 +1629,10 @@ export function buildThreadConversationOutline( sequenceStart: contextBoundarySeq ?? 0, threadId: thread.id, }); - const decodedRawEvents = rawEventRows.map((row) => - toThreadEventWithMeta(row), - ); + const decodedRawEvents = expandSelectedCompletedItemRows( + db, + rawEventRows, + ).map((row) => toThreadEventWithMeta(row)); const decodedEvents = compactThreadTimelineSummaryEvents(decodedRawEvents); const clientRequestContextRows = selectClientRequestContextRows(db, { rows: rawEventRows, @@ -1926,7 +1932,11 @@ function buildTimelineTurnSummaryDetailsPage( : sourceSeqStart, sourceRange.sourceSeqStart, ); - const projectionEvents = projectionEventRows + const projectionEvents = expandSelectedCompletedItemRows( + db, + projectionEventRows, + snapshot.maxSeq, + ) .filter((row) => row.sequence <= snapshot.maxSeq) .map((row) => toThreadEventWithMeta(row)); const children = buildThreadTimelineTurnDetailsFromEvents({ diff --git a/apps/server/test/services/threads/timeline-event-budget.test.ts b/apps/server/test/services/threads/timeline-event-budget.test.ts index 7d4d807f713..6432d76c040 100644 --- a/apps/server/test/services/threads/timeline-event-budget.test.ts +++ b/apps/server/test/services/threads/timeline-event-budget.test.ts @@ -18,6 +18,8 @@ import { } from "@bb/domain"; import type { ClientTurnRequestId, Thread } from "@bb/domain"; import { + advanceThreadPruning, + listStoredEventRows, createConnection, createProject, createThread, @@ -1390,3 +1392,123 @@ it("resolves acceptance after the next conversation boundary", () => { expect(rows).toEqual(expected.rows); db.$client.close(); }); + +it.each([1, 2, 5, 20])( + "traverses compacted items completely with event budget %i and a small byte budget", + (eventBudget) => { + const { db, thread } = setup(); + try { + insertTurns(db, thread, 3, [2, 15, 2]); + db.$client + .prepare("UPDATE events SET sequence = -sequence WHERE thread_id = ?") + .run(thread.id); + db.$client + .prepare( + "UPDATE events SET sequence = -10 * sequence WHERE thread_id = ?", + ) + .run(thread.id); + const owners = listStoredEventRows(db, { + threadId: thread.id, + types: ["item/completed"], + }); + const additions: Parameters[2] = []; + const ends = new Map(); + for (const row of owners) { + if (!row.turnId || !row.itemId) + throw new Error("Missing fixture scope"); + const base = { + threadId: thread.id, + scope: turnScope(row.turnId), + providerThreadId, + itemId: row.itemId, + itemKind: row.itemKind, + parentToolCallId: null, + }; + const data = JSON.parse(row.data); + additions.push({ + ...base, + type: "item/started", + sequence: row.sequence - 2, + createdAt: row.createdAt - 2, + data: JSON.stringify({ item: { ...data.item, text: "" } }), + }); + additions.push({ + ...base, + type: "item/agentMessage/delta", + itemKind: null, + sequence: row.sequence - 1, + createdAt: row.createdAt - 1, + data: JSON.stringify({ itemId: row.itemId, delta: data.item.text }), + }); + ends.set(row.turnId, row.sequence + 1); + } + for (const [turnId, sequence] of ends) + additions.push({ + threadId: thread.id, + scope: turnScope(turnId), + providerThreadId, + itemId: null, + itemKind: null, + parentToolCallId: null, + type: "turn/completed", + sequence, + data: JSON.stringify({ status: "completed" }), + }); + insertEvents( + db, + noopNotifier, + additions.sort((a, b) => a.sequence - b.sequence), + ); + const options = { + completedTurnDisplay: "collapse", + includeDiagnosticOperations: false, + includeNestedRows: true, + maxInlineOutputChars: null, + maxSeq: 0, + } as const; + const canonical = buildThreadTimelineWithProfile(db, thread, { + ...options, + eventBudget: LARGE_BUDGET, + page: { kind: "latest", segmentLimit: 100 }, + }).response.rows; + let removed = 0; + for (let i = 0; i < 100; i++) { + const result = advanceThreadPruning(db, "completed-items"); + removed += result.removed; + if (result.action === "cycle-complete") break; + } + expect(removed).toBe(owners.length * 2); + let rows: TimelineRow[] = []; + let cursor: TimelinePaginationCursor | null = null; + const cursors = new Set(); + do { + const response: ThreadTimelineResponse = buildThreadTimelineWithProfile( + db, + thread, + { + ...options, + eventBudget, + responseByteBudget: 512, + page: cursor + ? { kind: "older", beforeCursor: cursor, segmentLimit: 2 } + : { kind: "latest", segmentLimit: 2 }, + }, + ).response; + rows = prependOlderTimelineRows({ + loadedRows: rows, + olderRows: response.rows, + }); + cursor = response.timelinePage.olderCursor; + if (cursor) { + const key = JSON.stringify(cursor); + expect(cursors.has(key)).toBe(false); + cursors.add(key); + expect(cursors.size).toBeLessThan(100); + } + } while (cursor); + expect(rows).toEqual(canonical); + } finally { + db.$client.close(); + } + }, +); diff --git a/apps/server/test/system/event-pruning.test.ts b/apps/server/test/system/event-pruning.test.ts index 0d19a7fed36..69a581e237e 100644 --- a/apps/server/test/system/event-pruning.test.ts +++ b/apps/server/test/system/event-pruning.test.ts @@ -171,6 +171,83 @@ function seedResolvedAssistantMessage( } describe("thread event pruning", () => { + it("refreshes a cached visible timeline after a live completed-item rewrite and exposes combined raw records", async () => { + await withTestHarness(async (harness) => { + const host = seedHost(harness.deps); + const { project } = seedProjectWithSource(harness.deps, { + hostId: host.id, + }); + const environment = seedEnvironment(harness.deps, { + hostId: host.id, + projectId: project.id, + }); + const thread = seedThread(harness.deps, { + projectId: project.id, + environmentId: environment.id, + }); + seedStoredEvent(harness.deps, { + threadId: thread.id, + sequence: 1, + scope: turnScope("turn-1"), + type: "turn/started", + itemId: null, + itemKind: null, + data: {}, + }); + seedResolvedAssistantMessage(harness, { + threadId: thread.id, + itemId: "message", + deltaSequences: [2], + completedSequence: 3, + }); + seedStoredEvent(harness.deps, { + threadId: thread.id, + sequence: 4, + scope: turnScope("turn-1"), + type: "turn/completed", + itemId: null, + itemKind: null, + data: { status: "completed" }, + }); + harness.db.$client + .prepare("UPDATE events SET provider_thread_id = ? WHERE thread_id = ?") + .run("provider", thread.id); + const build = () => + buildThreadTimelineWithProfile(harness.db, thread, { + completedTurnDisplay: "collapse", + includeDiagnosticOperations: false, + includeNestedRows: true, + maxInlineOutputChars: 8000, + eventBudget: 10000, + maxSeq: 4, + page: { kind: "latest", segmentLimit: 20 }, + }).response; + const before = build(); + expect(JSON.stringify(before)).toContain("Final answer"); + expect(build()).toEqual(before); + const notify = vi.spyOn(harness.deps.hub, "notifyThread"); + for (let i = 0; i < 5; i++) + pruneThreadEventHistoryBestEffort(harness.deps, { + threadId: thread.id, + mode: "idle", + }); + expect(notify).toHaveBeenCalledWith(thread.id, ["history-rewritten"]); + expect(build()).toEqual(before); + const raw = await harness.app.request( + `/api/v1/threads/${thread.id}/events?limit=1&afterSeq=1`, + ); + expect(raw.status).toBe(200); + const text = await raw.text(); + expect(text).toContain("item/completed"); + expect(text).not.toContain("completed_item_history"); + expect( + listEvents(harness.db, { threadId: thread.id }).map( + (row) => row.sequence, + ), + ).toEqual([1, 2, 4]); + }); + }); + it("prunes idle-thread noise rows and resolved item deltas", async () => { await withTestHarness(async (harness) => { const host = seedHost(harness.deps); diff --git a/docs/completed-item-compaction-verification.md b/docs/completed-item-compaction-verification.md new file mode 100644 index 00000000000..6bd943fa32e --- /dev/null +++ b/docs/completed-item-compaction-verification.md @@ -0,0 +1,98 @@ +# Completed-item compaction verification + +Measured on 2026-09-16 in an isolated worktree based on `5aca5733a5`. +PR1 (`c663ff1911`, #3766) is included. These are observations from private +sanitized SQLite copies, not latency or savings guarantees. + +## Correctness and storage + +| Check | Result | +| --- | --- | +| Physical events | 2,082,639 → 1,080,480 (1,002,159 removed) | +| All table rows | 2,571,187 → 1,569,030 (net 1,002,157 removed) | +| Combined owners | 793,808 | +| Internal metadata | 182,951,145 bytes | +| Allocated SQLite btree bytes | 5,634,584,576 → 4,411,244,544 (1,223,340,032 saved) | +| Database file bytes | 6,006,726,656 before and after; freed pages are reusable | +| Independent logical reconstruction | All 2,082,639 records match across 2,326 threads, except 119,716 eligible empty command-delta text markers | +| Complete visible timelines | All 2,326 match, including all 2,223 non-null context values | +| Highwater and provider recovery | All 2,326 match | +| Existing side tables | All 46 non-event/non-cursor/non-migration tables match, including output, search and attachment ownership | +| Default-budget server pages | All 30 match exactly | +| Complete large-thread pagination | Three largest threads: 3,816 rendered rows match after complete traversal at a 1,000-row budget; baseline used 10,000 | +| Fork inheritance | 24 ordinary plus 10 interleaved real cases; 2,638 copied records match | +| Message edit suffixes | 24 real boundary cases; no future output survives | +| Other consumers | 2,326 latest-output checks, ten large outlines and 100 rollback-only output mutations match | + +The two-row difference between physical removals and net removals is existing +migration/cursor bookkeeping. There are no added tables or indexes and no full +VACUUM. The worker performs actual forward discovery and mutation; this is not a +conversion of the prototype's auxiliary-table results. + +The first worker pass removes 13 fewer rows than the prototype. A direct cohort +comparison finds 17 additional command owners left ordinary by the bounded +boundary probe (29 rows), offset by 16 additional eligible start rows. All shared +owners have the same reconstructed record counts. User-message/context-clear +crossings remain excluded, as do ambiguous/unsettled/incompatible lifecycles, +unsupported or repeated delta streams, malformed payloads and work exceeding +support/input budgets. No arbitrary mid-item rewind repair was added. + +## Timing and catch-up + +Measurements ran sequentially, without overlapping heavy benchmarks. The host +was shared; these are warm local observations, not cold-I/O or network bounds. + +| Measurement | Result | +| --- | --- | +| Full completed-item worker pass | 120,866 advances; 306.51 seconds | +| Background advance median / p99 / maximum | 1.90 / 15.81 / 212.28 ms | +| Full live cleanup wrapper, 2,500 rotating advances | median 0.257 ms; p99 6.55 ms; maximum 270.07 ms | +| Completed-item subset of live wrapper, 500 advances | median 0.932 ms; p99 8.43 ms; maximum 270.07 ms | +| Median of 30 page case medians, before / after | 94.50 / 111.75 ms | +| Largest individual page call, before / after | 374.21 / 473.56 ms | +| Actual background sweep, 100 calls | 1,989 total advances; 398 completed-item advances | +| Maximum sweep / advance in that sample | 193.34 / 158.88 ms; the largest advance was PR1 resolved-item pruning | + +The full live wrapper includes policy rotation, transaction, generation checks, +logging and notification calls; the offline harness collects logs/notifications +in process. No server or provider runs on a research copy. The background sweep +sample explicitly makes its private performance copy idle; the original copied +activity state otherwise blocks maintenance. + +At 3.98 completed-item advances per sweep and the existing ten-second sweep +cadence, the first-pass catch-up estimate is about **84 idle hours**. Activity, +other policies, I/O and larger excluded lifecycles can extend elapsed time. +The implementation retains PR1's scheduler and allocator. Bounded rows/bytes do +not impose a hard stall cap. Reader median overhead and initial catch-up time +remain limitations for review. + +## Automated and UI verification + +- Turbo DB tests: 44 files, 589 tests pass, using migrated real SQLite. +- Turbo server tests: 16 relevant files, 200 tests pass, covering timeline caches, + pagination, truncation, context clearing, message editing, forks and live pruning. +- New complete traversal tests pass at row budgets 1, 2, 5 and 20 with a 512-byte + response budget, preserving canonical content through client page merging. +- Turbo DB/server typechecks pass. +- Plugin Guide tests/typecheck pass: ten files, 73 tests. +- Fresh isolated Chromium UI: expand command output, file diff and reasoning; + preserve text and durations through an actual background rewrite; reload and + verify persisted rendering. A second rewrite holds highwater at 15 while rows + fall from 15 to eight. The open page automatically requests + `timeline?afterSequence=15`, without navigation and with identical visible text. +- The synthetic UI's first fixture initially used the default starting status + and produced a provisioning error. The fixture was repaired to idle before the + measured rewrite. No provider turn was sent. Browser and dev processes were + stopped after verification. + +The verification inventory reports pre-existing unmapped `browser` CLI-family +drift on the fetched base. This does not represent a passed inventory check. +No iOS or provider-resume claim is made; this change does not alter drawer UI or +provider execution. Native edit/fork behavior was exercised through the actual +server functions on private copies and existing route tests. + +Reproduction scripts, source briefs, full JSON comparisons, timings, logs and UI +screenshots are preserved in the implementing thread's `pr2-start-position` +storage directory. Research databases remain private and mode 0600; Connect +plugin records were verified absent. No live database was opened, no copied live +configuration was launched, and no deployment or merge was performed. diff --git a/docs/completed-item-compaction.md b/docs/completed-item-compaction.md new file mode 100644 index 00000000000..889ac602bf0 --- /dev/null +++ b/docs/completed-item-compaction.md @@ -0,0 +1,62 @@ +# Completed item history compaction + +The existing event-maintenance rotation combines eligible settled item history +into its `item/completed` row. The completion keeps its ID, payload, original +creation timestamp and retained-output ownership. Its physical sequence becomes +the first retained lifecycle sequence. One versioned internal column stores the +original completion sequence and lossless start/delta information. No table, +index, allocator, scheduler or daemon protocol change is involved. + +Eligibility requires a unique compatible completion, at most one start, a later +retained completion of the turn, and no late lifecycle events or reopened turn. +Supported kinds are command executions, file changes, assistant messages and +reasoning. Unpruned repeated delta streams, file-change output deltas, malformed +payloads, incompatible envelopes and oversized discovery/payload windows remain +ordinary. A lifetime crossing a user request or completed context clear remains +ordinary so message editing can keep its existing suffix-deletion behavior. + +Assistant and reasoning delta text is retained. A command delta can become an +empty timing marker only when the command has completed, failed or been +interrupted and has nonempty aggregated output. Without a retained start, the +aggregated output must contain that delta. Otherwise the entire lifecycle stays +ordinary. Starts never derive mutable output fields from a completion, so later +output truncation or expiry cannot change reconstructed starts. + +Discovery uses the existing physical sequence/type/turn/item indexes. Each +advance bounds candidate/support rows and input bytes; ambiguous or larger +lifecycles are skipped, not partially rewritten. Source deletion, completion +movement and cursor progress commit together. PR1's generation increment and +`history-rewritten` notification invalidate cached views after the commit. +Limits bound work rather than guaranteeing a maximum elapsed time. + +Timeline queries select physical rows first and only then decode their internal +metadata for projection. Selected fork history recovers completion order before +applying the existing completed-turn and event-type rules. Forks still create +new IDs and sequence numbers. No arbitrary deleted-ID lookup or virtual raw-event +pagination is provided. + +## SDK, CLI and exports + +`threads.events.list`, the raw events HTTP route and `bb thread log --json` +(including `--all` JSON exports) return physical records. A combined row is still +`item/completed`; it has its original completion ID/payload/timestamp and its new +first lifecycle sequence. Internal metadata is not returned. Deleted start/delta +IDs, original raw counts and old completion positions are not retained as a raw +API contract. Consumers must not treat these exports as an exact provider wire +transcript. + +Physical-row limits and sequence cursors count combined rows. Restart a traversal +after a history rewrite rather than continuing an old cursor against changed +history. Human CLI formats and the UI reconstruct original ordering, timing, +text and edit cards. A fixed physical-row budget can include more history after +compaction; exact page boundaries are not promised. Response byte limits remain +independent of row limits, and stored timeline byte accounting includes metadata. + +## Verification + +The migrated SQLite regression tests cover eligibility, lossless reconstruction, +command discard rules, output ownership, atomic rollback, late arrivals, +highwater, boundary exclusions and one-row raw traversal. Server tests exercise +the existing live wrapper, rewrite notifications and warmed timeline caches. +Full-copy measurements and UI evidence accompany the draft PR; prototype +measurements are not implementation guarantees. diff --git a/packages/db/src/completed-item-history.ts b/packages/db/src/completed-item-history.ts new file mode 100644 index 00000000000..482dbd54845 --- /dev/null +++ b/packages/db/src/completed-item-history.ts @@ -0,0 +1,223 @@ +import { isDeepStrictEqual } from "node:util"; +import { + jsonValueSchema, + threadEventTypeSchema, + type JsonObject, + type JsonValue, + type ThreadEventType, +} from "@bb/domain"; + +export const COMPACTED_ITEM_KINDS = [ + "commandExecution", + "fileChange", + "reasoning", + "agentMessage", +] as const; +export const COMPACTED_HISTORY_TYPES: readonly ThreadEventType[] = [ + "item/started", + "item/agentMessage/delta", + "item/commandExecution/outputDelta", + "item/reasoning/textDelta", + "item/reasoning/summaryTextDelta", +]; +type ItemKind = (typeof COMPACTED_ITEM_KINDS)[number]; +export function isCompactedItemKind(value: string | null): value is ItemKind { + return COMPACTED_ITEM_KINDS.some((kind) => kind === value); +} +interface HistoryRecord { + id: string; + sequence: number; + createdAt: number; + type: ThreadEventType; + itemKind: ItemKind | null; + payload: JsonObject; + sharedFields: string[]; + sharedItemFields: string[]; +} +const mutableOutputFields = new Set([ + "aggregatedOutput", + "result", + "resultText", + "truncation", +]); + +function isObject(value: JsonValue | undefined): value is JsonObject { + return value !== null && typeof value === "object" && !Array.isArray(value); +} + +export function parseHistoryPayload(data: string): JsonObject { + const parsed = jsonValueSchema.parse(JSON.parse(data)); + if (!isObject(parsed)) + throw new Error("Completed item history payload must be an object"); + return parsed; +} + +function fieldNames(value: JsonValue | undefined): string[] { + if ( + !Array.isArray(value) || + !value.every((key): key is string => typeof key === "string") + ) + throw new Error("Invalid completed item history field names"); + return value; +} + +export function decodeHistory(data: string): HistoryRecord[] { + const value = jsonValueSchema.parse(JSON.parse(data)); + if ( + !Array.isArray(value) || + value.length !== 2 || + value[0] !== 1 || + !Array.isArray(value[1]) || + value[1].length < 1 || + value[1].length > 3 + ) + throw new Error("Invalid completed item history version or records"); + const records = value[1].map((record): HistoryRecord => { + if (!Array.isArray(record) || record.length !== 6) + throw new Error("Invalid completed item history record"); + const [id, sequence, createdAt, type, itemKind, encoding] = record; + if ( + typeof id !== "string" || + id.length === 0 || + typeof sequence !== "number" || + !Number.isSafeInteger(sequence) || + sequence < 1 || + typeof createdAt !== "number" || + !Number.isFinite(createdAt) + ) + throw new Error("Invalid completed item history identity"); + if ( + itemKind !== null && + itemKind !== "commandExecution" && + itemKind !== "fileChange" && + itemKind !== "reasoning" && + itemKind !== "agentMessage" + ) + throw new Error("Invalid completed item history kind"); + if ( + !Array.isArray(encoding) || + encoding.length !== 3 || + !isObject(encoding[0]) + ) + throw new Error("Invalid completed item history payload"); + const sharedFields = fieldNames(encoding[1]); + const sharedItemFields = fieldNames(encoding[2]); + if ( + sharedFields.includes("item") || + sharedItemFields.some((key) => mutableOutputFields.has(key)) + ) + throw new Error( + "Mutable completed output cannot own reconstructed history", + ); + const eventType = threadEventTypeSchema.parse(type); + if (!COMPACTED_HISTORY_TYPES.includes(eventType)) + throw new Error("Unsupported completed item history event type"); + return { + id, + sequence, + createdAt, + type: eventType, + itemKind, + payload: encoding[0], + sharedFields, + sharedItemFields, + }; + }); + if ( + new Set(records.map((record) => record.id)).size !== records.length || + new Set(records.map((record) => record.sequence)).size !== records.length + ) + throw new Error("Duplicate completed item history identity"); + return records; +} + +export function encodeHistory(records: readonly HistoryRecord[]): string { + return JSON.stringify([ + 1, + records.map((record) => [ + record.id, + record.sequence, + record.createdAt, + record.type, + record.itemKind, + [record.payload, record.sharedFields, record.sharedItemFields], + ]), + ]); +} + +export function compactHistoryPayload(data: JsonObject, owner: JsonObject) { + const payload = { ...data }; + const sharedFields: string[] = []; + const sharedItemFields: string[] = []; + for (const key of Object.keys(payload)) { + if ( + key !== "item" && + Object.hasOwn(owner, key) && + isDeepStrictEqual(payload[key], owner[key]) + ) { + sharedFields.push(key); + delete payload[key]; + } + } + if (isObject(payload.item) && isObject(owner.item)) { + const item = { ...payload.item }; + for (const key of Object.keys(item)) { + if ( + !mutableOutputFields.has(key) && + Object.hasOwn(owner.item, key) && + isDeepStrictEqual(item[key], owner.item[key]) + ) { + sharedItemFields.push(key); + delete item[key]; + } + } + payload.item = item; + } + return { payload, sharedFields, sharedItemFields }; +} + +export function restoreHistoryPayload( + record: HistoryRecord, + owner: JsonObject, +): string { + const payload = { ...record.payload }; + for (const key of record.sharedFields) { + const value = owner[key]; + if (!Object.hasOwn(owner, key) || value === undefined) + throw new Error("Missing completed item history field"); + Object.defineProperty(payload, key, { value, enumerable: true }); + } + if (record.sharedItemFields.length > 0) { + if (!isObject(payload.item) || !isObject(owner.item)) + throw new Error("Missing completed item history item"); + const item = { ...payload.item }; + for (const key of record.sharedItemFields) { + const value = owner.item[key]; + if (!Object.hasOwn(owner.item, key) || value === undefined) + throw new Error("Missing completed item history item field"); + Object.defineProperty(item, key, { value, enumerable: true }); + } + payload.item = item; + } + return JSON.stringify(payload); +} + +export function decodeCompletedItemHistory(data: string) { + const value = jsonValueSchema.parse(JSON.parse(data)); + if ( + !Array.isArray(value) || + value.length !== 4 || + value[0] !== 1 || + typeof value[1] !== "number" || + !Number.isSafeInteger(value[1]) || + value[1] < 1 || + typeof value[2] !== "number" || + !Number.isFinite(value[2]) + ) + throw new Error("Invalid combined completed item history"); + const records = decodeHistory(JSON.stringify(value[3])); + const sequence = value[1]; + if (records.some((row) => row.sequence >= sequence)) + throw new Error("Completed item history exceeds completion sequence"); + return { sequence: value[1], createdAt: value[2], records }; +} diff --git a/packages/db/src/data/completed-item-compaction.ts b/packages/db/src/data/completed-item-compaction.ts new file mode 100644 index 00000000000..c187957ae06 --- /dev/null +++ b/packages/db/src/data/completed-item-compaction.ts @@ -0,0 +1,395 @@ +import { eq, inArray, sql } from "drizzle-orm"; +import { parseStoredThreadEvent, type ThreadEventType } from "@bb/domain"; +import type { DbQueryConnection } from "../connection.js"; +import { events } from "../schema.js"; +import { + compactHistoryPayload, + encodeHistory, + isCompactedItemKind, + parseHistoryPayload, +} from "../completed-item-history.js"; + +const INPUT_BYTE_BUDGET = 1024 * 1024; +const supportTypesByKind = { + commandExecution: [ + "item/started", + "item/completed", + "item/commandExecution/outputDelta", + ], + fileChange: ["item/started", "item/completed", "item/fileChange/outputDelta"], + agentMessage: ["item/started", "item/completed", "item/agentMessage/delta"], + reasoning: [ + "item/started", + "item/completed", + "item/reasoning/textDelta", + "item/reasoning/summaryTextDelta", + ], +} satisfies Record; +interface Candidate { + id: string; + sequence: number; + type: ThreadEventType; + itemId: string | null; + itemKind: typeof events.$inferSelect.itemKind; + parentToolCallId: string | null; + turnId: string | null; + providerThreadId: string | null; + environmentId: string | null; + dataBytes: number; + hasHistory: number; +} +const candidateFields = sql`id, sequence, type, item_id AS itemId, item_kind AS itemKind, + parent_tool_call_id AS parentToolCallId, turn_id AS turnId, provider_thread_id AS providerThreadId, + environment_id AS environmentId, octet_length(data) AS dataBytes, completed_item_history IS NOT NULL AS hasHistory`; + +export function advanceCompletedItemCompaction( + db: DbQueryConnection, + args: { + threadId: string; + afterSequence: number; + throughSequence: number; + limit: number; + }, +) { + const pageSize = Math.min(8, Math.max(1, Math.floor(args.limit / 64))); + const candidates = db.all(sql`SELECT ${candidateFields} + FROM events INDEXED BY events_thread_type_sequence_idx + WHERE thread_id = ${args.threadId} AND type = 'item/completed' + AND sequence > ${args.afterSequence} AND sequence <= ${args.throughSequence} + ORDER BY sequence LIMIT ${pageSize}`); + let scanned = candidates.length; + let removed = 0; + let removedBytes = 0; + let processedBytes = 0; + let nextSequence = args.afterSequence; + const skipped: Record = {}; + const skip = (kind: string, reason: string) => { + const key = `${kind}:${reason}`; + skipped[key] = (skipped[key] ?? 0) + 1; + }; + for (const candidate of candidates) { + if (processedBytes >= INPUT_BYTE_BUDGET) break; + nextSequence = candidate.sequence; + const kind = candidate.itemKind; + if (!isCompactedItemKind(kind)) { + skip(kind ?? "unknown", "unsupported-kind"); + continue; + } + if (candidate.turnId === null || candidate.itemId === null) { + skip(kind, "missing-scope"); + continue; + } + if (candidate.hasHistory) continue; + const support: Candidate[] = []; + let exhausted = true; + for (const type of [ + ...supportTypesByKind[kind], + "turn/started", + "turn/completed", + ] as const) { + const limit = Math.min(8, args.limit - scanned - 2); + if (limit <= 0) { + exhausted = false; + break; + } + const rows = db.all(sql`SELECT ${candidateFields} + FROM events INDEXED BY events_thread_turn_type_item_sequence_idx + WHERE thread_id = ${args.threadId} AND turn_id = ${candidate.turnId} + AND type = ${type} AND item_id IS ${type.startsWith("turn/") ? null : candidate.itemId} + ORDER BY sequence LIMIT ${limit}`); + scanned += Math.max(1, rows.length); + support.push(...rows); + if (rows.length === limit) { + exhausted = false; + break; + } + } + if (!exhausted) { + skip(kind, "support-budget"); + continue; + } + if ( + support.some( + (row) => + !row.type.startsWith("turn/") && + (row.parentToolCallId !== candidate.parentToolCallId || + ((row.type === "item/started" || row.type === "item/completed") && + row.itemKind !== kind)), + ) + ) { + skip(kind, "incompatible-lifecycle"); + continue; + } + const peers = support.filter( + (row) => + !row.type.startsWith("turn/") && + row.parentToolCallId === candidate.parentToolCallId && + (row.itemKind === kind || + (row.type !== "item/started" && row.type !== "item/completed")), + ); + const starts = peers.filter((row) => row.type === "item/started"); + const completions = peers.filter((row) => row.type === "item/completed"); + const deltas = peers.filter( + (row) => row.type !== "item/started" && row.type !== "item/completed", + ); + if (completions.length !== 1 || completions[0]?.id !== candidate.id) { + skip(kind, "ambiguous-completion"); + continue; + } + if (starts.length > 1) { + skip(kind, "ambiguous-start"); + continue; + } + if (peers.some((row) => row.sequence > candidate.sequence)) { + skip(kind, "late-item-event"); + continue; + } + if ( + peers.some( + (row) => + row.environmentId !== candidate.environmentId || + row.providerThreadId !== candidate.providerThreadId, + ) + ) { + skip(kind, "incompatible-envelope"); + continue; + } + const turnRows = support.filter( + (row) => + row.type.startsWith("turn/") && + (row.parentToolCallId === null || + row.parentToolCallId === candidate.parentToolCallId) && + row.providerThreadId === candidate.providerThreadId, + ); + const settled = turnRows.find( + (row) => + row.type === "turn/completed" && row.sequence > candidate.sequence, + ); + if ( + settled === undefined || + turnRows.some( + (row) => + row.type === "turn/started" && row.sequence > candidate.sequence, + ) + ) { + skip(kind, "unsettled"); + continue; + } + if ( + deltas.some((row) => row.type === "item/fileChange/outputDelta") || + new Set(deltas.map((row) => row.type)).size !== deltas.length || + deltas.length > 2 + ) { + skip(kind, "unpruned-or-unsupported-deltas"); + continue; + } + const source = [...starts, ...deltas].sort( + (a, b) => a.sequence - b.sequence, + ); + if (source.length === 0) continue; + const firstSequence = source[0]!.sequence; + const boundaryRows: { id: string; dataBytes: number }[] = []; + let boundaryBudgetExhausted = false; + for (const type of ["client/turn/requested", "system/operation"]) { + const limit = Math.min(8, args.limit - scanned); + if (limit <= 0) { + boundaryBudgetExhausted = true; + break; + } + const found = db.all<{ id: string; dataBytes: number }>(sql` + SELECT id, octet_length(data) AS dataBytes + FROM events INDEXED BY events_thread_type_sequence_idx + WHERE thread_id = ${args.threadId} AND type = ${type} + AND sequence > ${firstSequence} AND sequence <= ${candidate.sequence} + ORDER BY sequence LIMIT ${limit} + `); + scanned += Math.max(1, found.length); + boundaryRows.push(...found); + if (found.length === limit) { + boundaryBudgetExhausted = true; + break; + } + } + if (boundaryBudgetExhausted) { + skip(kind, "boundary-budget"); + continue; + } + const boundaryBytes = boundaryRows.reduce( + (total, row) => total + row.dataBytes, + 0, + ); + if (boundaryBytes > INPUT_BYTE_BUDGET - processedBytes) { + skip(kind, "boundary-payload-budget"); + continue; + } + processedBytes += boundaryBytes; + let crossesBoundary = false; + for (const boundary of boundaryRows) { + const row = db + .select({ data: events.data }) + .from(events) + .where(eq(events.id, boundary.id)) + .get(); + try { + if (!row) throw new Error("Missing boundary"); + const payload = parseHistoryPayload(row.data); + if ( + payload.initiator === "user" || + (payload.operation === "context_clear" && + payload.status === "completed") + ) + crossesBoundary = true; + } catch { + crossesBoundary = true; + } + } + if (crossesBoundary) { + skip(kind, "edit-or-context-boundary"); + continue; + } + const bytes = peers.reduce( + (total, row) => total + row.dataBytes, + settled.dataBytes, + ); + if (bytes > INPUT_BYTE_BUDGET - processedBytes) { + skip(kind, "payload-budget"); + continue; + } + const rows = db + .select() + .from(events) + .where(inArray(events.id, [...peers.map((row) => row.id), settled.id])) + .all(); + processedBytes += bytes; + const owner = rows.find((row) => row.id === candidate.id); + if (owner === undefined) + throw new Error("Missing completed item compaction owner"); + let payloads: Map>; + try { + payloads = new Map( + rows.map((row) => { + const payload = parseHistoryPayload(row.data); + const event = parseStoredThreadEvent({ + threadId: args.threadId, + providerThreadId: row.providerThreadId, + type: row.type, + scope: { kind: "turn", turnId: candidate.turnId! }, + data: payload, + }); + if ( + (event.type === "item/started" || + event.type === "item/completed") && + (event.item.id !== candidate.itemId || event.item.type !== kind) + ) + throw new Error("Mismatched item identity"); + return [row.id, payload]; + }), + ); + } catch { + skip(kind, "malformed-payload"); + continue; + } + const ownerPayload = payloads.get(candidate.id); + if (ownerPayload === undefined) + throw new Error("Missing completed item payload"); + const item = ownerPayload.item; + if (item === null || typeof item !== "object" || Array.isArray(item)) { + skip(kind, "malformed-completion"); + continue; + } + if ( + (kind === "commandExecution" || kind === "fileChange") && + item.status !== "completed" && + item.status !== "failed" && + item.status !== "interrupted" + ) { + skip(kind, "completion-status"); + continue; + } + const compacted = source.filter((row) => { + if (row.type !== "item/commandExecution/outputDelta") return true; + const payload = payloads.get(row.id); + if (payload === undefined || typeof payload.delta !== "string") { + skip(kind, "malformed-delta"); + return false; + } + if ( + item.status !== "completed" && + item.status !== "failed" && + item.status !== "interrupted" + ) { + skip(kind, "command-status"); + return false; + } + if ( + typeof item.aggregatedOutput !== "string" || + item.aggregatedOutput.length === 0 + ) { + skip(kind, "empty-output"); + return false; + } + if ( + starts.length === 0 && + !item.aggregatedOutput.includes(payload.delta) + ) { + skip(kind, "missing-start-uncontained-text"); + return false; + } + payload.delta = ""; + return true; + }); + if (compacted.length !== source.length) continue; + const records = compacted.map((row) => { + const payload = payloads.get(row.id); + const stored = rows.find((entry) => entry.id === row.id); + if (payload === undefined || stored === undefined) + throw new Error("Missing completed item source"); + if (stored.itemKind !== null && !isCompactedItemKind(stored.itemKind)) + throw new Error("Unsupported compacted item kind"); + return { + id: row.id, + sequence: row.sequence, + createdAt: stored.createdAt, + type: row.type, + itemKind: stored.itemKind, + ...compactHistoryPayload(payload, ownerPayload), + }; + }); + const data = JSON.stringify([ + 1, + candidate.sequence, + owner.createdAt, + JSON.parse(encodeHistory(records)), + ]); + const result = db + .delete(events) + .where( + inArray( + events.id, + records.map((row) => row.id), + ), + ) + .run(); + if (result.changes !== records.length) + throw new Error("Completed item compaction lost a source row"); + db.update(events) + .set({ sequence: firstSequence, completedItemHistory: data }) + .where(eq(events.id, candidate.id)) + .run(); + removed += result.changes; + removedBytes += + compacted.reduce((total, row) => total + row.dataBytes, 0) - + Buffer.byteLength(data); + } + return { + scanned, + removed, + removedBytes, + processedBytes, + skipped, + nextSequence, + complete: + candidates.length < pageSize && + nextSequence === (candidates.at(-1)?.sequence ?? args.afterSequence), + }; +} diff --git a/packages/db/src/data/completed-item-history.ts b/packages/db/src/data/completed-item-history.ts new file mode 100644 index 00000000000..1612b7687c9 --- /dev/null +++ b/packages/db/src/data/completed-item-history.ts @@ -0,0 +1,64 @@ +import { inArray } from "drizzle-orm"; +import type { DbQueryConnection } from "../connection.js"; +import { events } from "../schema.js"; +import { + decodeCompletedItemHistory, + parseHistoryPayload, + restoreHistoryPayload, +} from "../completed-item-history.js"; +import type { StoredEventRow } from "./events.js"; + +export function expandSelectedCompletedItemRows( + db: DbQueryConnection, + rows: readonly StoredEventRow[], + throughSequence = Number.MAX_SAFE_INTEGER, +): StoredEventRow[] { + const ids = [ + ...new Set( + rows.filter((row) => row.type === "item/completed").map((row) => row.id), + ), + ]; + const histories = new Map< + string, + ReturnType + >(); + for (let offset = 0; offset < ids.length; offset += 250) { + const selected = db + .select({ id: events.id, history: events.completedItemHistory }) + .from(events) + .where(inArray(events.id, ids.slice(offset, offset + 250))) + .all(); + for (const row of selected) { + if (row.history !== null) + histories.set(row.id, decodeCompletedItemHistory(row.history)); + } + } + const expanded: StoredEventRow[] = []; + for (const row of rows) { + const history = histories.get(row.id); + if (!history) { + if (row.sequence <= throughSequence) expanded.push(row); + continue; + } + if (history.sequence <= throughSequence) + expanded.push({ + ...row, + sequence: history.sequence, + createdAt: history.createdAt, + }); + const payload = parseHistoryPayload(row.data); + for (const record of history.records) { + if (record.sequence <= throughSequence) + expanded.push({ + ...row, + id: record.id, + sequence: record.sequence, + createdAt: record.createdAt, + type: record.type, + itemKind: record.itemKind, + data: restoreHistoryPayload(record, payload), + }); + } + } + return expanded.sort((a, b) => a.sequence - b.sequence); +} diff --git a/packages/db/src/data/events.ts b/packages/db/src/data/events.ts index c53cbed1c51..74fa3049465 100644 --- a/packages/db/src/data/events.ts +++ b/packages/db/src/data/events.ts @@ -3103,7 +3103,7 @@ export function findStoredTimelineWindowByteBudgetFloor( const query = db .select({ createdAt: events.createdAt, - dataBytes: sql`length(CAST(${data} AS BLOB))`.as("data_bytes"), + dataBytes: sql`length(CAST(${data} AS BLOB)) + COALESCE(length(CAST(${events.completedItemHistory} AS BLOB)), 0)`.as("data_bytes"), sequence: events.sequence, turnId: events.turnId, }) diff --git a/packages/db/src/data/thread-pruning.ts b/packages/db/src/data/thread-pruning.ts index bf5d21884eb..e2cb11f2208 100644 --- a/packages/db/src/data/thread-pruning.ts +++ b/packages/db/src/data/thread-pruning.ts @@ -1,3 +1,4 @@ +import { advanceCompletedItemCompaction } from "./completed-item-compaction.js"; import { isBeforeLatestThreadEvent } from "./event-pruning-guards.js"; import { advanceLiveEventPruning, @@ -19,6 +20,7 @@ export const THREAD_PRUNING_POLICIES = [ "usage", "turn-diffs", "resolved-items", + "completed-items", ] as const; export type ThreadPruningPolicy = (typeof THREAD_PRUNING_POLICIES)[number]; const VERSION_BY_POLICY: Record = { @@ -26,6 +28,7 @@ const VERSION_BY_POLICY: Record = { usage: 1, "turn-diffs": 1, "resolved-items": 1, + "completed-items": 1, }; const BATCH_SIZE = 500; const LIVE_BATCH_SIZE = 32; @@ -173,6 +176,19 @@ function advanceThreadPruningTransaction( cursor.sequence = batch.nextSequence; if (batch.complete || cursor.sequence >= cursor.upperSequence) action = "thread-complete"; + } else if (policy === "completed-items") { + const batch = advanceCompletedItemCompaction(tx, { + threadId, + afterSequence: cursor.sequence, + throughSequence: cursor.upperSequence, + limit: batchSize, + }); + scanned = batch.scanned; + removed = batch.removed; + removedBytes = batch.removedBytes; + cursor.sequence = batch.nextSequence; + if (batch.complete || cursor.sequence >= cursor.upperSequence) + action = "thread-complete"; } else if (policy === "resolved-items") { const batch = advanceLiveEventPruning(tx, { threadId, diff --git a/packages/db/src/index.ts b/packages/db/src/index.ts index 84f1f95ee68..dd561ab61d7 100644 --- a/packages/db/src/index.ts +++ b/packages/db/src/index.ts @@ -26,3 +26,5 @@ export { noopNotifier } from "./notifier.js"; export type { DbNotifier } from "./notifier.js"; export * from "./data/index.js"; + +export { expandSelectedCompletedItemRows } from "./data/completed-item-history.js"; diff --git a/packages/db/src/schema.ts b/packages/db/src/schema.ts index 68c9b152330..a98a4adf88b 100644 --- a/packages/db/src/schema.ts +++ b/packages/db/src/schema.ts @@ -775,6 +775,7 @@ export const events = sqliteTable( itemKind: text("item_kind").$type(), parentToolCallId: text("parent_tool_call_id"), data: text("data").notNull().default("{}"), + completedItemHistory: text("completed_item_history"), createdAt: integer("created_at").notNull(), }, (table) => [ diff --git a/packages/db/test/data/completed-item-compaction.test.ts b/packages/db/test/data/completed-item-compaction.test.ts new file mode 100644 index 00000000000..df48e227e4b --- /dev/null +++ b/packages/db/test/data/completed-item-compaction.test.ts @@ -0,0 +1,366 @@ +import { describe, expect, it } from "vitest"; +import { eq } from "drizzle-orm"; +import { + events, + retainedEventOutputs, + threadPruningCursors, +} from "../../src/schema.js"; +import { noopNotifier } from "../../src/notifier.js"; +import { upsertHost } from "../../src/data/hosts.js"; +import { createProject } from "../../src/data/projects.js"; +import { createThread } from "../../src/data/threads.js"; +import { advanceThreadPruning } from "../../src/data/thread-pruning.js"; +import { advanceCompletedItemCompaction } from "../../src/data/completed-item-compaction.js"; +import { expandSelectedCompletedItemRows } from "../../src/data/completed-item-history.js"; +import { + listStoredEventRows, + getHighWaterMarks, + deleteThreadEventSuffixInTransaction, +} from "../../src/data/events.js"; +import { getThreadEventRewriteGeneration } from "../../src/data/event-rewrite-generation.js"; +import { createMigratedConnection } from "../helpers/migrated-connection.js"; + +function setup() { + const db = createMigratedConnection(); + const host = upsertHost(db, noopNotifier, { name: "compaction" }); + const { project } = createProject(db, noopNotifier, { + name: "compaction", + source: { type: "local_path", hostId: host.id, path: "/tmp/compaction" }, + }); + const thread = createThread(db, noopNotifier, { + projectId: project.id, + providerId: "codex", + }); + const seed = ( + sequence: number, + values: Partial, + ) => + db + .insert(events) + .values({ + id: `event-${sequence}`, + threadId: thread.id, + sequence, + scopeKind: "turn", + turnId: "turn", + providerThreadId: "provider", + type: "item/started", + data: "{}", + createdAt: sequence * 100, + ...values, + }) + .run(); + const item = { + id: "message", + type: "agentMessage", + text: "lossless assistant text", + }; + seed(1, { type: "turn/started", data: "{}" }); + seed(2, { + itemId: item.id, + itemKind: "agentMessage", + data: JSON.stringify({ item: { ...item, text: "" } }), + }); + seed(3, { + type: "item/agentMessage/delta", + itemId: item.id, + data: JSON.stringify({ itemId: item.id, delta: "lossless assistant text" }), + }); + seed(5, { + type: "item/completed", + itemId: item.id, + itemKind: "agentMessage", + data: JSON.stringify({ item }), + }); + seed(6, { + type: "turn/completed", + data: JSON.stringify({ status: "completed" }), + }); + const rows = () => listStoredEventRows(db, { threadId: thread.id }); + const advance = () => advanceThreadPruning(db, "completed-items"); + return { db, thread, seed, rows, advance }; +} + +function normalized(rows: ReturnType["rows"]>) { + return rows.map((row) => ({ ...row, data: JSON.parse(row.data) })); +} + +describe("completed items at first lifecycle position", () => { + it("preserves completion ownership, timestamps, lossless history and highwater through atomic progress", () => { + const f = setup(); + try { + const before = f.rows(); + const generation = getThreadEventRewriteGeneration(f.thread.id); + f.db + .insert(retainedEventOutputs) + .values({ + eventId: "event-5", + outputPath: "resultText", + value: "retained", + expiresAt: 999999, + }) + .run(); + const result = f.advance(); + expect(result.removed).toBe(2); + const owner = f.rows().find((row) => row.id === "event-5"); + expect(owner).toMatchObject({ + sequence: 2, + createdAt: 500, + type: "item/completed", + data: before.find((row) => row.id === "event-5")!.data, + }); + expect( + normalized(expandSelectedCompletedItemRows(f.db, f.rows())), + ).toEqual(normalized(before)); + expect(getHighWaterMarks(f.db, [f.thread.id])[f.thread.id]).toBe(6); + expect(f.db.select().from(retainedEventOutputs).all()).toHaveLength(1); + expect(getThreadEventRewriteGeneration(f.thread.id)).toBe(generation + 1); + expect(f.db.select().from(threadPruningCursors).all()).toHaveLength(1); + expect(f.advance().removed).toBe(0); + } finally { + f.db.$client.close(); + } + }); + + it.each(["user", "context"])( + "leaves %s boundary crossings ordinary so edit suffix deletion remains valid", + (boundary) => { + const f = setup(); + try { + f.seed(4, { + type: + boundary === "user" ? "client/turn/requested" : "system/operation", + scopeKind: "thread", + turnId: null, + data: JSON.stringify( + boundary === "user" + ? { initiator: "user" } + : { operation: "context_clear", status: "completed" }, + ), + }); + expect(f.advance().removed).toBe(0); + f.db.transaction((tx) => + deleteThreadEventSuffixInTransaction(tx, { + threadId: f.thread.id, + cutoffSequence: 4, + oldMaxSequence: 6, + }), + ); + expect(f.rows().map((row) => row.sequence)).toEqual([1, 2, 3]); + } finally { + f.db.$client.close(); + } + }, + ); + + it.each(["unsettled", "duplicate", "late", "malformed", "oversized"])( + "skips %s lifecycles", + (reason) => { + const f = setup(); + try { + if (reason === "unsettled") + f.db.delete(events).where(eq(events.id, "event-6")).run(); + if (reason === "duplicate") + f.seed(4, { + type: "item/completed", + itemId: "message", + itemKind: "agentMessage", + }); + if (reason === "late") + f.seed(7, { type: "item/agentMessage/delta", itemId: "message" }); + if (reason === "malformed") + f.db + .update(events) + .set({ data: "invalid" }) + .where(eq(events.id, "event-3")) + .run(); + if (reason === "oversized") + f.db + .update(events) + .set({ + data: JSON.stringify({ + itemId: "message", + delta: "x".repeat(1024 * 1024), + }), + }) + .where(eq(events.id, "event-3")) + .run(); + const before = f.rows(); + expect(f.advance().removed).toBe(0); + expect(f.rows()).toEqual(before); + } finally { + f.db.$client.close(); + } + }, + ); + + it("rolls back moved owners and source deletion together", () => { + const f = setup(); + try { + const before = f.rows(); + expect(() => + f.db.transaction((tx) => { + expect( + advanceCompletedItemCompaction(tx, { + threadId: f.thread.id, + afterSequence: 0, + throughSequence: 6, + limit: 32, + }).removed, + ).toBe(2); + throw new Error("rollback"); + }), + ).toThrow("rollback"); + expect(f.rows()).toEqual(before); + } finally { + f.db.$client.close(); + } + }); + + it("keeps later arrivals ordinary and reconstructs history after completion output mutation", () => { + const f = setup(); + try { + const before = f.rows(); + expect(f.advance().removed).toBe(2); + f.db + .update(events) + .set({ + data: JSON.stringify({ + item: { + id: "message", + type: "agentMessage", + text: "lossless assistant text", + resultText: "expired", + }, + }), + }) + .where(eq(events.id, "event-5")) + .run(); + f.seed(7, { + type: "item/agentMessage/delta", + itemId: "message", + data: JSON.stringify({ itemId: "message", delta: "late text" }), + }); + for (let i = 0; i < 10; i++) + advanceThreadPruning(f.db, { threadId: f.thread.id }); + const expanded = expandSelectedCompletedItemRows(f.db, f.rows()); + expect(normalized(expanded.filter((row) => row.sequence <= 3))).toEqual( + normalized(before.filter((row) => row.sequence <= 3)), + ); + expect(expanded.find((row) => row.sequence === 7)).toBeDefined(); + } finally { + f.db.$client.close(); + } + }); + it.each([ + { output: "kept output", delta: "kept", start: true, removed: 2 }, + { output: "kept output", delta: "uncontained", start: true, removed: 2 }, + { output: "kept output", delta: "kept", start: false, removed: 1 }, + { output: "kept output", delta: "uncontained", start: false, removed: 0 }, + { output: "", delta: "keep this", start: true, removed: 0 }, + ])( + "only discards eligible command delta text: %j", + ({ output, delta, start, removed }) => { + const f = setup(); + try { + const item = { + id: "message", + type: "commandExecution", + command: "echo output", + cwd: "/tmp", + status: "completed", + approvalStatus: null, + aggregatedOutput: output, + }; + f.db + .update(events) + .set({ itemKind: "commandExecution", data: JSON.stringify({ item }) }) + .where(eq(events.id, "event-5")) + .run(); + f.db + .update(events) + .set({ + itemKind: "commandExecution", + data: JSON.stringify({ + item: { ...item, status: "pending", aggregatedOutput: "" }, + }), + }) + .where(eq(events.id, "event-2")) + .run(); + f.db + .update(events) + .set({ + type: "item/commandExecution/outputDelta", + data: JSON.stringify({ itemId: "message", delta }), + }) + .where(eq(events.id, "event-3")) + .run(); + if (!start) f.db.delete(events).where(eq(events.id, "event-2")).run(); + expect(f.advance().removed).toBe(removed); + const expanded = expandSelectedCompletedItemRows(f.db, f.rows()); + expect( + JSON.parse(expanded.find((row) => row.id === "event-3")!.data).delta, + ).toBe(removed ? "" : delta); + } finally { + f.db.$client.close(); + } + }, + ); + + it("retains both reasoning streams and traverses combined physical records at one-row budgets", () => { + const f = setup(); + try { + const item = { + id: "message", + type: "reasoning", + content: ["full reasoning"], + summary: ["summary"], + }; + f.db + .update(events) + .set({ itemKind: "reasoning", data: JSON.stringify({ item }) }) + .where(eq(events.id, "event-5")) + .run(); + f.db + .update(events) + .set({ + itemKind: "reasoning", + data: JSON.stringify({ item: { ...item, content: [], summary: [] } }), + }) + .where(eq(events.id, "event-2")) + .run(); + f.db + .update(events) + .set({ + type: "item/reasoning/textDelta", + data: JSON.stringify({ itemId: "message", delta: "full reasoning" }), + }) + .where(eq(events.id, "event-3")) + .run(); + f.seed(4, { + type: "item/reasoning/summaryTextDelta", + itemId: "message", + data: JSON.stringify({ itemId: "message", delta: "summary" }), + }); + const before = f.rows(); + expect(f.advance().removed).toBe(3); + const seen = []; + let afterSequence = 0; + for (;;) { + const page = listStoredEventRows(f.db, { + threadId: f.thread.id, + afterSequence, + limit: 1, + }); + if (!page.length) break; + seen.push(...page); + afterSequence = page[0]!.sequence; + } + expect(normalized(expandSelectedCompletedItemRows(f.db, seen))).toEqual( + normalized(before), + ); + } finally { + f.db.$client.close(); + } + }); +}); diff --git a/packages/db/test/data/thread-pruning.test.ts b/packages/db/test/data/thread-pruning.test.ts index 5892ea3fcfb..a7b45437083 100644 --- a/packages/db/test/data/thread-pruning.test.ts +++ b/packages/db/test/data/thread-pruning.test.ts @@ -119,15 +119,17 @@ describe("thread pruning", () => { f = { ...f, db: createConnection(saved) }; } } - expect(policies.slice(0, 8)).toEqual([ + expect(policies.slice(0, 10)).toEqual([ "rate-limits", "usage", "turn-diffs", "resolved-items", + "completed-items", "rate-limits", "usage", "turn-diffs", "resolved-items", + "completed-items", ]); expect(sequences(f)).toEqual([1, 200, 201]); expect( diff --git a/packages/db/test/migrate.test.ts b/packages/db/test/migrate.test.ts index 4da166327e3..e4814f35e2d 100644 --- a/packages/db/test/migrate.test.ts +++ b/packages/db/test/migrate.test.ts @@ -872,6 +872,15 @@ function rewindMachineProvidersMigration(db: DbConnection): void { `ALTER TABLE queued_thread_messages DROP COLUMN ${name}`, ); } + if ( + db.$client + .prepare<[], TableInfoRow>("PRAGMA table_info(events)") + .all() + .some((column) => column.name === "completed_item_history") + ) { + db.$client.exec("ALTER TABLE events DROP COLUMN completed_item_history"); + + } db.$client.exec("DROP TABLE IF EXISTS thread_pruning_cursors"); db.$client.exec("DROP TABLE IF EXISTS project_attachment_threads"); db.$client.exec("DROP TABLE IF EXISTS project_attachments"); @@ -4567,6 +4576,7 @@ describe("migrate", () => { "data", "created_at", "parent_tool_call_id", + "completed_item_history", ]); const eventIndexNames = readIndexNames({ db, diff --git a/packages/plugin-api-map/src/surfaces.ts b/packages/plugin-api-map/src/surfaces.ts index 3b13c06fd48..ac715037e87 100644 --- a/packages/plugin-api-map/src/surfaces.ts +++ b/packages/plugin-api-map/src/surfaces.ts @@ -995,6 +995,7 @@ export const SURFACE_GROUPS: SurfaceGroup[] = [ "Spawn or fork with lifecycleOwnerThreadId to archive/delete a dependent with a live owner across projects; ownership is immutable, independent of sidebar parents and supports different hosts/environments. Thread responses return the owner or null. Unarchive owner first; Stop does not cascade", "List machines and suspend, resume, or remove provider-managed machines", "Read recorded context usage with sdk.threads.context({ threadId }); usage is null when unavailable, and its snapshot is present only when the latest measurement includes a breakdown", + "Read physical records through sdk.threads.events.list: settled items may combine at their first lifecycle sequence, retaining completion ID/payload/timestamp; raw exports omit internal start/delta history and budgets count physical rows", "Reach the same operations the [bb CLI](cli) and the bb UI use", "Have the threads it creates attributed back to the plugin", "Read the server's loopback URL, public app URL, and data directory when it needs server facts", diff --git a/packages/templates/src/templates/bb-guide-threads.md b/packages/templates/src/templates/bb-guide-threads.md index 4e9483fc6e3..1034e6d8e21 100644 --- a/packages/templates/src/templates/bb-guide-threads.md +++ b/packages/templates/src/templates/bb-guide-threads.md @@ -209,6 +209,13 @@ Inspecting: walks a consistent history snapshot and joins paginated group contents. Appends stay outside that walk; rerun the command if a history edit invalidates it. + Raw JSON (including JSON exports and SDK threads.events.list) contains physical + records. Settled items may be one item/completed at their first lifecycle + sequence, keeping the completion ID, payload and completion timestamp. Separate + start/delta IDs disappear; internal timing metadata is not part of raw output. + Limits and sequence cursors count these physical rows. Human timelines retain + original ordering and timing; the same row budget can include older history. + bb thread output [id] Get the final output of a thread --self Target current thread diff --git a/plugins/bb-guide/skills/bb-cli/references/thread-operation.md b/plugins/bb-guide/skills/bb-cli/references/thread-operation.md index 774043f029a..e7acd33b475 100644 --- a/plugins/bb-guide/skills/bb-cli/references/thread-operation.md +++ b/plugins/bb-guide/skills/bb-cli/references/thread-operation.md @@ -106,7 +106,11 @@ hostId, providerId, projectId, parentThreadId, groupBy })`. snapshot and join paginated group contents. Appends remain outside that walk; rerun after a cursor-invalidated error from a history edit. JSON accepts any positive limit. It defaults to the oldest 100 raw events and warns when more exist. Page with - `--after-seq ` or pass `--all`. + `--after-seq ` or pass `--all`. Raw JSON exports use physical records: + a settled item can appear as one `item/completed` at its first lifecycle + sequence, retaining its completion ID, payload and timestamp. Start/delta IDs + are no longer separate rows. Human timelines preserve original timing/order; + smaller physical-row budgets can include more history. Grep the `--all` output, not the default page, when checking whether a thread ever received a message. - Use `bb thread output ` to read the latest final output, or From b52c1047f234ba274bfaac2ff8fe3593152b0ea3 Mon Sep 17 00:00:00 2001 From: Michael Yong Date: Wed, 16 Sep 2026 18:13:16 -0700 Subject: [PATCH 2/7] Preserve output order and keep item compaction in idle maintenance --- apps/server/test/system/event-pruning.test.ts | 22 ++- .../test/system/thread-pruning-sweep.test.ts | 3 +- .../completed-item-compaction-verification.md | 186 ++++++++++-------- docs/completed-item-compaction.md | 12 +- .../db/src/data/completed-item-compaction.ts | 30 ++- packages/db/src/data/thread-pruning.ts | 7 +- .../data/completed-item-compaction.test.ts | 44 +++++ packages/db/test/data/thread-pruning.test.ts | 4 +- 8 files changed, 203 insertions(+), 105 deletions(-) diff --git a/apps/server/test/system/event-pruning.test.ts b/apps/server/test/system/event-pruning.test.ts index 69a581e237e..106af999242 100644 --- a/apps/server/test/system/event-pruning.test.ts +++ b/apps/server/test/system/event-pruning.test.ts @@ -1,4 +1,6 @@ -import { getThread, listEvents } from "@bb/db"; +import { getThread, listEvents, threads } from "@bb/db"; +import { eq } from "drizzle-orm"; +import { runThreadPruningSweep } from "../../src/services/system/thread-pruning-sweep.js"; import { turnScope } from "@bb/domain"; import { groupHostDaemonEvents } from "@bb/host-daemon-contract"; import { describe, expect, it, vi } from "vitest"; @@ -171,7 +173,7 @@ function seedResolvedAssistantMessage( } describe("thread event pruning", () => { - it("refreshes a cached visible timeline after a live completed-item rewrite and exposes combined raw records", async () => { + it("refreshes a cached visible timeline after an idle background completed-item rewrite and exposes combined raw records", async () => { await withTestHarness(async (harness) => { const host = seedHost(harness.deps); const { project } = seedProjectWithSource(harness.deps, { @@ -226,11 +228,21 @@ describe("thread event pruning", () => { expect(JSON.stringify(before)).toContain("Final answer"); expect(build()).toEqual(before); const notify = vi.spyOn(harness.deps.hub, "notifyThread"); - for (let i = 0; i < 5; i++) - pruneThreadEventHistoryBestEffort(harness.deps, { + for (let i = 0; i < 10; i++) { + const result = pruneThreadEventHistoryBestEffort(harness.deps, { threadId: thread.id, - mode: "idle", + mode: "active", }); + expect(result?.policy).not.toBe("completed-items"); + } + expect(listEvents(harness.db, { threadId: thread.id })).toHaveLength(4); + expect(notify).not.toHaveBeenCalled(); + harness.db + .update(threads) + .set({ status: "idle" }) + .where(eq(threads.id, thread.id)) + .run(); + for (let i = 0; i < 5; i++) await runThreadPruningSweep(harness.deps); expect(notify).toHaveBeenCalledWith(thread.id, ["history-rewritten"]); expect(build()).toEqual(before); const raw = await harness.app.request( diff --git a/apps/server/test/system/thread-pruning-sweep.test.ts b/apps/server/test/system/thread-pruning-sweep.test.ts index e6582d29933..981850d6295 100644 --- a/apps/server/test/system/thread-pruning-sweep.test.ts +++ b/apps/server/test/system/thread-pruning-sweep.test.ts @@ -7,6 +7,7 @@ import { getNextThreadPruningPolicy, getThreadEventRewriteGeneration, threadPruningCursors, + THREAD_PRUNING_POLICIES, threads, } from "@bb/db"; import { runThreadPruningSweep } from "../../src/services/system/thread-pruning-sweep.js"; @@ -182,7 +183,7 @@ describe("thread pruning sweep", () => { ); expect(steps.length).toBeLessThanOrEqual(64); expect(harness.db.select().from(threadPruningCursors).all()).toHaveLength( - 4, + THREAD_PRUNING_POLICIES.length, ); const before = getNextThreadPruningPolicy(harness.db, new Set()); expect(before).not.toBeNull(); diff --git a/docs/completed-item-compaction-verification.md b/docs/completed-item-compaction-verification.md index 6bd943fa32e..59f53b0c5f7 100644 --- a/docs/completed-item-compaction-verification.md +++ b/docs/completed-item-compaction-verification.md @@ -1,98 +1,112 @@ # Completed-item compaction verification -Measured on 2026-09-16 in an isolated worktree based on `5aca5733a5`. -PR1 (`c663ff1911`, #3766) is included. These are observations from private -sanitized SQLite copies, not latency or savings guarantees. +Measured on 2026-09-16 in an isolated worktree based on `5aca5733a5`, +including PR1 (`c663ff1911`, #3766). The final measurements include the +output-order safeguard and exclusion of compaction from synchronous per-thread +cleanup. These are observations from private sanitized SQLite copies. ## Correctness and storage -| Check | Result | -| --- | --- | -| Physical events | 2,082,639 → 1,080,480 (1,002,159 removed) | -| All table rows | 2,571,187 → 1,569,030 (net 1,002,157 removed) | -| Combined owners | 793,808 | -| Internal metadata | 182,951,145 bytes | -| Allocated SQLite btree bytes | 5,634,584,576 → 4,411,244,544 (1,223,340,032 saved) | -| Database file bytes | 6,006,726,656 before and after; freed pages are reusable | -| Independent logical reconstruction | All 2,082,639 records match across 2,326 threads, except 119,716 eligible empty command-delta text markers | -| Complete visible timelines | All 2,326 match, including all 2,223 non-null context values | -| Highwater and provider recovery | All 2,326 match | -| Existing side tables | All 46 non-event/non-cursor/non-migration tables match, including output, search and attachment ownership | -| Default-budget server pages | All 30 match exactly | -| Complete large-thread pagination | Three largest threads: 3,816 rendered rows match after complete traversal at a 1,000-row budget; baseline used 10,000 | -| Fork inheritance | 24 ordinary plus 10 interleaved real cases; 2,638 copied records match | -| Message edit suffixes | 24 real boundary cases; no future output survives | -| Other consumers | 2,326 latest-output checks, ten large outlines and 100 rollback-only output mutations match | - -The two-row difference between physical removals and net removals is existing -migration/cursor bookkeeping. There are no added tables or indexes and no full -VACUUM. The worker performs actual forward discovery and mutation; this is not a -conversion of the prototype's auxiliary-table results. - -The first worker pass removes 13 fewer rows than the prototype. A direct cohort -comparison finds 17 additional command owners left ordinary by the bounded -boundary probe (29 rows), offset by 16 additional eligible start rows. All shared -owners have the same reconstructed record counts. User-message/context-clear -crossings remain excluded, as do ambiguous/unsettled/incompatible lifecycles, -unsupported or repeated delta streams, malformed payloads and work exceeding -support/input budgets. No arbitrary mid-item rewind repair was added. +| Check | Result | +| ---------------------------------- | ------------------------------------------------------------------------------------------------------------------------- | +| Physical events | 2,082,639 → 1,080,655 (1,001,984 removed) | +| All table rows | 2,571,187 → 1,569,205 (net 1,001,982 removed) | +| Combined owners | 793,715 | +| Internal metadata | 182,920,304 bytes | +| Allocated SQLite btree bytes | 5,634,584,576 → 4,411,314,176 (1,223,270,400 saved) | +| Database file bytes | 6,006,726,656 before and after; freed pages are reusable | +| Independent logical reconstruction | All 2,082,639 records match across 2,326 threads, except 119,716 eligible empty command-delta text markers | +| Complete visible timelines | All 2,326 match, including all 2,223 non-null context values | +| Highwater and provider recovery | All 2,326 match | +| Existing side tables | All 46 non-event/non-cursor/non-migration tables match, including output, search and attachment ownership | +| Default-budget server pages | All 30 match exactly | +| Complete large-thread pagination | Three largest threads: all 3,816 rendered rows match after complete traversal at a 1,000-row budget; baseline used 10,000 | +| Other consumers | 2,326 latest-output checks, ten large outlines and 100 rollback-only output mutations match | + +There are no added tables or indexes and no full VACUUM. The two-row difference +between physical and total removals is migration/cursor bookkeeping. The worker +performs actual discovery and mutation on a fresh clone. + +The output-order safeguard leaves 93 more owners ordinary, retaining 175 rows +that the first revision removed. Assistant items crossing another assistant +completion or manager output must stay ordinary: otherwise moving the completion +can make the output API, plugin summaries and child notifications choose an older +message. Both cases were reproduced through the actual server output helper and +now pass regression tests. Existing user-message/context-clear exclusions and +bounded support/input discovery remain in place. ## Timing and catch-up -Measurements ran sequentially, without overlapping heavy benchmarks. The host -was shared; these are warm local observations, not cold-I/O or network bounds. - -| Measurement | Result | -| --- | --- | -| Full completed-item worker pass | 120,866 advances; 306.51 seconds | -| Background advance median / p99 / maximum | 1.90 / 15.81 / 212.28 ms | -| Full live cleanup wrapper, 2,500 rotating advances | median 0.257 ms; p99 6.55 ms; maximum 270.07 ms | -| Completed-item subset of live wrapper, 500 advances | median 0.932 ms; p99 8.43 ms; maximum 270.07 ms | -| Median of 30 page case medians, before / after | 94.50 / 111.75 ms | -| Largest individual page call, before / after | 374.21 / 473.56 ms | -| Actual background sweep, 100 calls | 1,989 total advances; 398 completed-item advances | -| Maximum sweep / advance in that sample | 193.34 / 158.88 ms; the largest advance was PR1 resolved-item pruning | - -The full live wrapper includes policy rotation, transaction, generation checks, -logging and notification calls; the offline harness collects logs/notifications -in process. No server or provider runs on a research copy. The background sweep -sample explicitly makes its private performance copy idle; the original copied -activity state otherwise blocks maintenance. - -At 3.98 completed-item advances per sweep and the existing ten-second sweep -cadence, the first-pass catch-up estimate is about **84 idle hours**. Activity, -other policies, I/O and larger excluded lifecycles can extend elapsed time. -The implementation retains PR1's scheduler and allocator. Bounded rows/bytes do -not impose a hard stall cap. Reader median overhead and initial catch-up time -remain limitations for review. +Heavy measurements ran sequentially. The host was shared; these observations do +not establish cold-I/O bounds or a maximum event-loop stall. + +| Measurement | Result | +| ------------------------------------------------------- | ------------------------------------------------- | +| Full completed-item worker pass | 120,866 advances; 304.50 seconds | +| Background advance median / p99 / maximum | 1.90 / 15.18 / 372.18 ms | +| Final live wrapper, 2,500 calls | median 0.288 ms; p99 5.46 ms; maximum 322.25 ms | +| PR1 live wrapper, same 2,500-call workload | median 0.243 ms; p99 5.28 ms; maximum 87.34 ms | +| Compaction calls / event deletions in final live sample | zero / zero | +| Median of 30 page case medians, before / after | 103.62 / 121.55 ms | +| Median paired page increase | 8.42 ms | +| Largest individual page call, before / after | 391.81 / 464.44 ms | +| Actual background sweep, 100 calls | 1,565 total advances; 313 completed-item advances | +| Maximum sweep / advance in that sample | 216.46 / 177.78 ms | + +The original 270 ms live compaction stall was independently reproduced at 249 ms. +An isolated replay attributed the delay to transaction commit. Disabling automatic +checkpointing on a disposable copy reduced that compaction call to 1.53 ms; +an explicit checkpoint immediately afterwards took 333.05 ms for 1,002 WAL frames. +That experiment moved the cost; it was not a fix. No checkpoint settings change +in this PR. + +Completed-item compaction now runs only in the existing idle background sweep. +The synchronous per-thread rotation retains PR1's four policies. This removes +compaction writes from ingestion and turn-completion handlers, but does not +eliminate SQLite checkpoint stalls from existing writes. The final live maximum +occurred during the existing usage policy with zero event deletions; cursor +transactions still write. The PR1 comparison also exceeded 50 ms. These samples +do not prove identical worst-case latency or quantify a regression from the +individual maxima. No sub-50 ms guarantee is made. + +The live harness includes the real wrapper, transaction, generation checks and +in-process notification/logging calls. The background sample explicitly marks +its private copy idle. No server or provider runs on a research copy. + +At 3.13 completed-item advances per sweep and the existing ten-second cadence, +the latest sample extrapolates to about **107 idle hours** for first-pass catch-up. +The initial revision's sample estimated 84 hours. Actual processing time remains +about five minutes; scheduling, activity and I/O determine elapsed catch-up. +These are short-sample estimates, not an end-to-end scheduled run. Reader overhead, +checkpoint stalls and multi-day idle catch-up remain limitations. ## Automated and UI verification -- Turbo DB tests: 44 files, 589 tests pass, using migrated real SQLite. -- Turbo server tests: 16 relevant files, 200 tests pass, covering timeline caches, - pagination, truncation, context clearing, message editing, forks and live pruning. -- New complete traversal tests pass at row budgets 1, 2, 5 and 20 with a 512-byte - response budget, preserving canonical content through client page merging. +Final follow-up checks: + +- Turbo DB tests: 44 files, 591 tests pass, using migrated real SQLite. +- Turbo affected server tests: 14 tests pass, including the previously failing + scheduler test and cached timeline refresh through the actual idle sweep. +- Live cleanup leaves the completion ordinary; idle background cleanup combines + it and invalidates the warmed timeline. Raw output still excludes metadata. - Turbo DB/server typechecks pass. -- Plugin Guide tests/typecheck pass: ten files, 73 tests. -- Fresh isolated Chromium UI: expand command output, file diff and reasoning; - preserve text and durations through an actual background rewrite; reload and - verify persisted rendering. A second rewrite holds highwater at 15 while rows - fall from 15 to eight. The open page automatically requests - `timeline?afterSequence=15`, without navigation and with identical visible text. -- The synthetic UI's first fixture initially used the default starting status - and produced a provisioning error. The fixture was repaired to idle before the - measured rewrite. No provider turn was sent. Browser and dev processes were - stopped after verification. - -The verification inventory reports pre-existing unmapped `browser` CLI-family -drift on the fetched base. This does not represent a passed inventory check. -No iOS or provider-resume claim is made; this change does not alter drawer UI or -provider execution. Native edit/fork behavior was exercised through the actual -server functions on private copies and existing route tests. - -Reproduction scripts, source briefs, full JSON comparisons, timings, logs and UI -screenshots are preserved in the implementing thread's `pr2-start-position` -storage directory. Research databases remain private and mode 0600; Connect -plugin records were verified absent. No live database was opened, no copied live -configuration was launched, and no deployment or merge was performed. +- Full-copy comparisons and pagination listed above were rerun after the fixes. + +The initial revision additionally passed 200 selected server tests and 73 Plugin +Guide tests, small-budget traversal tests at budgets 1/2/5/20 with 512-byte +responses, 34 real fork cases (2,638 copied records), and 24 real edit-boundary +rewinds. Fresh synthetic Chromium verification proved command output, file diffs, +reasoning, answers and durations survive background rewriting and reload. A +rewrite with unchanged highwater refreshed the open browser automatically. Those +UI/fork/edit checks were not rerun in the follow-up; renderer, fork and edit code +are unchanged, and compaction eligibility is narrower. + +The initial verification inventory reported pre-existing unmapped `browser` +CLI-family drift. No iOS or provider-resume claim is made. Dev processes/browser +from the initial verification were stopped. + +Initial evidence remains under implementing thread `thr_mwh5k9hti7` in +`pr2-start-position`. Follow-up scripts, full comparisons, timings and logs are +under parent thread `thr_vmdgc3ke5y` in `pr2-review/final`. Copies are private and +mode 0600, with Connect plugin records verified absent. No live database or copied +Connect configuration was used; nothing was merged or deployed. diff --git a/docs/completed-item-compaction.md b/docs/completed-item-compaction.md index 889ac602bf0..6d01dae858f 100644 --- a/docs/completed-item-compaction.md +++ b/docs/completed-item-compaction.md @@ -1,6 +1,6 @@ # Completed item history compaction -The existing event-maintenance rotation combines eligible settled item history +The existing idle background event-maintenance rotation combines eligible settled item history into its `item/completed` row. The completion keeps its ID, payload, original creation timestamp and retained-output ownership. Its physical sequence becomes the first retained lifecycle sequence. One versioned internal column stores the @@ -14,6 +14,8 @@ reasoning. Unpruned repeated delta streams, file-change output deltas, malformed payloads, incompatible envelopes and oversized discovery/payload windows remain ordinary. A lifetime crossing a user request or completed context clear remains ordinary so message editing can keep its existing suffix-deletion behavior. +Assistant lifetimes crossing another assistant completion or manager output also +remain ordinary, preserving the output API and child completion notifications. Assistant and reasoning delta text is retained. A command delta can become an empty timing marker only when the command has completed, failed or been @@ -27,7 +29,13 @@ advance bounds candidate/support rows and input bytes; ambiguous or larger lifecycles are skipped, not partially rewritten. Source deletion, completion movement and cursor progress commit together. PR1's generation increment and `history-rewritten` notification invalidate cached views after the commit. -Limits bound work rather than guaranteeing a maximum elapsed time. +Completed-item compaction is excluded from the synchronous per-thread cleanup +rotation used by ingestion and turn-completion handlers. Existing PR1 live +cleanup remains unchanged; the idle sweep performs compaction and refreshes +open clients. This avoids adding compaction writes to the live path. SQLite +checkpointing can still stall a background transaction (and unrelated live +writes); limits bound work rather than guaranteeing a maximum elapsed time. +No checkpoint setting is changed. Timeline queries select physical rows first and only then decode their internal metadata for projection. Selected fork history recovers completion order before diff --git a/packages/db/src/data/completed-item-compaction.ts b/packages/db/src/data/completed-item-compaction.ts index c187957ae06..1a0fa42743a 100644 --- a/packages/db/src/data/completed-item-compaction.ts +++ b/packages/db/src/data/completed-item-compaction.ts @@ -190,21 +190,37 @@ export function advanceCompletedItemCompaction( const firstSequence = source[0]!.sequence; const boundaryRows: { id: string; dataBytes: number }[] = []; let boundaryBudgetExhausted = false; - for (const type of ["client/turn/requested", "system/operation"]) { + let crossesOutputBoundary = false; + const boundaryTypes = ["client/turn/requested", "system/operation"]; + if (kind === "agentMessage") + boundaryTypes.push("item/completed", "system/manager/user_message"); + for (const type of boundaryTypes) { const limit = Math.min(8, args.limit - scanned); if (limit <= 0) { boundaryBudgetExhausted = true; break; } - const found = db.all<{ id: string; dataBytes: number }>(sql` - SELECT id, octet_length(data) AS dataBytes + const found = db.all<{ + id: string; + dataBytes: number; + itemKind: string | null; + }>(sql` + SELECT id, octet_length(data) AS dataBytes, item_kind AS itemKind FROM events INDEXED BY events_thread_type_sequence_idx WHERE thread_id = ${args.threadId} AND type = ${type} - AND sequence > ${firstSequence} AND sequence <= ${candidate.sequence} + AND sequence > ${firstSequence} AND sequence < ${candidate.sequence} ORDER BY sequence LIMIT ${limit} `); scanned += Math.max(1, found.length); - boundaryRows.push(...found); + if (type === "item/completed") { + crossesOutputBoundary ||= found.some( + (row) => row.itemKind === "agentMessage", + ); + } else if (type === "system/manager/user_message") { + crossesOutputBoundary ||= found.length > 0; + } else { + boundaryRows.push(...found); + } if (found.length === limit) { boundaryBudgetExhausted = true; break; @@ -214,6 +230,10 @@ export function advanceCompletedItemCompaction( skip(kind, "boundary-budget"); continue; } + if (crossesOutputBoundary) { + skip(kind, "output-order-boundary"); + continue; + } const boundaryBytes = boundaryRows.reduce( (total, row) => total + row.dataBytes, 0, diff --git a/packages/db/src/data/thread-pruning.ts b/packages/db/src/data/thread-pruning.ts index e2cb11f2208..42a8ade5b1a 100644 --- a/packages/db/src/data/thread-pruning.ts +++ b/packages/db/src/data/thread-pruning.ts @@ -53,9 +53,10 @@ export function getNextThreadPruningPolicy( .all(); const updated = new Map(rows.map((row) => [row.policy, row.updatedAt])); return ( - THREAD_PRUNING_POLICIES.filter((policy) => !excluded.has(policy)).sort( - (a, b) => (updated.get(a) ?? 0) - (updated.get(b) ?? 0), - )[0] ?? null + THREAD_PRUNING_POLICIES.filter( + (policy) => + !excluded.has(policy) && (scope === "" || policy !== "completed-items"), + ).sort((a, b) => (updated.get(a) ?? 0) - (updated.get(b) ?? 0))[0] ?? null ); } diff --git a/packages/db/test/data/completed-item-compaction.test.ts b/packages/db/test/data/completed-item-compaction.test.ts index df48e227e4b..b28f59ed2a3 100644 --- a/packages/db/test/data/completed-item-compaction.test.ts +++ b/packages/db/test/data/completed-item-compaction.test.ts @@ -15,6 +15,7 @@ import { expandSelectedCompletedItemRows } from "../../src/data/completed-item-h import { listStoredEventRows, getHighWaterMarks, + getLatestThreadOutputEventRow, deleteThreadEventSuffixInTransaction, } from "../../src/data/events.js"; import { getThreadEventRewriteGeneration } from "../../src/data/event-rewrite-generation.js"; @@ -194,6 +195,49 @@ describe("completed items at first lifecycle position", () => { }, ); + it.each(["assistant", "manager"])( + "preserves latest output when a completion crosses another %s output", + (kind) => { + const f = setup(); + try { + f.seed( + 4, + kind === "manager" + ? { + type: "system/manager/user_message", + scopeKind: "thread", + turnId: null, + data: JSON.stringify({ text: "earlier manager output" }), + } + : { + type: "item/completed", + itemId: "earlier-message", + itemKind: "agentMessage", + data: JSON.stringify({ + item: { + id: "earlier-message", + type: "agentMessage", + text: "earlier assistant output", + }, + }), + }, + ); + const before = f.rows(); + const output = getLatestThreadOutputEventRow(f.db, { + threadId: f.thread.id, + }); + expect(output?.id).toBe("event-5"); + expect(f.advance().removed).toBe(0); + expect(f.rows()).toEqual(before); + expect( + getLatestThreadOutputEventRow(f.db, { threadId: f.thread.id }), + ).toEqual(output); + } finally { + f.db.$client.close(); + } + }, + ); + it("rolls back moved owners and source deletion together", () => { const f = setup(); try { diff --git a/packages/db/test/data/thread-pruning.test.ts b/packages/db/test/data/thread-pruning.test.ts index a7b45437083..5892ea3fcfb 100644 --- a/packages/db/test/data/thread-pruning.test.ts +++ b/packages/db/test/data/thread-pruning.test.ts @@ -119,17 +119,15 @@ describe("thread pruning", () => { f = { ...f, db: createConnection(saved) }; } } - expect(policies.slice(0, 10)).toEqual([ + expect(policies.slice(0, 8)).toEqual([ "rate-limits", "usage", "turn-diffs", "resolved-items", - "completed-items", "rate-limits", "usage", "turn-diffs", "resolved-items", - "completed-items", ]); expect(sequences(f)).toEqual([1, 200, 201]); expect( From 12091ea89d72ac7b22bf3845ddb2c82232c115ca Mon Sep 17 00:00:00 2001 From: Michael Yong Date: Thu, 17 Sep 2026 15:13:36 -0700 Subject: [PATCH 3/7] Reduce completed-item history reconstruction overhead --- .../completed-item-compaction-verification.md | 31 +++++++++++ packages/db/src/completed-item-history.ts | 7 ++- .../db/src/data/completed-item-history.ts | 21 ++++++-- .../db/test/completed-item-history.test.ts | 52 +++++++++++++++++++ .../data/completed-item-compaction.test.ts | 17 ++++++ 5 files changed, 122 insertions(+), 6 deletions(-) create mode 100644 packages/db/test/completed-item-history.test.ts diff --git a/docs/completed-item-compaction-verification.md b/docs/completed-item-compaction-verification.md index 59f53b0c5f7..3b4ca829034 100644 --- a/docs/completed-item-compaction-verification.md +++ b/docs/completed-item-compaction-verification.md @@ -110,3 +110,34 @@ Initial evidence remains under implementing thread `thr_mwh5k9hti7` in under parent thread `thr_vmdgc3ke5y` in `pr2-review/final`. Copies are private and mode 0600, with Connect plugin records verified absent. No live database or copied Connect configuration was used; nothing was merged or deployed. + +## Reader optimization follow-up (2026-09-17) + +Compared with rebased PR2 `e2091fbea2` (base `0188d91972`), the reader now +fetches selected completion metadata in one parameterized query instead of +250-ID batches. SQLite uses the existing events primary-key index; `json_each` +expands only the supplied ID array. Nested history no longer goes through an +extra JSON serialization, parse and validation, and reconstruction skips parsing +completion payloads when it needs none of their fields. Storage, compaction +rules, cache state and indexes are unchanged. + +Ten large selections, alternating original and optimized reconstruction ten times +per case, matched exactly. Median relative reconstruction improvement was 22.6%. +One 10,000-row selection improved from 170.1 to 135.5 ms, with 35 metadata queries +reduced to one. The query plan uses `sqlite_autoindex_events_1`, not an events scan. + +Thirty matching page requests were also measured with six alternating calls per +implementation, after warming both. All response fields and pagination matched. +The median paired elapsed reduction was 7.3 ms; process CPU time fell by a median +paired 8.3 ms, with lower CPU usage in 26 of 30 cases. The host was busy, so elapsed +timings are approximate and are not expected production latency. This compares +optimized PR2 with the original rebased PR2; it does not establish that PR2 is as +fast as main. Component reconstruction savings are not whole-page percentages. + +The follow-up passes 602 DB tests, 74 server tests, and DB/server typechecks. +All 2,326 complete timelines and context values match the uncompacted baseline, +with zero differences or errors. This reader change does not establish a new +maximum live or background event-loop stall time. + +Artifacts and harnesses: +`/Users/michael/.bb/thread-storage/thr_vmdgc3ke5y/pr2-reader-optimization/`. diff --git a/packages/db/src/completed-item-history.ts b/packages/db/src/completed-item-history.ts index 482dbd54845..9a8d3833fa2 100644 --- a/packages/db/src/completed-item-history.ts +++ b/packages/db/src/completed-item-history.ts @@ -62,7 +62,10 @@ function fieldNames(value: JsonValue | undefined): string[] { } export function decodeHistory(data: string): HistoryRecord[] { - const value = jsonValueSchema.parse(JSON.parse(data)); + return decodeHistoryValue(jsonValueSchema.parse(JSON.parse(data))); +} + +function decodeHistoryValue(value: JsonValue): HistoryRecord[] { if ( !Array.isArray(value) || value.length !== 2 || @@ -215,7 +218,7 @@ export function decodeCompletedItemHistory(data: string) { !Number.isFinite(value[2]) ) throw new Error("Invalid combined completed item history"); - const records = decodeHistory(JSON.stringify(value[3])); + const records = decodeHistoryValue(value[3]); const sequence = value[1]; if (records.some((row) => row.sequence >= sequence)) throw new Error("Completed item history exceeds completion sequence"); diff --git a/packages/db/src/data/completed-item-history.ts b/packages/db/src/data/completed-item-history.ts index 1612b7687c9..ae29f46dd3a 100644 --- a/packages/db/src/data/completed-item-history.ts +++ b/packages/db/src/data/completed-item-history.ts @@ -1,4 +1,4 @@ -import { inArray } from "drizzle-orm"; +import { and, inArray, isNotNull, sql } from "drizzle-orm"; import type { DbQueryConnection } from "../connection.js"; import { events } from "../schema.js"; import { @@ -22,11 +22,19 @@ export function expandSelectedCompletedItemRows( string, ReturnType >(); - for (let offset = 0; offset < ids.length; offset += 250) { + if (ids.length > 0) { const selected = db .select({ id: events.id, history: events.completedItemHistory }) .from(events) - .where(inArray(events.id, ids.slice(offset, offset + 250))) + .where( + and( + inArray( + events.id, + sql`(select value from json_each(${JSON.stringify(ids)}))`, + ), + isNotNull(events.completedItemHistory), + ), + ) .all(); for (const row of selected) { if (row.history !== null) @@ -46,7 +54,12 @@ export function expandSelectedCompletedItemRows( sequence: history.sequence, createdAt: history.createdAt, }); - const payload = parseHistoryPayload(row.data); + const payload = history.records.some( + (record) => + record.sharedFields.length > 0 || record.sharedItemFields.length > 0, + ) + ? parseHistoryPayload(row.data) + : {}; for (const record of history.records) { if (record.sequence <= throughSequence) expanded.push({ diff --git a/packages/db/test/completed-item-history.test.ts b/packages/db/test/completed-item-history.test.ts new file mode 100644 index 00000000000..66b1394ce50 --- /dev/null +++ b/packages/db/test/completed-item-history.test.ts @@ -0,0 +1,52 @@ +import { describe, expect, it } from "vitest"; +import { + decodeCompletedItemHistory, + decodeHistory, +} from "../src/completed-item-history.js"; + +const record = [ + "start", + 1, + 100, + "item/started", + "agentMessage", + [{ item: { text: "" } }, [], ["id", "type"]], +]; + +describe("completed item history decoding", () => { + it("decodes nested records with the standalone history contract", () => { + const history = [1, [record]]; + expect( + decodeCompletedItemHistory(JSON.stringify([1, 3, 300, history])), + ).toEqual({ + sequence: 3, + createdAt: 300, + records: decodeHistory(JSON.stringify(history)), + }); + }); + + it.each([ + [2, [record]], + [1, []], + [1, [record, record]], + [1, [["start", 1, 100, "turn/completed", "agentMessage", [{}, [], []]]]], + [ + 1, + [ + [ + "start", + 1, + 100, + "item/started", + "agentMessage", + [{}, [], ["resultText"]], + ], + ], + ], + [1, [["start", 3, 100, "item/started", "agentMessage", [{}, [], []]]]], + ])("rejects invalid nested history %j", (...history) => { + expect(() => + decodeCompletedItemHistory(JSON.stringify([1, 3, 300, history])), + ).toThrow(); + }); +}); diff --git a/packages/db/test/data/completed-item-compaction.test.ts b/packages/db/test/data/completed-item-compaction.test.ts index b28f59ed2a3..808ac40efee 100644 --- a/packages/db/test/data/completed-item-compaction.test.ts +++ b/packages/db/test/data/completed-item-compaction.test.ts @@ -87,6 +87,23 @@ function normalized(rows: ReturnType["rows"]>) { } describe("completed items at first lifecycle position", () => { + it("expands selected history alongside more than a thousand ordinary completions", () => { + const f = setup(); + try { + for (let sequence = 10; sequence < 1011; sequence++) { + f.seed(sequence, { type: "item/completed" }); + } + const before = f.rows(); + for (let index = 0; index < 20; index++) f.advance(); + expect(f.rows().length).toBe(before.length - 2); + expect( + normalized(expandSelectedCompletedItemRows(f.db, f.rows())), + ).toEqual(normalized(before)); + } finally { + f.db.$client.close(); + } + }); + it("preserves completion ownership, timestamps, lossless history and highwater through atomic progress", () => { const f = setup(); try { From 38daa92ae07d94d68d068a3fbbed4709c1544573 Mon Sep 17 00:00:00 2001 From: Michael Yong Date: Thu, 17 Sep 2026 16:13:12 -0700 Subject: [PATCH 4/7] Bound reconstructed timeline work and avoid redundant history decoding --- .../threads/stored-event-decode-cache.ts | 8 +- .../src/services/threads/thread-data.ts | 13 +- .../threads/timeline-selection-memo.ts | 6 +- apps/server/src/services/threads/timeline.ts | 11 +- docs/completed-item-compaction.md | 14 +- packages/db/src/completed-item-history.ts | 34 ++++- .../db/src/data/completed-item-history.ts | 102 +++++++++++---- packages/db/src/data/events.ts | 54 +++++--- packages/db/src/index.ts | 6 +- .../db/test/completed-item-history.test.ts | 30 +++++ .../data/completed-item-compaction.test.ts | 123 +++++++++++++++++- 11 files changed, 335 insertions(+), 66 deletions(-) diff --git a/apps/server/src/services/threads/stored-event-decode-cache.ts b/apps/server/src/services/threads/stored-event-decode-cache.ts index 8f116ed2dc2..75ac2ae0fe3 100644 --- a/apps/server/src/services/threads/stored-event-decode-cache.ts +++ b/apps/server/src/services/threads/stored-event-decode-cache.ts @@ -1,4 +1,8 @@ -import type { DbConnection, StoredEventRow } from "@bb/db"; +import type { + DbConnection, + StoredEventRow, + ProjectionStoredEventRow, +} from "@bb/db"; import type { ThreadEvent } from "@bb/domain"; import { parseStoredEvent } from "./thread-data.js"; @@ -119,7 +123,7 @@ function rememberEntry( export function decodeStoredEventRowCached( db: DbConnection, - row: StoredEventRow, + row: ProjectionStoredEventRow, ): ThreadEvent { const cache = getDecodeCache(db); const byRow = cache.byRow.get(row); diff --git a/apps/server/src/services/threads/thread-data.ts b/apps/server/src/services/threads/thread-data.ts index a9ee37f1486..855bc497da0 100644 --- a/apps/server/src/services/threads/thread-data.ts +++ b/apps/server/src/services/threads/thread-data.ts @@ -6,7 +6,11 @@ import { hydrateRetainedEventOutputRows, listStoredEventRows as listStoredEventRowRecords, } from "@bb/db"; -import type { DbConnection, StoredEventRow } from "@bb/db"; +import type { + DbConnection, + StoredEventRow, + ProjectionStoredEventRow, +} from "@bb/db"; import { toRecord } from "@bb/core-ui"; import { buildThreadEventRow, parseStoredThreadEvent } from "@bb/domain"; import { threadScope, turnScope } from "@bb/domain"; @@ -21,8 +25,8 @@ import { ApiError } from "../../errors.js"; const THREAD_EVENT_RESPONSE_DATA_BYTE_LIMIT = 8 * 1024 * 1024; type StoredEventPayloadRow = Pick< - StoredEventRow, - "data" | "sequence" | "threadId" | "type" + ProjectionStoredEventRow, + "data" | "sequence" | "threadId" | "type" | "parsedData" >; interface ListThreadEventRowsArgs { @@ -43,6 +47,7 @@ interface FindThreadEventArgs { export function parseStoredEventPayload( row: StoredEventPayloadRow, ): Record { + if (row.parsedData !== undefined) return row.parsedData; let data: unknown; try { data = JSON.parse(row.data); @@ -88,7 +93,7 @@ function parseStoredEventScope(row: StoredEventRow): ThreadEventScope { } } -export function parseStoredEvent(row: StoredEventRow): ThreadEvent { +export function parseStoredEvent(row: ProjectionStoredEventRow): ThreadEvent { return parseStoredThreadEvent({ type: row.type, data: parseStoredEventPayload(row), diff --git a/apps/server/src/services/threads/timeline-selection-memo.ts b/apps/server/src/services/threads/timeline-selection-memo.ts index 05acfd36619..03f5e8b0442 100644 --- a/apps/server/src/services/threads/timeline-selection-memo.ts +++ b/apps/server/src/services/threads/timeline-selection-memo.ts @@ -136,7 +136,7 @@ function getSelectionMemo(db: DbConnection): TimelineSelectionMemo { function dataCharsOfRows(rows: readonly StoredEventRow[]): number { let chars = 0; for (const row of rows) { - chars += row.data.length; + chars += row.data.length + (row.completedItemHistory?.length ?? 0); } return chars; } @@ -280,7 +280,9 @@ export function lookupLatestTimelineSelection( const budgetFloor: TimelineBudgetFloor = { sequence: findTimelineWindowBudgetFloorSequence(db, { threadId: args.threadId, - sequenceStart: args.epochSequenceStart, + sequenceStart: + entry.hints[args.page.segmentLimit]?.sequence ?? + args.epochSequenceStart, beforeSequence: args.maxSeq + 1, eventBudget: args.eventBudget, excludedTypes: THREAD_TIMELINE_EXCLUDED_EVENT_TYPES, diff --git a/apps/server/src/services/threads/timeline.ts b/apps/server/src/services/threads/timeline.ts index 0c0fa69d231..c5b4c2409d1 100644 --- a/apps/server/src/services/threads/timeline.ts +++ b/apps/server/src/services/threads/timeline.ts @@ -1,4 +1,4 @@ -import { expandSelectedCompletedItemRows } from "@bb/db"; +import { expandSelectedCompletedItemRowsForProjection } from "@bb/db"; import { paginateTimelineContents } from "./timeline-content-pagination.js"; import { getTimelineGroupingContext, @@ -973,7 +973,8 @@ function selectStandardTimelineEventRows( knownBudgetFloor === null ? findTimelineWindowBudgetFloorSequence(db, { threadId: thread.id, - sequenceStart: epochSequenceStart, + sequenceStart: + hints[page.segmentLimit]?.sequence ?? epochSequenceStart, beforeSequence, eventBudget, excludedTypes: THREAD_TIMELINE_EXCLUDED_EVENT_TYPES, @@ -1358,7 +1359,7 @@ function buildThreadTimelineInternal( rows: hydrateRetainedEventOutputRows(db, storedEventSelection.rows), } : storedEventSelection; - const rawEventRows = expandSelectedCompletedItemRows( + const rawEventRows = expandSelectedCompletedItemRowsForProjection( db, eventSelection.rows, snapshot.maxSeq, @@ -1629,7 +1630,7 @@ export function buildThreadConversationOutline( sequenceStart: contextBoundarySeq ?? 0, threadId: thread.id, }); - const decodedRawEvents = expandSelectedCompletedItemRows( + const decodedRawEvents = expandSelectedCompletedItemRowsForProjection( db, rawEventRows, ).map((row) => toThreadEventWithMeta(row)); @@ -1932,7 +1933,7 @@ function buildTimelineTurnSummaryDetailsPage( : sourceSeqStart, sourceRange.sourceSeqStart, ); - const projectionEvents = expandSelectedCompletedItemRows( + const projectionEvents = expandSelectedCompletedItemRowsForProjection( db, projectionEventRows, snapshot.maxSeq, diff --git a/docs/completed-item-compaction.md b/docs/completed-item-compaction.md index 6d01dae858f..16850e1e8bb 100644 --- a/docs/completed-item-compaction.md +++ b/docs/completed-item-compaction.md @@ -37,8 +37,11 @@ checkpointing can still stall a background transaction (and unrelated live writes); limits bound work rather than guaranteeing a maximum elapsed time. No checkpoint setting is changed. -Timeline queries select physical rows first and only then decode their internal -metadata for projection. Selected fork history recovers completion order before +Timeline queries select physical rows with their internal metadata, then reconstruct +only the selected history for projection. The timeline event budget charges for +the records inside each combined item, so removing physical rows does not cause +a page to decode substantially more history. Reconstruction passes validated +payload objects directly to projection instead of parsing them again. Selected fork history recovers completion order before applying the existing completed-turn and event-type rules. Forks still create new IDs and sequence numbers. No arbitrary deleted-ID lookup or virtual raw-event pagination is provided. @@ -56,9 +59,10 @@ transcript. Physical-row limits and sequence cursors count combined rows. Restart a traversal after a history rewrite rather than continuing an old cursor against changed history. Human CLI formats and the UI reconstruct original ordering, timing, -text and edit cards. A fixed physical-row budget can include more history after -compaction; exact page boundaries are not promised. Response byte limits remain -independent of row limits, and stored timeline byte accounting includes metadata. +text and edit cards. Raw export limits count physical rows; the timeline work +budget counts reconstructed records. Exact page boundaries are not promised. +Response byte limits remain independent of row limits, and stored timeline byte +accounting includes metadata. ## Verification diff --git a/packages/db/src/completed-item-history.ts b/packages/db/src/completed-item-history.ts index 9a8d3833fa2..aff48d4106b 100644 --- a/packages/db/src/completed-item-history.ts +++ b/packages/db/src/completed-item-history.ts @@ -1,6 +1,5 @@ import { isDeepStrictEqual } from "node:util"; import { - jsonValueSchema, threadEventTypeSchema, type JsonObject, type JsonValue, @@ -45,8 +44,22 @@ function isObject(value: JsonValue | undefined): value is JsonObject { return value !== null && typeof value === "object" && !Array.isArray(value); } +function assertFiniteJsonNumbers(value: unknown): void { + if (typeof value === "number" && !Number.isFinite(value)) + throw new Error("Non-finite completed item history number"); + if (value !== null && typeof value === "object") { + for (const child of Object.values(value)) assertFiniteJsonNumbers(child); + } +} + +function parseHistoryJson(data: string): JsonValue { + const value: unknown = JSON.parse(data); + assertFiniteJsonNumbers(value); + return value as JsonValue; +} + export function parseHistoryPayload(data: string): JsonObject { - const parsed = jsonValueSchema.parse(JSON.parse(data)); + const parsed = parseHistoryJson(data); if (!isObject(parsed)) throw new Error("Completed item history payload must be an object"); return parsed; @@ -62,7 +75,7 @@ function fieldNames(value: JsonValue | undefined): string[] { } export function decodeHistory(data: string): HistoryRecord[] { - return decodeHistoryValue(jsonValueSchema.parse(JSON.parse(data))); + return decodeHistoryValue(parseHistoryJson(data)); } function decodeHistoryValue(value: JsonValue): HistoryRecord[] { @@ -179,10 +192,10 @@ export function compactHistoryPayload(data: JsonObject, owner: JsonObject) { return { payload, sharedFields, sharedItemFields }; } -export function restoreHistoryPayload( +export function restoreHistoryPayloadObject( record: HistoryRecord, owner: JsonObject, -): string { +): JsonObject { const payload = { ...record.payload }; for (const key of record.sharedFields) { const value = owner[key]; @@ -202,11 +215,18 @@ export function restoreHistoryPayload( } payload.item = item; } - return JSON.stringify(payload); + return payload; +} + +export function restoreHistoryPayload( + record: HistoryRecord, + owner: JsonObject, +): string { + return JSON.stringify(restoreHistoryPayloadObject(record, owner)); } export function decodeCompletedItemHistory(data: string) { - const value = jsonValueSchema.parse(JSON.parse(data)); + const value = parseHistoryJson(data); if ( !Array.isArray(value) || value.length !== 4 || diff --git a/packages/db/src/data/completed-item-history.ts b/packages/db/src/data/completed-item-history.ts index ae29f46dd3a..1c6ec59a915 100644 --- a/packages/db/src/data/completed-item-history.ts +++ b/packages/db/src/data/completed-item-history.ts @@ -1,27 +1,62 @@ +import type { JsonObject } from "@bb/domain"; import { and, inArray, isNotNull, sql } from "drizzle-orm"; import type { DbQueryConnection } from "../connection.js"; import { events } from "../schema.js"; import { decodeCompletedItemHistory, parseHistoryPayload, - restoreHistoryPayload, + restoreHistoryPayloadObject, } from "../completed-item-history.js"; import type { StoredEventRow } from "./events.js"; -export function expandSelectedCompletedItemRows( +export type ProjectionStoredEventRow = StoredEventRow & { + parsedData?: JsonObject; +}; + +function reconstructCompletedItemHistory(metadata: string, ownerData: string) { + const history = decodeCompletedItemHistory(metadata); + const payload = parseHistoryPayload(ownerData); + return { + sequence: history.sequence, + createdAt: history.createdAt, + payload, + records: history.records.map((record) => { + const data = restoreHistoryPayloadObject(record, payload); + return { + id: record.id, + sequence: record.sequence, + createdAt: record.createdAt, + type: record.type, + itemKind: record.itemKind, + data: JSON.stringify(data), + payload: data, + }; + }), + }; +} + +function expandSelectedCompletedItemRowsInternal( db: DbQueryConnection, rows: readonly StoredEventRow[], - throughSequence = Number.MAX_SAFE_INTEGER, -): StoredEventRow[] { + throughSequence: number, + includeParsedData: boolean, +): ProjectionStoredEventRow[] { const ids = [ ...new Set( - rows.filter((row) => row.type === "item/completed").map((row) => row.id), + rows + .filter( + (row) => + row.type === "item/completed" && + row.completedItemHistory === undefined, + ) + .map((row) => row.id), ), ]; - const histories = new Map< - string, - ReturnType - >(); + const histories = new Map(); + for (const row of rows) { + if (row.type === "item/completed" && row.completedItemHistory != null) + histories.set(row.id, row.completedItemHistory); + } if (ids.length > 0) { const selected = db .select({ id: events.id, history: events.completedItemHistory }) @@ -37,29 +72,25 @@ export function expandSelectedCompletedItemRows( ) .all(); for (const row of selected) { - if (row.history !== null) - histories.set(row.id, decodeCompletedItemHistory(row.history)); + if (row.history !== null) histories.set(row.id, row.history); } } - const expanded: StoredEventRow[] = []; - for (const row of rows) { - const history = histories.get(row.id); - if (!history) { + const expanded: ProjectionStoredEventRow[] = []; + for (const selectedRow of rows) { + const { completedItemHistory: _history, ...row } = selectedRow; + const metadata = histories.get(row.id); + if (metadata === undefined) { if (row.sequence <= throughSequence) expanded.push(row); continue; } + const history = reconstructCompletedItemHistory(metadata, row.data); if (history.sequence <= throughSequence) expanded.push({ ...row, sequence: history.sequence, createdAt: history.createdAt, + ...(includeParsedData ? { parsedData: history.payload } : {}), }); - const payload = history.records.some( - (record) => - record.sharedFields.length > 0 || record.sharedItemFields.length > 0, - ) - ? parseHistoryPayload(row.data) - : {}; for (const record of history.records) { if (record.sequence <= throughSequence) expanded.push({ @@ -69,9 +100,36 @@ export function expandSelectedCompletedItemRows( createdAt: record.createdAt, type: record.type, itemKind: record.itemKind, - data: restoreHistoryPayload(record, payload), + data: record.data, + ...(includeParsedData ? { parsedData: record.payload } : {}), }); } } return expanded.sort((a, b) => a.sequence - b.sequence); } + +export function expandSelectedCompletedItemRows( + db: DbQueryConnection, + rows: readonly StoredEventRow[], + throughSequence = Number.MAX_SAFE_INTEGER, +): StoredEventRow[] { + return expandSelectedCompletedItemRowsInternal( + db, + rows, + throughSequence, + false, + ); +} + +export function expandSelectedCompletedItemRowsForProjection( + db: DbQueryConnection, + rows: readonly StoredEventRow[], + throughSequence = Number.MAX_SAFE_INTEGER, +): ProjectionStoredEventRow[] { + return expandSelectedCompletedItemRowsInternal( + db, + rows, + throughSequence, + true, + ); +} diff --git a/packages/db/src/data/events.ts b/packages/db/src/data/events.ts index 74fa3049465..1e4d1de6825 100644 --- a/packages/db/src/data/events.ts +++ b/packages/db/src/data/events.ts @@ -1145,7 +1145,7 @@ const storedEventRowFields = { export type StoredEventRow = Pick< typeof events.$inferSelect, keyof typeof storedEventRowFields ->; +> & { completedItemHistory?: string | null }; export type InlineOutputCharLimit = number | null; @@ -2876,15 +2876,29 @@ export function findTimelineWindowBudgetFloorSequence( conditions.push(lt(events.sequence, args.beforeSequence)); } - const row = db - .select({ sequence: events.sequence }) + const query = db + .select({ + sequence: events.sequence, + count: + sql`1 + coalesce(json_array_length(${events.completedItemHistory}, '$[3][1]'), 0)`.as( + "count", + ), + }) .from(events) .where(and(...conditions)) .orderBy(desc(events.sequence)) - .limit(1) - .offset(args.eventBudget) - .get(); - return row?.sequence; + .limit(args.eventBudget + 1) + .toSQL(); + const statement = db.$client.prepare< + unknown[], + { sequence: number; count: number } + >(query.sql); + let count = 0; + for (const row of statement.iterate(...query.params)) { + count += row.count; + if (count > args.eventBudget) return row.sequence; + } + return undefined; } export function listTimelineInterruptionRows( @@ -3103,7 +3117,10 @@ export function findStoredTimelineWindowByteBudgetFloor( const query = db .select({ createdAt: events.createdAt, - dataBytes: sql`length(CAST(${data} AS BLOB)) + COALESCE(length(CAST(${events.completedItemHistory} AS BLOB)), 0)`.as("data_bytes"), + dataBytes: + sql`length(CAST(${data} AS BLOB)) + COALESCE(length(CAST(${events.completedItemHistory} AS BLOB)), 0)`.as( + "data_bytes", + ), sequence: events.sequence, turnId: events.turnId, }) @@ -3178,7 +3195,12 @@ export function listStoredTimelineTurnEventRows( fixedVariableCount: 32, queryBatch: (turnIds) => db - .select(storedEventRowSqlFields(args.maxInlineOutputChars)) + .select({ + ...storedEventRowSqlFields(args.maxInlineOutputChars), + completedItemHistory: sql< + string | null + >`${events.completedItemHistory}`, + }) .from( sql`${events} INDEXED BY events_thread_turn_type_item_sequence_idx`, ) @@ -3220,9 +3242,10 @@ export function listStoredTimelineThreadWindowEventRows( args: ListStoredTimelineWindowEventRowsArgs, ): StoredEventRow[] { return db - .select( - storedEventRowFieldsWithInlineOutputLimit(args.maxInlineOutputChars), - ) + .select({ + ...storedEventRowFieldsWithInlineOutputLimit(args.maxInlineOutputChars), + completedItemHistory: events.completedItemHistory, + }) .from(events) .where(and(...storedTimelineWindowConditions(args), isNull(events.turnId))) .orderBy(events.sequence) @@ -3234,9 +3257,10 @@ export function listStoredTimelineWindowEventRows( args: ListStoredTimelineWindowEventRowsArgs, ): StoredEventRow[] { return db - .select( - storedEventRowFieldsWithInlineOutputLimit(args.maxInlineOutputChars), - ) + .select({ + ...storedEventRowFieldsWithInlineOutputLimit(args.maxInlineOutputChars), + completedItemHistory: events.completedItemHistory, + }) .from(events) .where(and(...storedTimelineWindowConditions(args))) .orderBy(events.sequence) diff --git a/packages/db/src/index.ts b/packages/db/src/index.ts index dd561ab61d7..b9d7dfb445f 100644 --- a/packages/db/src/index.ts +++ b/packages/db/src/index.ts @@ -27,4 +27,8 @@ export type { DbNotifier } from "./notifier.js"; export * from "./data/index.js"; -export { expandSelectedCompletedItemRows } from "./data/completed-item-history.js"; +export { + expandSelectedCompletedItemRows, + expandSelectedCompletedItemRowsForProjection, + type ProjectionStoredEventRow, +} from "./data/completed-item-history.js"; diff --git a/packages/db/test/completed-item-history.test.ts b/packages/db/test/completed-item-history.test.ts index 66b1394ce50..24a1a145f0a 100644 --- a/packages/db/test/completed-item-history.test.ts +++ b/packages/db/test/completed-item-history.test.ts @@ -2,6 +2,7 @@ import { describe, expect, it } from "vitest"; import { decodeCompletedItemHistory, decodeHistory, + parseHistoryPayload, } from "../src/completed-item-history.js"; const record = [ @@ -25,6 +26,35 @@ describe("completed item history decoding", () => { }); }); + it("rejects non-finite numbers nested inside parsed JSON", () => { + expect(() => parseHistoryPayload('{"nested":[{"value":1e400}]}')).toThrow( + "Non-finite", + ); + const metadata = JSON.stringify([ + 1, + 3, + 300, + [ + 1, + [ + [ + "start", + 1, + 100, + "item/started", + "agentMessage", + [{ nested: [0] }, [], []], + ], + ], + ], + ]); + expect(() => + decodeCompletedItemHistory( + metadata.replace('"nested":[0]', '"nested":[1e400]'), + ), + ).toThrow("Non-finite"); + }); + it.each([ [2, [record]], [1, []], diff --git a/packages/db/test/data/completed-item-compaction.test.ts b/packages/db/test/data/completed-item-compaction.test.ts index 808ac40efee..8b31e2ea1ca 100644 --- a/packages/db/test/data/completed-item-compaction.test.ts +++ b/packages/db/test/data/completed-item-compaction.test.ts @@ -1,5 +1,5 @@ -import { describe, expect, it } from "vitest"; -import { eq } from "drizzle-orm"; +import { describe, expect, it, vi } from "vitest"; +import { eq, sql } from "drizzle-orm"; import { events, retainedEventOutputs, @@ -11,9 +11,14 @@ import { createProject } from "../../src/data/projects.js"; import { createThread } from "../../src/data/threads.js"; import { advanceThreadPruning } from "../../src/data/thread-pruning.js"; import { advanceCompletedItemCompaction } from "../../src/data/completed-item-compaction.js"; -import { expandSelectedCompletedItemRows } from "../../src/data/completed-item-history.js"; +import { + expandSelectedCompletedItemRows, + expandSelectedCompletedItemRowsForProjection, +} from "../../src/data/completed-item-history.js"; import { listStoredEventRows, + listStoredTimelineTurnEventRows, + findTimelineWindowBudgetFloorSequence, getHighWaterMarks, getLatestThreadOutputEventRow, deleteThreadEventSuffixInTransaction, @@ -104,6 +109,118 @@ describe("completed items at first lifecycle position", () => { } }); + it("loads history with the selected timeline rows and keeps raw rows private", () => { + const f = setup(); + try { + const before = f.rows(); + f.advance(); + const selected = listStoredTimelineTurnEventRows(f.db, { + threadId: f.thread.id, + turnIds: ["turn"], + sequenceStart: 0, + maxInlineOutputChars: 8000, + }); + const prepare = vi.spyOn(f.db.$client, "prepare"); + try { + const projected = expandSelectedCompletedItemRowsForProjection( + f.db, + selected, + ); + const raw = expandSelectedCompletedItemRows(f.db, selected); + expect(prepare).not.toHaveBeenCalled(); + expect(normalized(raw)).toEqual(normalized(before)); + for (const row of projected) { + if (row.parsedData !== undefined) + expect(row.parsedData).toEqual(JSON.parse(row.data)); + } + expect(projected.some((row) => row.parsedData !== undefined)).toBe( + true, + ); + } finally { + prepare.mockRestore(); + } + } finally { + f.db.$client.close(); + } + }); + + it("charges the timeline budget for records inside a combined item", () => { + const f = setup(); + try { + const args = { + threadId: f.thread.id, + sequenceStart: 0, + excludedTypes: [], + eventBudget: 4, + }; + expect(findTimelineWindowBudgetFloorSequence(f.db, args)).toBe(1); + f.advance(); + expect(findTimelineWindowBudgetFloorSequence(f.db, args)).toBe(1); + expect( + findTimelineWindowBudgetFloorSequence(f.db, { + ...args, + eventBudget: 0, + }), + ).toBe(6); + expect( + findTimelineWindowBudgetFloorSequence(f.db, { + ...args, + eventBudget: 5, + }), + ).toBeUndefined(); + } finally { + f.db.$client.close(); + } + }); + + it("reads changed owner payloads and metadata without reusing stale history", () => { + const f = setup(); + try { + f.advance(); + expandSelectedCompletedItemRows(f.db, f.rows()); + f.db + .update(events) + .set({ + data: JSON.stringify({ + item: { id: "changed", type: "agentMessage", text: "new output" }, + }), + }) + .where(eq(events.id, "event-5")) + .run(); + const changedOwner = expandSelectedCompletedItemRows(f.db, f.rows()); + expect( + JSON.parse(changedOwner.find((row) => row.id === "event-2")!.data).item + .id, + ).toBe("changed"); + f.db + .update(events) + .set({ + completedItemHistory: sql`json_set(${events.completedItemHistory}, '$[3][1][0][5][0].item.text', 'changed start')`, + }) + .where(eq(events.id, "event-5")) + .run(); + const changedHistory = expandSelectedCompletedItemRows(f.db, f.rows()); + expect( + JSON.parse(changedHistory.find((row) => row.id === "event-2")!.data) + .item.text, + ).toBe("changed start"); + f.db + .update(events) + .set({ completedItemHistory: "broken" }) + .where(eq(events.id, "event-5")) + .run(); + expect(() => expandSelectedCompletedItemRows(f.db, f.rows())).toThrow(); + f.db + .update(events) + .set({ completedItemHistory: null }) + .where(eq(events.id, "event-5")) + .run(); + expect(expandSelectedCompletedItemRows(f.db, f.rows())).toEqual(f.rows()); + } finally { + f.db.$client.close(); + } + }); + it("preserves completion ownership, timestamps, lossless history and highwater through atomic progress", () => { const f = setup(); try { From da46d997b46ccecdf6a205126227c88b7542967e Mon Sep 17 00:00:00 2001 From: Michael Yong Date: Thu, 17 Sep 2026 16:15:41 -0700 Subject: [PATCH 5/7] Regenerate compaction migration after current main --- packages/db/drizzle/0127_plain_cammi.sql | 1 + packages/db/drizzle/meta/0127_snapshot.json | 5008 +++++++++++++++++++ packages/db/drizzle/meta/_journal.json | 7 + packages/db/test/migrate.test.ts | 5 +- 4 files changed, 5017 insertions(+), 4 deletions(-) create mode 100644 packages/db/drizzle/0127_plain_cammi.sql create mode 100644 packages/db/drizzle/meta/0127_snapshot.json diff --git a/packages/db/drizzle/0127_plain_cammi.sql b/packages/db/drizzle/0127_plain_cammi.sql new file mode 100644 index 00000000000..04d4ca8405f --- /dev/null +++ b/packages/db/drizzle/0127_plain_cammi.sql @@ -0,0 +1 @@ +ALTER TABLE `events` ADD `completed_item_history` text; \ No newline at end of file diff --git a/packages/db/drizzle/meta/0127_snapshot.json b/packages/db/drizzle/meta/0127_snapshot.json new file mode 100644 index 00000000000..d58783f160f --- /dev/null +++ b/packages/db/drizzle/meta/0127_snapshot.json @@ -0,0 +1,5008 @@ +{ + "version": "6", + "dialect": "sqlite", + "id": "7d1de893-ff6d-445e-bcd2-6fbbe311d502", + "prevId": "48979c46-bc2b-411d-a6cc-18dfcb7cd457", + "tables": { + "app_settings": { + "name": "app_settings", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "caffeinate": { + "name": "caffeinate", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "show_keyboard_hints": { + "name": "show_keyboard_hints", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": true + }, + "steer_active_thread_on_enter": { + "name": "steer_active_thread_on_enter", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "show_unhandled_provider_events": { + "name": "show_unhandled_provider_events", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "codex_memory_enabled": { + "name": "codex_memory_enabled", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": true + }, + "claude_code_memory_enabled": { + "name": "claude_code_memory_enabled", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": true + }, + "codex_subagents_disabled": { + "name": "codex_subagents_disabled", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "claude_code_subagents_disabled": { + "name": "claude_code_subagents_disabled", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "claude_code_workflows_disabled": { + "name": "claude_code_workflows_disabled", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "keybinding_overrides": { + "name": "keybinding_overrides", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'[]'" + }, + "onboarding_completed_at": { + "name": "onboarding_completed_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "app_settings_values": { + "name": "app_settings_values", + "columns": { + "key": { + "name": "key", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "value": { + "name": "value", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "app_theme": { + "name": "app_theme", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "theme_id": { + "name": "theme_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "favicon_color": { + "name": "favicon_color", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'default'" + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "apikey": { + "name": "apikey", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "start": { + "name": "start", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "prefix": { + "name": "prefix", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "key": { + "name": "key", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "referenceId": { + "name": "referenceId", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "refillInterval": { + "name": "refillInterval", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "refillAmount": { + "name": "refillAmount", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "lastRefillAt": { + "name": "lastRefillAt", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "enabled": { + "name": "enabled", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "rateLimitEnabled": { + "name": "rateLimitEnabled", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "rateLimitTimeWindow": { + "name": "rateLimitTimeWindow", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "rateLimitMax": { + "name": "rateLimitMax", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "requestCount": { + "name": "requestCount", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "remaining": { + "name": "remaining", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "lastRequest": { + "name": "lastRequest", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "expiresAt": { + "name": "expiresAt", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updatedAt": { + "name": "updatedAt", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "permissions": { + "name": "permissions", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "metadata": { + "name": "metadata", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "configId": { + "name": "configId", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "apikey_key_unique": { + "name": "apikey_key_unique", + "columns": [ + "key" + ], + "isUnique": true + }, + "apikey_reference_id_idx": { + "name": "apikey_reference_id_idx", + "columns": [ + "referenceId" + ], + "isUnique": false + }, + "apikey_config_id_idx": { + "name": "apikey_config_id_idx", + "columns": [ + "configId" + ], + "isUnique": false + } + }, + "foreignKeys": { + "apikey_referenceId_user_id_fk": { + "name": "apikey_referenceId_user_id_fk", + "tableFrom": "apikey", + "tableTo": "user", + "columnsFrom": [ + "referenceId" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "user": { + "name": "user", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "email": { + "name": "email", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "emailVerified": { + "name": "emailVerified", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "image": { + "name": "image", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updatedAt": { + "name": "updatedAt", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "user_email_unique": { + "name": "user_email_unique", + "columns": [ + "email" + ], + "isUnique": true + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "environment_hook_operations": { + "name": "environment_hook_operations", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "operation_id": { + "name": "operation_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "host_id": { + "name": "host_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "path": { + "name": "path", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "started_at": { + "name": "started_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "finished_at": { + "name": "finished_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "error": { + "name": "error", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "environment_variables": { + "name": "environment_variables", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": true + }, + "project_id": { + "name": "project_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "ciphertext": { + "name": "ciphertext", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "encryption_version": { + "name": "encryption_version", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "note": { + "name": "note", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "environment_variables_global_name": { + "name": "environment_variables_global_name", + "columns": [ + "name" + ], + "isUnique": true, + "where": "\"environment_variables\".\"project_id\" IS NULL" + }, + "environment_variables_project_name": { + "name": "environment_variables_project_name", + "columns": [ + "project_id", + "name" + ], + "isUnique": true, + "where": "\"environment_variables\".\"project_id\" IS NOT NULL" + } + }, + "foreignKeys": { + "environment_variables_project_id_projects_id_fk": { + "name": "environment_variables_project_id_projects_id_fk", + "tableFrom": "environment_variables", + "tableTo": "projects", + "columnsFrom": [ + "project_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "environments": { + "name": "environments", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "project_id": { + "name": "project_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "host_id": { + "name": "host_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "path": { + "name": "path", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "is_git_repo": { + "name": "is_git_repo", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "is_worktree": { + "name": "is_worktree", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "branch_name": { + "name": "branch_name", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "base_branch": { + "name": "base_branch", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "default_branch": { + "name": "default_branch", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "merge_base_branch": { + "name": "merge_base_branch", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "environment_provider_id": { + "name": "environment_provider_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "environment_provider_plugin_id": { + "name": "environment_provider_plugin_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "provider_owns_path": { + "name": "provider_owns_path", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "environment_provider_selection": { + "name": "environment_provider_selection", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "environment_provider_instance_key": { + "name": "environment_provider_instance_key", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "retire_at": { + "name": "retire_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "teardown_attempt": { + "name": "teardown_attempt", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "teardown_status": { + "name": "teardown_status", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "teardown_message": { + "name": "teardown_message", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "resource": { + "name": "resource", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "owner_thread_id": { + "name": "owner_thread_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "attempt": { + "name": "attempt", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "status_message": { + "name": "status_message", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "pending_log": { + "name": "pending_log", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "''" + }, + "claim_path": { + "name": "claim_path", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'provisioning'" + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "environments_project_host_path_idx": { + "name": "environments_project_host_path_idx", + "columns": [ + "project_id", + "host_id", + "path" + ], + "isUnique": true + }, + "environments_host_path_lookup_idx": { + "name": "environments_host_path_lookup_idx", + "columns": [ + "host_id", + "path" + ], + "isUnique": false + }, + "environments_owner_thread_idx": { + "name": "environments_owner_thread_idx", + "columns": [ + "owner_thread_id" + ], + "isUnique": true + }, + "environments_claim_idx": { + "name": "environments_claim_idx", + "columns": [ + "host_id", + "claim_path" + ], + "isUnique": false + }, + "environments_project_idx": { + "name": "environments_project_idx", + "columns": [ + "project_id" + ], + "isUnique": false + }, + "environments_status_idx": { + "name": "environments_status_idx", + "columns": [ + "status" + ], + "isUnique": false + }, + "environments_provider_instance_idx": { + "name": "environments_provider_instance_idx", + "columns": [ + "environment_provider_id", + "environment_provider_instance_key" + ], + "isUnique": false + } + }, + "foreignKeys": { + "environments_project_id_projects_id_fk": { + "name": "environments_project_id_projects_id_fk", + "tableFrom": "environments", + "tableTo": "projects", + "columnsFrom": [ + "project_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "environments_host_id_hosts_id_fk": { + "name": "environments_host_id_hosts_id_fk", + "tableFrom": "environments", + "tableTo": "hosts", + "columnsFrom": [ + "host_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "events": { + "name": "events", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "thread_id": { + "name": "thread_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "environment_id": { + "name": "environment_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "scope_kind": { + "name": "scope_kind", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "turn_id": { + "name": "turn_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "provider_thread_id": { + "name": "provider_thread_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "sequence": { + "name": "sequence", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "item_id": { + "name": "item_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "item_kind": { + "name": "item_kind", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "parent_tool_call_id": { + "name": "parent_tool_call_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "data": { + "name": "data", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'{}'" + }, + "completed_item_history": { + "name": "completed_item_history", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "events_thread_sequence_idx": { + "name": "events_thread_sequence_idx", + "columns": [ + "thread_id", + "sequence" + ], + "isUnique": true + }, + "events_delegating_item_lookup_idx": { + "name": "events_delegating_item_lookup_idx", + "columns": [ + "thread_id", + "item_id", + "sequence", + "item_kind" + ], + "isUnique": false, + "where": "\"events\".\"item_kind\" IN ('toolCall', 'delegation')" + }, + "events_plan_steps_thread_sequence_idx": { + "name": "events_plan_steps_thread_sequence_idx", + "columns": [ + "thread_id", + "sequence" + ], + "isUnique": false, + "where": "(\"events\".\"item_kind\" = 'planSteps' AND \"events\".\"type\" = 'item/completed') OR \"events\".\"type\" = 'turn/plan/updated'" + }, + "events_parent_tool_call_thread_parent_sequence_idx": { + "name": "events_parent_tool_call_thread_parent_sequence_idx", + "columns": [ + "thread_id", + "parent_tool_call_id", + "sequence" + ], + "isUnique": false, + "where": "\"events\".\"parent_tool_call_id\" IS NOT NULL" + }, + "events_thread_type_item_kind_sequence_idx": { + "name": "events_thread_type_item_kind_sequence_idx", + "columns": [ + "thread_id", + "type", + "item_kind", + "sequence" + ], + "isUnique": false + }, + "events_background_task_thread_type_item_sequence_idx": { + "name": "events_background_task_thread_type_item_sequence_idx", + "columns": [ + "thread_id", + "type", + "item_id", + "sequence" + ], + "isUnique": false, + "where": "\"events\".\"item_kind\" = 'backgroundTask'" + }, + "events_thread_type_sequence_idx": { + "name": "events_thread_type_sequence_idx", + "columns": [ + "thread_id", + "type", + "sequence" + ], + "isUnique": false + }, + "events_thread_turn_type_item_sequence_idx": { + "name": "events_thread_turn_type_item_sequence_idx", + "columns": [ + "thread_id", + "turn_id", + "type", + "item_id", + "sequence" + ], + "isUnique": false + }, + "events_item_lifecycle_thread_item_sequence_idx": { + "name": "events_item_lifecycle_thread_item_sequence_idx", + "columns": [ + "thread_id", + "item_id", + "sequence" + ], + "isUnique": false, + "where": "\"events\".\"type\" IN ('item/started', 'item/completed', 'item/backgroundTask/completed')" + }, + "events_environment_idx": { + "name": "events_environment_idx", + "columns": [ + "environment_id" + ], + "isUnique": false + }, + "events_provider_identity_idx": { + "name": "events_provider_identity_idx", + "columns": [ + "provider_thread_id", + "created_at" + ], + "isUnique": false, + "where": "\"events\".\"type\" = 'thread/identity'" + }, + "events_completed_item_truncation_idx": { + "name": "events_completed_item_truncation_idx", + "columns": [ + "item_kind", + "created_at", + "id" + ], + "isUnique": false, + "where": "\"events\".\"type\" = 'item/completed'" + }, + "events_thread_state_thread_sequence_idx": { + "name": "events_thread_state_thread_sequence_idx", + "columns": [ + "thread_id", + "sequence" + ], + "isUnique": false, + "where": "\"events\".\"type\" IN ('thread/goal/updated', 'thread/goal/cleared', 'thread/extensionState/updated')" + } + }, + "foreignKeys": { + "events_thread_id_threads_id_fk": { + "name": "events_thread_id_threads_id_fk", + "tableFrom": "events", + "tableTo": "threads", + "columnsFrom": [ + "thread_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "events_environment_id_environments_id_fk": { + "name": "events_environment_id_environments_id_fk", + "tableFrom": "events", + "tableTo": "environments", + "columnsFrom": [ + "environment_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": { + "events_scope_shape_check": { + "name": "events_scope_shape_check", + "value": "(\n (\"events\".\"scope_kind\" = 'turn' AND \"events\".\"turn_id\" IS NOT NULL)\n OR\n (\"events\".\"scope_kind\" = 'thread' AND \"events\".\"turn_id\" IS NULL)\n )" + } + } + }, + "host_daemon_sessions": { + "name": "host_daemon_sessions", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "host_id": { + "name": "host_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "instance_id": { + "name": "instance_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "host_name": { + "name": "host_name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "data_dir": { + "name": "data_dir", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "protocol_version": { + "name": "protocol_version", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "heartbeat_interval_ms": { + "name": "heartbeat_interval_ms", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "lease_timeout_ms": { + "name": "lease_timeout_ms", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "lease_expires_at": { + "name": "lease_expires_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "closed_at": { + "name": "closed_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "close_reason": { + "name": "close_reason", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "host_daemon_sessions_host_status_idx": { + "name": "host_daemon_sessions_host_status_idx", + "columns": [ + "host_id", + "status" + ], + "isUnique": false + }, + "host_daemon_sessions_host_latest_idx": { + "name": "host_daemon_sessions_host_latest_idx", + "columns": [ + "host_id", + "updated_at", + "created_at", + "id" + ], + "isUnique": false + }, + "host_daemon_sessions_closed_prune_idx": { + "name": "host_daemon_sessions_closed_prune_idx", + "columns": [ + "status", + "closed_at", + "id" + ], + "isUnique": false + } + }, + "foreignKeys": { + "host_daemon_sessions_host_id_hosts_id_fk": { + "name": "host_daemon_sessions_host_id_hosts_id_fk", + "tableFrom": "host_daemon_sessions", + "tableTo": "hosts", + "columnsFrom": [ + "host_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "hosts": { + "name": "hosts", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "connect_machine_id": { + "name": "connect_machine_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "machine_provider_id": { + "name": "machine_provider_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "launch_key": { + "name": "launch_key", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "machine_inputs": { + "name": "machine_inputs", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "machine_attempt": { + "name": "machine_attempt", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "pending_log": { + "name": "pending_log", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "''" + }, + "machine_operation_id": { + "name": "machine_operation_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "server_access_provider_id": { + "name": "server_access_provider_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "server_access_grant_id": { + "name": "server_access_grant_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "resource": { + "name": "resource", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "phase": { + "name": "phase", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'active'" + }, + "suspended_at": { + "name": "suspended_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "status_message": { + "name": "status_message", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "suspend_retry_at": { + "name": "suspend_retry_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "remove_retry_at": { + "name": "remove_retry_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "teardown_attempt": { + "name": "teardown_attempt", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "teardown_status": { + "name": "teardown_status", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "max_permission_mode": { + "name": "max_permission_mode", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'full'" + }, + "destroyed_at": { + "name": "destroyed_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "last_seen_at": { + "name": "last_seen_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "last_rejected_protocol_version": { + "name": "last_rejected_protocol_version", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "hosts_last_seen_idx": { + "name": "hosts_last_seen_idx", + "columns": [ + "last_seen_at" + ], + "isUnique": false + }, + "hosts_live_launch_key_idx": { + "name": "hosts_live_launch_key_idx", + "columns": [ + "launch_key" + ], + "isUnique": true, + "where": "\"hosts\".\"destroyed_at\" is null" + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "plugins": { + "name": "plugins", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "source": { + "name": "source", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "provenance": { + "name": "provenance", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'direct'" + }, + "catalog_entry_id": { + "name": "catalog_entry_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "catalog_marketplace_name": { + "name": "catalog_marketplace_name", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "source_kind": { + "name": "source_kind", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'path'" + }, + "source_path": { + "name": "source_path", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "source_builtin_name": { + "name": "source_builtin_name", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "source_npm_package": { + "name": "source_npm_package", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "source_npm_registry": { + "name": "source_npm_registry", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "source_npm_requested_spec": { + "name": "source_npm_requested_spec", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "source_npm_spec_kind": { + "name": "source_npm_spec_kind", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "source_git_url": { + "name": "source_git_url", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "source_git_subdirectory": { + "name": "source_git_subdirectory", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "source_git_requested_ref": { + "name": "source_git_requested_ref", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "source_git_ref_kind": { + "name": "source_git_ref_kind", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "source_git_range": { + "name": "source_git_range", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "source_git_tag_prefix": { + "name": "source_git_tag_prefix", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "source_git_resolved_tag": { + "name": "source_git_resolved_tag", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "npm_resolved_version": { + "name": "npm_resolved_version", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "npm_integrity": { + "name": "npm_integrity", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "git_resolved_commit": { + "name": "git_resolved_commit", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "last_update_check_at": { + "name": "last_update_check_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "available_compatible_version": { + "name": "available_compatible_version", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "newest_incompatible_version": { + "name": "newest_incompatible_version", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "update_status_detail": { + "name": "update_status_detail", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "last_failure_version": { + "name": "last_failure_version", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "last_failure_at": { + "name": "last_failure_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "last_failure_detail": { + "name": "last_failure_detail", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "active_artifact_id": { + "name": "active_artifact_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "normalization_version": { + "name": "normalization_version", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "root_dir": { + "name": "root_dir", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "version": { + "name": "version", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "enabled": { + "name": "enabled", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": true + }, + "removed_at": { + "name": "removed_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "installed_at": { + "name": "installed_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": { + "plugins_active_artifact_id_plugin_artifacts_id_fk": { + "name": "plugins_active_artifact_id_plugin_artifacts_id_fk", + "tableFrom": "plugins", + "tableTo": "plugin_artifacts", + "columnsFrom": [ + "active_artifact_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "maintenance_scan_cursors": { + "name": "maintenance_scan_cursors", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "policy": { + "name": "policy", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "version": { + "name": "version", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "item_kind": { + "name": "item_kind", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "output_path": { + "name": "output_path", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "last_created_at": { + "name": "last_created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "last_event_id": { + "name": "last_event_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "''" + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "maintenance_scan_cursors_path_idx": { + "name": "maintenance_scan_cursors_path_idx", + "columns": [ + "policy", + "version", + "item_kind", + "output_path" + ], + "isUnique": true + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "pending_interactions": { + "name": "pending_interactions", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "thread_id": { + "name": "thread_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "origin_kind": { + "name": "origin_kind", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'provider'" + }, + "turn_id": { + "name": "turn_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "provider_id": { + "name": "provider_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "provider_thread_id": { + "name": "provider_thread_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "provider_request_id": { + "name": "provider_request_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "plugin_id": { + "name": "plugin_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "renderer_id": { + "name": "renderer_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "payload": { + "name": "payload", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "resolution": { + "name": "resolution", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "status_reason": { + "name": "status_reason", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "expires_at": { + "name": "expires_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "resolved_at": { + "name": "resolved_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "pending_interactions_provider_request_idx": { + "name": "pending_interactions_provider_request_idx", + "columns": [ + "provider_id", + "provider_thread_id", + "provider_request_id" + ], + "isUnique": true + }, + "pending_interactions_thread_created_idx": { + "name": "pending_interactions_thread_created_idx", + "columns": [ + "thread_id", + "created_at" + ], + "isUnique": false + }, + "pending_interactions_thread_status_created_idx": { + "name": "pending_interactions_thread_status_created_idx", + "columns": [ + "thread_id", + "status", + "created_at" + ], + "isUnique": false + }, + "pending_interactions_status_created_idx": { + "name": "pending_interactions_status_created_idx", + "columns": [ + "status", + "created_at" + ], + "isUnique": false + }, + "pending_interactions_plugin_status_created_idx": { + "name": "pending_interactions_plugin_status_created_idx", + "columns": [ + "plugin_id", + "status", + "created_at" + ], + "isUnique": false + } + }, + "foreignKeys": { + "pending_interactions_thread_id_threads_id_fk": { + "name": "pending_interactions_thread_id_threads_id_fk", + "tableFrom": "pending_interactions", + "tableTo": "threads", + "columnsFrom": [ + "thread_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "plugin_artifacts": { + "name": "plugin_artifacts", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "plugin_id": { + "name": "plugin_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "source_kind": { + "name": "source_kind", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "npm_resolved_version": { + "name": "npm_resolved_version", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "git_resolved_commit": { + "name": "git_resolved_commit", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "git_checkout_root": { + "name": "git_checkout_root", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "path": { + "name": "path", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "integrity": { + "name": "integrity", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "content_hash": { + "name": "content_hash", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "validation_result": { + "name": "validation_result", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "validated_at": { + "name": "validated_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "plugin_artifacts_plugin_idx": { + "name": "plugin_artifacts_plugin_idx", + "columns": [ + "plugin_id" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "plugin_kv": { + "name": "plugin_kv", + "columns": { + "plugin_id": { + "name": "plugin_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "key": { + "name": "key", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "value": { + "name": "value", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": { + "plugin_kv_plugin_id_key_pk": { + "columns": [ + "plugin_id", + "key" + ], + "name": "plugin_kv_plugin_id_key_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "plugin_marketplace_icons": { + "name": "plugin_marketplace_icons", + "columns": { + "marketplace_name": { + "name": "marketplace_name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "entry_id": { + "name": "entry_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "source_url": { + "name": "source_url", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "content_type": { + "name": "content_type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "etag": { + "name": "etag", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "content_hash": { + "name": "content_hash", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "bytes": { + "name": "bytes", + "type": "blob", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": { + "plugin_marketplace_icons_marketplace_name_entry_id_pk": { + "columns": [ + "marketplace_name", + "entry_id" + ], + "name": "plugin_marketplace_icons_marketplace_name_entry_id_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "plugin_marketplaces": { + "name": "plugin_marketplaces", + "columns": { + "name": { + "name": "name", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "source_kind": { + "name": "source_kind", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'https'" + }, + "manifest_url": { + "name": "manifest_url", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "source_git_ref": { + "name": "source_git_ref", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "source_git_commit": { + "name": "source_git_commit", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "manifest_json": { + "name": "manifest_json", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "stats_json": { + "name": "stats_json", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "etag": { + "name": "etag", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "last_modified": { + "name": "last_modified", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "last_successful_refresh_at": { + "name": "last_successful_refresh_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "last_attempted_refresh_at": { + "name": "last_attempted_refresh_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "last_error": { + "name": "last_error", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "plugin_schedules": { + "name": "plugin_schedules", + "columns": { + "plugin_id": { + "name": "plugin_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "cron": { + "name": "cron", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "next_run_at": { + "name": "next_run_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "last_run_at": { + "name": "last_run_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "last_status": { + "name": "last_status", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "last_error": { + "name": "last_error", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": { + "plugin_schedules_plugin_id_name_pk": { + "columns": [ + "plugin_id", + "name" + ], + "name": "plugin_schedules_plugin_id_name_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "plugin_settings": { + "name": "plugin_settings", + "columns": { + "plugin_id": { + "name": "plugin_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "key": { + "name": "key", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "value": { + "name": "value", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": { + "plugin_settings_plugin_id_key_pk": { + "columns": [ + "plugin_id", + "key" + ], + "name": "plugin_settings_plugin_id_key_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "plugin_state_snapshots": { + "name": "plugin_state_snapshots", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "plugin_id": { + "name": "plugin_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "from_artifact_id": { + "name": "from_artifact_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "to_artifact_id": { + "name": "to_artifact_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "snapshot_path": { + "name": "snapshot_path", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "database_path": { + "name": "database_path", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "state_path": { + "name": "state_path", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "secrets_path": { + "name": "secrets_path", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "registration_path": { + "name": "registration_path", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "rollback_candidate_version": { + "name": "rollback_candidate_version", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "rollback_source_fingerprint": { + "name": "rollback_source_fingerprint", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "rollback_bb_version": { + "name": "rollback_bb_version", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "rollback_sdk_version": { + "name": "rollback_sdk_version", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "rollback_detail": { + "name": "rollback_detail", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "retained_until": { + "name": "retained_until", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "plugin_state_snapshots_plugin_idx": { + "name": "plugin_state_snapshots_plugin_idx", + "columns": [ + "plugin_id" + ], + "isUnique": false + }, + "plugin_state_snapshots_retention_idx": { + "name": "plugin_state_snapshots_retention_idx", + "columns": [ + "retained_until" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "project_attachment_backfills": { + "name": "project_attachment_backfills", + "columns": { + "project_id": { + "name": "project_id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "phase": { + "name": "phase", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "thread_cursor": { + "name": "thread_cursor", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "input_cursor": { + "name": "input_cursor", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "input_id": { + "name": "input_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "input_sequence": { + "name": "input_sequence", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "attempted_at": { + "name": "attempted_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "error": { + "name": "error", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": { + "project_attachment_backfills_project_id_projects_id_fk": { + "name": "project_attachment_backfills_project_id_projects_id_fk", + "tableFrom": "project_attachment_backfills", + "tableTo": "projects", + "columnsFrom": [ + "project_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "project_attachment_threads": { + "name": "project_attachment_threads", + "columns": { + "attachment_id": { + "name": "attachment_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "thread_id": { + "name": "thread_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "project_attachment_threads_thread_idx": { + "name": "project_attachment_threads_thread_idx", + "columns": [ + "thread_id" + ], + "isUnique": false + } + }, + "foreignKeys": { + "project_attachment_threads_attachment_id_project_attachments_id_fk": { + "name": "project_attachment_threads_attachment_id_project_attachments_id_fk", + "tableFrom": "project_attachment_threads", + "tableTo": "project_attachments", + "columnsFrom": [ + "attachment_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "project_attachment_threads_thread_id_threads_id_fk": { + "name": "project_attachment_threads_thread_id_threads_id_fk", + "tableFrom": "project_attachment_threads", + "tableTo": "threads", + "columnsFrom": [ + "thread_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "project_attachment_threads_attachment_id_thread_id_pk": { + "columns": [ + "attachment_id", + "thread_id" + ], + "name": "project_attachment_threads_attachment_id_thread_id_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "project_attachments": { + "name": "project_attachments", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "project_id": { + "name": "project_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "stored_path": { + "name": "stored_path", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "original_name": { + "name": "original_name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "mime_type": { + "name": "mime_type", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "size_bytes": { + "name": "size_bytes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "ready_at": { + "name": "ready_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "deletion_claimed_at": { + "name": "deletion_claimed_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "project_attachments_project_path_idx": { + "name": "project_attachments_project_path_idx", + "columns": [ + "project_id", + "stored_path" + ], + "isUnique": true + }, + "project_attachments_project_created_idx": { + "name": "project_attachments_project_created_idx", + "columns": [ + "project_id", + "created_at" + ], + "isUnique": false + }, + "project_attachments_deletion_idx": { + "name": "project_attachments_deletion_idx", + "columns": [ + "project_id", + "deletion_claimed_at", + "id" + ], + "isUnique": false, + "where": "\"project_attachments\".\"deletion_claimed_at\" IS NOT NULL" + } + }, + "foreignKeys": { + "project_attachments_project_id_projects_id_fk": { + "name": "project_attachments_project_id_projects_id_fk", + "tableFrom": "project_attachments", + "tableTo": "projects", + "columnsFrom": [ + "project_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": { + "project_attachments_size_check": { + "name": "project_attachments_size_check", + "value": "\"project_attachments\".\"size_bytes\" >= 0" + } + } + }, + "project_execution_defaults": { + "name": "project_execution_defaults", + "columns": { + "project_id": { + "name": "project_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "provider_id": { + "name": "provider_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "model": { + "name": "model", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "service_tier": { + "name": "service_tier", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "reasoning_level": { + "name": "reasoning_level", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "permission_mode": { + "name": "permission_mode", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "project_execution_defaults_project_idx": { + "name": "project_execution_defaults_project_idx", + "columns": [ + "project_id" + ], + "isUnique": true + } + }, + "foreignKeys": { + "project_execution_defaults_project_id_projects_id_fk": { + "name": "project_execution_defaults_project_id_projects_id_fk", + "tableFrom": "project_execution_defaults", + "tableTo": "projects", + "columnsFrom": [ + "project_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "project_sources": { + "name": "project_sources", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "project_id": { + "name": "project_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "host_id": { + "name": "host_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "path": { + "name": "path", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "owns_path": { + "name": "owns_path", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "is_default": { + "name": "is_default", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "project_sources_project_idx": { + "name": "project_sources_project_idx", + "columns": [ + "project_id" + ], + "isUnique": false + }, + "project_sources_host_idx": { + "name": "project_sources_host_idx", + "columns": [ + "host_id" + ], + "isUnique": false + }, + "project_sources_project_host_idx": { + "name": "project_sources_project_host_idx", + "columns": [ + "project_id", + "host_id" + ], + "isUnique": true + } + }, + "foreignKeys": { + "project_sources_project_id_projects_id_fk": { + "name": "project_sources_project_id_projects_id_fk", + "tableFrom": "project_sources", + "tableTo": "projects", + "columnsFrom": [ + "project_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "project_sources_host_id_hosts_id_fk": { + "name": "project_sources_host_id_hosts_id_fk", + "tableFrom": "project_sources", + "tableTo": "hosts", + "columnsFrom": [ + "host_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": { + "project_sources_shape_check": { + "name": "project_sources_shape_check", + "value": "(\n \"project_sources\".\"type\" = 'local_path' AND \"project_sources\".\"host_id\" IS NOT NULL AND \"project_sources\".\"path\" IS NOT NULL\n )" + } + } + }, + "projects": { + "name": "projects", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'standard'" + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "git_remote_url": { + "name": "git_remote_url", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "sort_key": { + "name": "sort_key", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'V'" + }, + "deleted_at": { + "name": "deleted_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "projects_updated_idx": { + "name": "projects_updated_idx", + "columns": [ + "updated_at" + ], + "isUnique": false + }, + "projects_deleted_idx": { + "name": "projects_deleted_idx", + "columns": [ + "deleted_at" + ], + "isUnique": false + }, + "projects_sort_idx": { + "name": "projects_sort_idx", + "columns": [ + "sort_key", + "id" + ], + "isUnique": false + }, + "projects_personal_singleton_idx": { + "name": "projects_personal_singleton_idx", + "columns": [ + "kind" + ], + "isUnique": true, + "where": "\"projects\".\"kind\" = 'personal'" + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "prompt_history_entries": { + "name": "prompt_history_entries", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "project_id": { + "name": "project_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "thread_id": { + "name": "thread_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "scope": { + "name": "scope", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "request_sequence": { + "name": "request_sequence", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "input": { + "name": "input", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "prompt_history_entries_thread_request_idx": { + "name": "prompt_history_entries_thread_request_idx", + "columns": [ + "thread_id", + "request_sequence" + ], + "isUnique": true + }, + "prompt_history_entries_project_scope_created_idx": { + "name": "prompt_history_entries_project_scope_created_idx", + "columns": [ + "project_id", + "scope", + "created_at", + "request_sequence", + "id" + ], + "isUnique": false + }, + "prompt_history_entries_thread_scope_created_idx": { + "name": "prompt_history_entries_thread_scope_created_idx", + "columns": [ + "thread_id", + "scope", + "created_at", + "request_sequence", + "id" + ], + "isUnique": false + } + }, + "foreignKeys": { + "prompt_history_entries_project_id_projects_id_fk": { + "name": "prompt_history_entries_project_id_projects_id_fk", + "tableFrom": "prompt_history_entries", + "tableTo": "projects", + "columnsFrom": [ + "project_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "prompt_history_entries_thread_id_threads_id_fk": { + "name": "prompt_history_entries_thread_id_threads_id_fk", + "tableFrom": "prompt_history_entries", + "tableTo": "threads", + "columnsFrom": [ + "thread_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "provider_model_catalogs": { + "name": "provider_model_catalogs", + "columns": { + "host_id": { + "name": "host_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "provider_id": { + "name": "provider_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "scope_key": { + "name": "scope_key", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "fingerprint": { + "name": "fingerprint", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "models_json": { + "name": "models_json", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "selected_only_models_json": { + "name": "selected_only_models_json", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "fetched_at": { + "name": "fetched_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": { + "provider_model_catalogs_host_id_hosts_id_fk": { + "name": "provider_model_catalogs_host_id_hosts_id_fk", + "tableFrom": "provider_model_catalogs", + "tableTo": "hosts", + "columnsFrom": [ + "host_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "provider_model_catalogs_host_id_provider_id_scope_key_pk": { + "columns": [ + "host_id", + "provider_id", + "scope_key" + ], + "name": "provider_model_catalogs_host_id_provider_id_scope_key_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "queued_thread_messages": { + "name": "queued_thread_messages", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "system_notice": { + "name": "system_notice", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "thread_id": { + "name": "thread_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "content": { + "name": "content", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "sender_thread_id": { + "name": "sender_thread_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "origin": { + "name": "origin", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "origin_plugin_id": { + "name": "origin_plugin_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "requested_by_initiator": { + "name": "requested_by_initiator", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "requested_by_thread_id": { + "name": "requested_by_thread_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "model": { + "name": "model", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "reasoning_level": { + "name": "reasoning_level", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "permission_mode": { + "name": "permission_mode", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "service_tier": { + "name": "service_tier", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "group_with_next": { + "name": "group_with_next", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "send_at": { + "name": "send_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "waiting_on": { + "name": "waiting_on", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "wait_holder": { + "name": "wait_holder", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "failure_reason": { + "name": "failure_reason", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "payload_kind": { + "name": "payload_kind", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'inline'" + }, + "retry_of_turn_request_id": { + "name": "retry_of_turn_request_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "retry_attempt": { + "name": "retry_attempt", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "retry_reason": { + "name": "retry_reason", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "claimed_at": { + "name": "claimed_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "claim_token": { + "name": "claim_token", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "sort_key": { + "name": "sort_key", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "queued_thread_messages_thread_created_idx": { + "name": "queued_thread_messages_thread_created_idx", + "columns": [ + "thread_id", + "created_at", + "id" + ], + "isUnique": false + }, + "queued_thread_messages_thread_sort_idx": { + "name": "queued_thread_messages_thread_sort_idx", + "columns": [ + "thread_id", + "sort_key", + "id" + ], + "isUnique": false + }, + "queued_thread_messages_due_idx": { + "name": "queued_thread_messages_due_idx", + "columns": [ + "send_at", + "id" + ], + "isUnique": false, + "where": "\"queued_thread_messages\".\"send_at\" IS NOT NULL AND \"queued_thread_messages\".\"claimed_at\" IS NULL AND \"queued_thread_messages\".\"claim_token\" IS NULL" + }, + "queued_thread_messages_wait_holder_idx": { + "name": "queued_thread_messages_wait_holder_idx", + "columns": [ + "wait_holder", + "id" + ], + "isUnique": false, + "where": "\"queued_thread_messages\".\"wait_holder\" IS NOT NULL" + } + }, + "foreignKeys": { + "queued_thread_messages_thread_id_threads_id_fk": { + "name": "queued_thread_messages_thread_id_threads_id_fk", + "tableFrom": "queued_thread_messages", + "tableTo": "threads", + "columnsFrom": [ + "thread_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "retained_event_outputs": { + "name": "retained_event_outputs", + "columns": { + "event_id": { + "name": "event_id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "output_path": { + "name": "output_path", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "value": { + "name": "value", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "expires_at": { + "name": "expires_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "retained_event_outputs_expiry_idx": { + "name": "retained_event_outputs_expiry_idx", + "columns": [ + "expires_at", + "event_id" + ], + "isUnique": false + } + }, + "foreignKeys": { + "retained_event_outputs_event_id_events_id_fk": { + "name": "retained_event_outputs_event_id_events_id_fk", + "tableFrom": "retained_event_outputs", + "tableTo": "events", + "columnsFrom": [ + "event_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "system_experiments": { + "name": "system_experiments", + "columns": { + "key": { + "name": "key", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "value": { + "name": "value", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "terminal_sessions": { + "name": "terminal_sessions", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "thread_id": { + "name": "thread_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "environment_id": { + "name": "environment_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "host_id": { + "name": "host_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "daemon_session_id": { + "name": "daemon_session_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "initial_cwd": { + "name": "initial_cwd", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "cols": { + "name": "cols", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "rows": { + "name": "rows", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "exit_code": { + "name": "exit_code", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "close_reason": { + "name": "close_reason", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "last_user_input_at": { + "name": "last_user_input_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "terminal_sessions_thread_status_updated_idx": { + "name": "terminal_sessions_thread_status_updated_idx", + "columns": [ + "thread_id", + "status", + "updated_at" + ], + "isUnique": false + }, + "terminal_sessions_environment_status_idx": { + "name": "terminal_sessions_environment_status_idx", + "columns": [ + "environment_id", + "status" + ], + "isUnique": false + }, + "terminal_sessions_host_status_idx": { + "name": "terminal_sessions_host_status_idx", + "columns": [ + "host_id", + "status" + ], + "isUnique": false + }, + "terminal_sessions_daemon_session_idx": { + "name": "terminal_sessions_daemon_session_idx", + "columns": [ + "daemon_session_id" + ], + "isUnique": false + } + }, + "foreignKeys": { + "terminal_sessions_thread_id_threads_id_fk": { + "name": "terminal_sessions_thread_id_threads_id_fk", + "tableFrom": "terminal_sessions", + "tableTo": "threads", + "columnsFrom": [ + "thread_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "terminal_sessions_environment_id_environments_id_fk": { + "name": "terminal_sessions_environment_id_environments_id_fk", + "tableFrom": "terminal_sessions", + "tableTo": "environments", + "columnsFrom": [ + "environment_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "terminal_sessions_host_id_hosts_id_fk": { + "name": "terminal_sessions_host_id_hosts_id_fk", + "tableFrom": "terminal_sessions", + "tableTo": "hosts", + "columnsFrom": [ + "host_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "terminal_sessions_daemon_session_id_host_daemon_sessions_id_fk": { + "name": "terminal_sessions_daemon_session_id_host_daemon_sessions_id_fk", + "tableFrom": "terminal_sessions", + "tableTo": "host_daemon_sessions", + "columnsFrom": [ + "daemon_session_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "thread_conversation_outlines": { + "name": "thread_conversation_outlines", + "columns": { + "thread_id": { + "name": "thread_id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "projection_key": { + "name": "projection_key", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "items_json": { + "name": "items_json", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": { + "thread_conversation_outlines_thread_id_threads_id_fk": { + "name": "thread_conversation_outlines_thread_id_threads_id_fk", + "tableFrom": "thread_conversation_outlines", + "tableTo": "threads", + "columnsFrom": [ + "thread_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "thread_dynamic_context_file_states": { + "name": "thread_dynamic_context_file_states", + "columns": { + "thread_id": { + "name": "thread_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "file_key": { + "name": "file_key", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "content_status": { + "name": "content_status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "content_hash": { + "name": "content_hash", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "shown_at": { + "name": "shown_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "thread_dynamic_context_file_states_thread_file_idx": { + "name": "thread_dynamic_context_file_states_thread_file_idx", + "columns": [ + "thread_id", + "file_key" + ], + "isUnique": true + } + }, + "foreignKeys": { + "thread_dynamic_context_file_states_thread_id_threads_id_fk": { + "name": "thread_dynamic_context_file_states_thread_id_threads_id_fk", + "tableFrom": "thread_dynamic_context_file_states", + "tableTo": "threads", + "columnsFrom": [ + "thread_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "thread_plugin_metadata": { + "name": "thread_plugin_metadata", + "columns": { + "thread_id": { + "name": "thread_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "plugin_id": { + "name": "plugin_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "metadata_json": { + "name": "metadata_json", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": { + "thread_plugin_metadata_thread_id_threads_id_fk": { + "name": "thread_plugin_metadata_thread_id_threads_id_fk", + "tableFrom": "thread_plugin_metadata", + "tableTo": "threads", + "columnsFrom": [ + "thread_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "thread_plugin_metadata_thread_id_plugin_id_pk": { + "columns": [ + "thread_id", + "plugin_id" + ], + "name": "thread_plugin_metadata_thread_id_plugin_id_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "thread_pruning_cursors": { + "name": "thread_pruning_cursors", + "columns": { + "policy": { + "name": "policy", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "scope": { + "name": "scope", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "''" + }, + "thread_id": { + "name": "thread_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "version": { + "name": "version", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "last_thread_id": { + "name": "last_thread_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "''" + }, + "current_thread_id": { + "name": "current_thread_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "step": { + "name": "step", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "sequence": { + "name": "sequence", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "upper_sequence": { + "name": "upper_sequence", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "cycle": { + "name": "cycle", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "latest_root_sequence": { + "name": "latest_root_sequence", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "latest_context_sequence": { + "name": "latest_context_sequence", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "probe_event_id": { + "name": "probe_event_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "probe_phase": { + "name": "probe_phase", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "probe_sequence": { + "name": "probe_sequence", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "probe_witness_id": { + "name": "probe_witness_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "thread_pruning_cursors_thread_idx": { + "name": "thread_pruning_cursors_thread_idx", + "columns": [ + "thread_id" + ], + "isUnique": false + } + }, + "foreignKeys": { + "thread_pruning_cursors_thread_id_threads_id_fk": { + "name": "thread_pruning_cursors_thread_id_threads_id_fk", + "tableFrom": "thread_pruning_cursors", + "tableTo": "threads", + "columnsFrom": [ + "thread_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "thread_pruning_cursors_policy_scope_pk": { + "columns": [ + "policy", + "scope" + ], + "name": "thread_pruning_cursors_policy_scope_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": { + "thread_pruning_cursors_scope_check": { + "name": "thread_pruning_cursors_scope_check", + "value": "\"thread_pruning_cursors\".\"scope\" = coalesce(\"thread_pruning_cursors\".\"thread_id\", '')" + } + } + }, + "thread_search_segments": { + "name": "thread_search_segments", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "thread_id": { + "name": "thread_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "source_kind": { + "name": "source_kind", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "source_key": { + "name": "source_key", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "source_seq": { + "name": "source_seq", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "text": { + "name": "text", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "thread_search_segments_source_idx": { + "name": "thread_search_segments_source_idx", + "columns": [ + "thread_id", + "source_kind", + "source_key" + ], + "isUnique": true + }, + "thread_search_segments_thread_source_seq_idx": { + "name": "thread_search_segments_thread_source_seq_idx", + "columns": [ + "thread_id", + "source_seq" + ], + "isUnique": false + } + }, + "foreignKeys": { + "thread_search_segments_thread_id_threads_id_fk": { + "name": "thread_search_segments_thread_id_threads_id_fk", + "tableFrom": "thread_search_segments", + "tableTo": "threads", + "columnsFrom": [ + "thread_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "thread_sections": { + "name": "thread_sections", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "thread_sections_name_idx": { + "name": "thread_sections_name_idx", + "columns": [ + "name" + ], + "isUnique": true + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "thread_tabs": { + "name": "thread_tabs", + "columns": { + "thread_id": { + "name": "thread_id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "tabs_json": { + "name": "tabs_json", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "revision": { + "name": "revision", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": { + "thread_tabs_thread_id_threads_id_fk": { + "name": "thread_tabs_thread_id_threads_id_fk", + "tableFrom": "thread_tabs", + "tableTo": "threads", + "columnsFrom": [ + "thread_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "threads": { + "name": "threads", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "project_id": { + "name": "project_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "environment_id": { + "name": "environment_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "provider_id": { + "name": "provider_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "model_override": { + "name": "model_override", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "reasoning_level_override": { + "name": "reasoning_level_override", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "title_fallback": { + "name": "title_fallback", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "section_id": { + "name": "section_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'starting'" + }, + "startup_context": { + "name": "startup_context", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "parent_thread_id": { + "name": "parent_thread_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "lifecycle_owner_thread_id": { + "name": "lifecycle_owner_thread_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "source_thread_id": { + "name": "source_thread_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "origin_kind": { + "name": "origin_kind", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "origin_plugin_id": { + "name": "origin_plugin_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "visibility": { + "name": "visibility", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'visible'" + }, + "archived_at": { + "name": "archived_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "pinned_at": { + "name": "pinned_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "pin_sort_key": { + "name": "pin_sort_key", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "deleted_at": { + "name": "deleted_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "storage_deleted_at": { + "name": "storage_deleted_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "last_read_at": { + "name": "last_read_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "latest_attention_at": { + "name": "latest_attention_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "threads_project_id_idx": { + "name": "threads_project_id_idx", + "columns": [ + "project_id", + "id" + ], + "isUnique": false + }, + "threads_project_updated_idx": { + "name": "threads_project_updated_idx", + "columns": [ + "project_id", + "updated_at" + ], + "isUnique": false + }, + "threads_project_archived_deleted_idx": { + "name": "threads_project_archived_deleted_idx", + "columns": [ + "project_id", + "archived_at", + "deleted_at", + "id" + ], + "isUnique": false + }, + "threads_pin_sort_idx": { + "name": "threads_pin_sort_idx", + "columns": [ + "archived_at", + "deleted_at", + "pin_sort_key", + "id" + ], + "isUnique": false, + "where": "\"threads\".\"pinned_at\" IS NOT NULL" + }, + "threads_environment_idx": { + "name": "threads_environment_idx", + "columns": [ + "environment_id" + ], + "isUnique": false + }, + "threads_lifecycle_owner_idx": { + "name": "threads_lifecycle_owner_idx", + "columns": [ + "lifecycle_owner_thread_id" + ], + "isUnique": false + }, + "threads_parent_idx": { + "name": "threads_parent_idx", + "columns": [ + "parent_thread_id" + ], + "isUnique": false + }, + "threads_source_origin_idx": { + "name": "threads_source_origin_idx", + "columns": [ + "source_thread_id", + "origin_kind" + ], + "isUnique": false + }, + "threads_origin_plugin_archived_idx": { + "name": "threads_origin_plugin_archived_idx", + "columns": [ + "origin_plugin_id", + "archived_at" + ], + "isUnique": false + }, + "threads_section_archived_deleted_idx": { + "name": "threads_section_archived_deleted_idx", + "columns": [ + "section_id", + "archived_at", + "deleted_at", + "id" + ], + "isUnique": false + }, + "threads_archived_status_idx": { + "name": "threads_archived_status_idx", + "columns": [ + "archived_at", + "status" + ], + "isUnique": false + }, + "threads_environment_archived_deleted_idx": { + "name": "threads_environment_archived_deleted_idx", + "columns": [ + "environment_id", + "archived_at", + "deleted_at" + ], + "isUnique": false + }, + "threads_active_maintenance_idx": { + "name": "threads_active_maintenance_idx", + "columns": [ + "status" + ], + "isUnique": false, + "where": "\"threads\".\"deleted_at\" IS NULL" + } + }, + "foreignKeys": { + "threads_project_id_projects_id_fk": { + "name": "threads_project_id_projects_id_fk", + "tableFrom": "threads", + "tableTo": "projects", + "columnsFrom": [ + "project_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "threads_environment_id_environments_id_fk": { + "name": "threads_environment_id_environments_id_fk", + "tableFrom": "threads", + "tableTo": "environments", + "columnsFrom": [ + "environment_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + }, + "threads_section_id_thread_sections_id_fk": { + "name": "threads_section_id_thread_sections_id_fk", + "tableFrom": "threads", + "tableTo": "thread_sections", + "columnsFrom": [ + "section_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + }, + "threads_parent_thread_id_threads_id_fk": { + "name": "threads_parent_thread_id_threads_id_fk", + "tableFrom": "threads", + "tableTo": "threads", + "columnsFrom": [ + "parent_thread_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + }, + "threads_lifecycle_owner_thread_id_threads_id_fk": { + "name": "threads_lifecycle_owner_thread_id_threads_id_fk", + "tableFrom": "threads", + "tableTo": "threads", + "columnsFrom": [ + "lifecycle_owner_thread_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "restrict", + "onUpdate": "no action" + }, + "threads_source_thread_id_threads_id_fk": { + "name": "threads_source_thread_id_threads_id_fk", + "tableFrom": "threads", + "tableTo": "threads", + "columnsFrom": [ + "source_thread_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "ui_preferences": { + "name": "ui_preferences", + "columns": { + "key": { + "name": "key", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "value_json": { + "name": "value_json", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "revision": { + "name": "revision", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + } + }, + "views": {}, + "enums": {}, + "_meta": { + "schemas": {}, + "tables": {}, + "columns": {} + }, + "internal": { + "indexes": {} + } +} \ No newline at end of file diff --git a/packages/db/drizzle/meta/_journal.json b/packages/db/drizzle/meta/_journal.json index 6a30ce673b3..c2e5d65b131 100644 --- a/packages/db/drizzle/meta/_journal.json +++ b/packages/db/drizzle/meta/_journal.json @@ -890,6 +890,13 @@ "when": 1789607725218, "tag": "0126_overconfident_vin_gonzales", "breakpoints": true + }, + { + "idx": 127, + "version": "6", + "when": 1789686861487, + "tag": "0127_plain_cammi", + "breakpoints": true } ] } \ No newline at end of file diff --git a/packages/db/test/migrate.test.ts b/packages/db/test/migrate.test.ts index e4814f35e2d..f45452edab8 100644 --- a/packages/db/test/migrate.test.ts +++ b/packages/db/test/migrate.test.ts @@ -868,9 +868,7 @@ function rewindMachineProvidersMigration(db: DbConnection): void { "requested_by_thread_id", ]) { if (!queuedDispatchOrigin.some((column) => column.name === name)) continue; - db.$client.exec( - `ALTER TABLE queued_thread_messages DROP COLUMN ${name}`, - ); + db.$client.exec(`ALTER TABLE queued_thread_messages DROP COLUMN ${name}`); } if ( db.$client @@ -879,7 +877,6 @@ function rewindMachineProvidersMigration(db: DbConnection): void { .some((column) => column.name === "completed_item_history") ) { db.$client.exec("ALTER TABLE events DROP COLUMN completed_item_history"); - } db.$client.exec("DROP TABLE IF EXISTS thread_pruning_cursors"); db.$client.exec("DROP TABLE IF EXISTS project_attachment_threads"); From 820e4e5bfc5a029aa5168a7459726cbf008fa789 Mon Sep 17 00:00:00 2001 From: Michael Yong Date: Thu, 17 Sep 2026 17:13:39 -0700 Subject: [PATCH 6/7] Reuse bounded completed-item reconstruction across reads --- .../db/src/data/completed-item-history.ts | 79 ++++++++++- .../data/completed-item-compaction.test.ts | 132 ++++++++++++++++++ 2 files changed, 209 insertions(+), 2 deletions(-) diff --git a/packages/db/src/data/completed-item-history.ts b/packages/db/src/data/completed-item-history.ts index 1c6ec59a915..16014e8d347 100644 --- a/packages/db/src/data/completed-item-history.ts +++ b/packages/db/src/data/completed-item-history.ts @@ -35,6 +35,77 @@ function reconstructCompletedItemHistory(metadata: string, ownerData: string) { }; } +interface ReconstructionEntry { + metadata: string; + ownerData: string; + history: ReturnType; + chars: number; + records: number; +} + +interface ReconstructionCache { + entries: Map; + chars: number; + records: number; +} + +const reconstructionCaches = new WeakMap< + DbQueryConnection, + ReconstructionCache +>(); +const RECONSTRUCTION_CACHE_MAX_RECORDS = 10_000; +const RECONSTRUCTION_CACHE_MAX_CHARS = 8_000_000; + +function reconstructCached( + db: DbQueryConnection, + id: string, + metadata: string, + ownerData: string, +) { + let cache = reconstructionCaches.get(db); + if (cache === undefined) { + cache = { entries: new Map(), chars: 0, records: 0 }; + reconstructionCaches.set(db, cache); + } + const previous = cache.entries.get(id); + if (previous !== undefined) { + cache.entries.delete(id); + cache.chars -= previous.chars; + cache.records -= previous.records; + if (previous.metadata === metadata && previous.ownerData === ownerData) { + cache.entries.set(id, previous); + cache.chars += previous.chars; + cache.records += previous.records; + return previous.history; + } + } + const history = reconstructCompletedItemHistory(metadata, ownerData); + const chars = + metadata.length + + ownerData.length + + history.records.reduce((sum, record) => sum + record.data.length, 0); + const records = history.records.length + 1; + if ( + chars > RECONSTRUCTION_CACHE_MAX_CHARS || + records > RECONSTRUCTION_CACHE_MAX_RECORDS + ) + return history; + while ( + cache.records + records > RECONSTRUCTION_CACHE_MAX_RECORDS || + cache.chars + chars > RECONSTRUCTION_CACHE_MAX_CHARS + ) { + const oldest = cache.entries.entries().next().value; + if (oldest === undefined) break; + cache.entries.delete(oldest[0]); + cache.chars -= oldest[1].chars; + cache.records -= oldest[1].records; + } + cache.entries.set(id, { metadata, ownerData, history, chars, records }); + cache.chars += chars; + cache.records += records; + return history; +} + function expandSelectedCompletedItemRowsInternal( db: DbQueryConnection, rows: readonly StoredEventRow[], @@ -77,13 +148,17 @@ function expandSelectedCompletedItemRowsInternal( } const expanded: ProjectionStoredEventRow[] = []; for (const selectedRow of rows) { + const metadata = histories.get(selectedRow.id); + if (metadata === undefined && includeParsedData) { + if (selectedRow.sequence <= throughSequence) expanded.push(selectedRow); + continue; + } const { completedItemHistory: _history, ...row } = selectedRow; - const metadata = histories.get(row.id); if (metadata === undefined) { if (row.sequence <= throughSequence) expanded.push(row); continue; } - const history = reconstructCompletedItemHistory(metadata, row.data); + const history = reconstructCached(db, row.id, metadata, row.data); if (history.sequence <= throughSequence) expanded.push({ ...row, diff --git a/packages/db/test/data/completed-item-compaction.test.ts b/packages/db/test/data/completed-item-compaction.test.ts index 8b31e2ea1ca..6b5ff1c848a 100644 --- a/packages/db/test/data/completed-item-compaction.test.ts +++ b/packages/db/test/data/completed-item-compaction.test.ts @@ -144,6 +144,138 @@ describe("completed items at first lifecycle position", () => { } }); + it("reuses identical history while applying each snapshot and row scope", () => { + const f = setup(); + try { + f.advance(); + const owner = listStoredTimelineTurnEventRows(f.db, { + threadId: f.thread.id, + turnIds: ["turn"], + sequenceStart: 0, + maxInlineOutputChars: 8000, + }).find((row) => row.completedItemHistory != null)!; + const full = expandSelectedCompletedItemRowsForProjection(f.db, [owner]); + const again = expandSelectedCompletedItemRowsForProjection(f.db, [ + { ...owner }, + ]); + expect(again).toEqual(full); + expect(again[0]!.parsedData).toBe(full[0]!.parsedData); + const prefix = expandSelectedCompletedItemRowsForProjection( + f.db, + [owner], + 3, + ); + expect(prefix).toEqual(full.filter((row) => row.sequence <= 3)); + const scoped = expandSelectedCompletedItemRowsForProjection(f.db, [ + { + ...owner, + providerThreadId: "changed-provider", + parentToolCallId: "changed-parent", + }, + ]); + expect( + scoped.every( + (row) => + row.providerThreadId === "changed-provider" && + row.parentToolCallId === "changed-parent", + ), + ).toBe(true); + expect(scoped.map((row) => row.data)).toEqual( + full.map((row) => row.data), + ); + } finally { + f.db.$client.close(); + } + }); + + it.each(["characters", "records"] as const)( + "evicts reconstructed history at its %s bound", + (bound) => { + const f = setup(); + try { + f.advance(); + const owner = listStoredTimelineTurnEventRows(f.db, { + threadId: f.thread.id, + turnIds: ["turn"], + sequenceStart: 0, + maxInlineOutputChars: 8000, + }).find((row) => row.completedItemHistory != null)!; + const first = expandSelectedCompletedItemRowsForProjection(f.db, [ + owner, + ])[0]!.parsedData; + const copies = bound === "characters" ? 500 : 5000; + const data = + bound === "characters" + ? JSON.stringify({ + item: { + id: "message", + type: "agentMessage", + text: "x".repeat(20_000), + }, + }) + : owner.data; + let last = owner; + let lastPayload = first; + for (let index = 0; index < copies; index++) { + last = { ...owner, id: `cache-owner-${index}`, data }; + lastPayload = expandSelectedCompletedItemRowsForProjection(f.db, [ + last, + ])[0]!.parsedData; + } + expect( + expandSelectedCompletedItemRowsForProjection(f.db, [last])[0]! + .parsedData, + ).toBe(lastPayload); + expect( + expandSelectedCompletedItemRowsForProjection(f.db, [owner])[0]! + .parsedData, + ).not.toBe(first); + } finally { + f.db.$client.close(); + } + }, + ); + + it("does not retain an oversized reconstruction or evict a smaller one for it", () => { + const f = setup(); + try { + f.advance(); + const owner = listStoredTimelineTurnEventRows(f.db, { + threadId: f.thread.id, + turnIds: ["turn"], + sequenceStart: 0, + maxInlineOutputChars: 8000, + }).find((row) => row.completedItemHistory != null)!; + const first = expandSelectedCompletedItemRowsForProjection(f.db, [ + owner, + ])[0]!.parsedData; + const oversized = { + ...owner, + id: "oversized", + data: JSON.stringify({ + item: { + id: "message", + type: "agentMessage", + text: "x".repeat(8_000_001), + }, + }), + }; + const big = expandSelectedCompletedItemRowsForProjection(f.db, [ + oversized, + ])[0]!.parsedData; + expect( + expandSelectedCompletedItemRowsForProjection(f.db, [oversized])[0]! + .parsedData, + ).not.toBe(big); + expect( + expandSelectedCompletedItemRowsForProjection(f.db, [owner])[0]! + .parsedData, + ).toBe(first); + } finally { + f.db.$client.close(); + } + }); + it("charges the timeline budget for records inside a combined item", () => { const f = setup(); try { From f312cf6e451969cd4604ed07027b3497d42ca610 Mon Sep 17 00:00:00 2001 From: Michael Yong Date: Thu, 17 Sep 2026 18:25:54 -0700 Subject: [PATCH 7/7] Document full-copy reader performance verification --- .../completed-item-compaction-verification.md | 126 ++++++++++++------ docs/completed-item-compaction.md | 12 +- 2 files changed, 96 insertions(+), 42 deletions(-) diff --git a/docs/completed-item-compaction-verification.md b/docs/completed-item-compaction-verification.md index 3b4ca829034..8da87374d0e 100644 --- a/docs/completed-item-compaction-verification.md +++ b/docs/completed-item-compaction-verification.md @@ -1,9 +1,9 @@ # Completed-item compaction verification -Measured on 2026-09-16 in an isolated worktree based on `5aca5733a5`, +Storage and maintenance measured on 2026-09-16 in an isolated worktree based on `5aca5733a5`, including PR1 (`c663ff1911`, #3766). The final measurements include the output-order safeguard and exclusion of compaction from synchronous per-thread -cleanup. These are observations from private sanitized SQLite copies. +cleanup. These are observations from private sanitized SQLite copies. The reader section below supersedes the earlier 30-page timing samples and compares the rebased implementation with main. ## Correctness and storage @@ -19,7 +19,6 @@ cleanup. These are observations from private sanitized SQLite copies. | Complete visible timelines | All 2,326 match, including all 2,223 non-null context values | | Highwater and provider recovery | All 2,326 match | | Existing side tables | All 46 non-event/non-cursor/non-migration tables match, including output, search and attachment ownership | -| Default-budget server pages | All 30 match exactly | | Complete large-thread pagination | Three largest threads: all 3,816 rendered rows match after complete traversal at a 1,000-row budget; baseline used 10,000 | | Other consumers | 2,326 latest-output checks, ten large outlines and 100 rollback-only output mutations match | @@ -47,9 +46,6 @@ not establish cold-I/O bounds or a maximum event-loop stall. | Final live wrapper, 2,500 calls | median 0.288 ms; p99 5.46 ms; maximum 322.25 ms | | PR1 live wrapper, same 2,500-call workload | median 0.243 ms; p99 5.28 ms; maximum 87.34 ms | | Compaction calls / event deletions in final live sample | zero / zero | -| Median of 30 page case medians, before / after | 103.62 / 121.55 ms | -| Median paired page increase | 8.42 ms | -| Largest individual page call, before / after | 391.81 / 464.44 ms | | Actual background sweep, 100 calls | 1,565 total advances; 313 completed-item advances | | Maximum sweep / advance in that sample | 216.46 / 177.78 ms | @@ -77,12 +73,13 @@ At 3.13 completed-item advances per sweep and the existing ten-second cadence, the latest sample extrapolates to about **107 idle hours** for first-pass catch-up. The initial revision's sample estimated 84 hours. Actual processing time remains about five minutes; scheduling, activity and I/O determine elapsed catch-up. -These are short-sample estimates, not an end-to-end scheduled run. Reader overhead, -checkpoint stalls and multi-day idle catch-up remain limitations. +These are short-sample estimates, not an end-to-end scheduled run. Checkpoint +stalls and multi-day idle catch-up remain limitations. Reader performance was +subsequently fixed and measured separately below. ## Automated and UI verification -Final follow-up checks: +Earlier maintenance follow-up checks (reader-pass checks follow below): - Turbo DB tests: 44 files, 591 tests pass, using migrated real SQLite. - Turbo affected server tests: 14 tests pass, including the previously failing @@ -111,33 +108,84 @@ under parent thread `thr_vmdgc3ke5y` in `pr2-review/final`. Copies are private a mode 0600, with Connect plugin records verified absent. No live database or copied Connect configuration was used; nothing was merged or deployed. -## Reader optimization follow-up (2026-09-17) - -Compared with rebased PR2 `e2091fbea2` (base `0188d91972`), the reader now -fetches selected completion metadata in one parameterized query instead of -250-ID batches. SQLite uses the existing events primary-key index; `json_each` -expands only the supplied ID array. Nested history no longer goes through an -extra JSON serialization, parse and validation, and reconstruction skips parsing -completion payloads when it needs none of their fields. Storage, compaction -rules, cache state and indexes are unchanged. - -Ten large selections, alternating original and optimized reconstruction ten times -per case, matched exactly. Median relative reconstruction improvement was 22.6%. -One 10,000-row selection improved from 170.1 to 135.5 ms, with 35 metadata queries -reduced to one. The query plan uses `sqlite_autoindex_events_1`, not an events scan. - -Thirty matching page requests were also measured with six alternating calls per -implementation, after warming both. All response fields and pagination matched. -The median paired elapsed reduction was 7.3 ms; process CPU time fell by a median -paired 8.3 ms, with lower CPU usage in 26 of 30 cases. The host was busy, so elapsed -timings are approximate and are not expected production latency. This compares -optimized PR2 with the original rebased PR2; it does not establish that PR2 is as -fast as main. Component reconstruction savings are not whole-page percentages. - -The follow-up passes 602 DB tests, 74 server tests, and DB/server typechecks. -All 2,326 complete timelines and context values match the uncompacted baseline, -with zero differences or errors. This reader change does not establish a new -maximum live or background event-loop stall time. - -Artifacts and harnesses: -`/Users/michael/.bb/thread-storage/thr_vmdgc3ke5y/pr2-reader-optimization/`. +## Full-copy reader performance (2026-09-17) + +Compared main `0bb64f3789` (including #3874 and #3876) with implementation +`820e4e5bfc`. Later main commits were checked for changes to the measured DB and +timeline paths. The source fix makes the timeline budget count reconstructed +records, selects metadata with the physical rows, passes already parsed payloads +to projection, and reuses reconstruction in a bounded per-connection cache. +Cache hits require identical stored metadata and completion payloads. Limits are +8 million accounted text characters and 10,000 reconstructed records; this is +not a byte-exact heap cap. There are still no new tables or indexes. + +The benchmark uses the complete sanitized pre-compaction database and its +compacted counterpart: all 2,326 threads, with every older cursor followed to the +end. Each request runs both versions and compares the complete response, +including pagination and context. Three alternating warm samples per version +follow the first call. The measured interval is the complete server timeline +builder, including selection, reconstruction and projection; it excludes HTTP, +network and browser rendering. SQLite uses production cache/mmap settings. + +| Workload | Matching pages | Main / PR2 median warm page time | Median paired change | Sum of warm page medians | +| --- | ---: | ---: | ---: | ---: | +| Product settings | 3,027 | 9.73 / 8.78 ms | −1.14 ms | 43.49 / 38.22 s (12.1% lower) | +| Expanded stress settings | 2,733 | 11.77 / 10.87 ms | −1.13 ms | 58.45 / 51.78 s (11.4% lower) | + +Product settings use the 1,500-event budget, 20 segments, lazy nested rows, +32,000 inline output characters, and saved provider display/diagnostic settings. +The stress workload uses a 10,000-event budget, expanded nested rows, collapsed +completed turns and 8,000 inline characters. These are two complete traversals +of the same database, not 5,760 distinct stored pages. Every response matches. +Neither census has a warm elapsed-time case more than both 5 ms and 10% slower. +Warm process CPU totals are 12.5% lower for product settings and 11.7% lower for +the stress workload. Among the 93 product-setting requests taking at least 50 ms +on main, the median paired improvement is 11.75 ms. + +First visits are noisier: product-setting median paired elapsed change is ++0.15 ms, with summed first-call time 7.2% higher but CPU 3.0% lower. Stress +first-call elapsed and CPU totals are lower. This does not establish that every +first read is faster. Cases crossing the 5 ms / 10% threshold in elapsed or CPU, +plus controls, were selected for repeated cold-application-cache and warm checks. +A cold application cache means a fresh DB wrapper and empty per-connection JS +caches, not a cold OS disk cache. + +All 457 selected product-setting requests and 78 stress requests match in six +alternating samples per version and cache mode. None exceeds both elapsed-time +thresholds in either mode. Median paired changes are −1.39 / −2.37 ms for product +cold/warm checks and −1.05 / −1.85 ms for stress cold/warm checks. Three CPU-only +outliers were repeated with twelve samples per version and mode; all three then +have lower PR2 elapsed and CPU medians. All original samples remain in the +artifacts rather than being discarded. + +The two originally blocking large-page regressions now measure 214.11 → 200.16 ms +and 191.56 → 151.21 ms in the repeated warm stress checks. Fresh application-cache +medians also improve: 214.22 → 193.46 ms and 204.57 → 185.67 ms. Their responses +remain identical. + +Before backfill, both versions were also run against the same uncompacted copy +for 45 requests from large threads. All responses match across nine alternating +samples per version and cache mode. Median paired elapsed overhead is +0.88 ms +with fresh application caches and +0.76 ms warm; no elapsed case exceeds both +5 ms and 10%. Summed warm elapsed medians increase 2.7%, and CPU medians 3.4%. +Two CPU-only cases cross the threshold in different cache modes; this is a small +pre-backfill cost, not a claim of zero overhead before any rows are compacted. + +The inherited benchmark mixed a CommonJS Drizzle driver with the app's ES-module +schema. Profiling exposed extra generic row-mapping overhead. The final harness +uses the same ES-module driver as `createConnection` and asserts matching driver +and native SQLite constructors. Only the corrected `production-*` artifacts +support these timing results; earlier timing samples are superseded. Earlier +response-equality checks are supplemented by the complete corrected traversals. + +Final source validation: 611 DB tests, 198 affected server tests, and DB/server +typechecks pass through Turbo. Added tests cover logical work budgets, metadata +selection, parsed payloads, malformed JSON, cache invalidation, scope/snapshot +filtering, eviction and oversized-entry bypass. Migration 0127 adds only the +metadata column; events indexes match the preceding main snapshot. + +These measurements address the observed reader regression. They do not establish +a maximum event-loop stall, a cold-disk latency bound, or a new catch-up estimate. +The historical maintenance/checkpoint measurements above remain separate. +Scripts, raw samples and the final report are in parent thread storage +`pr2-perf-fix/`. diff --git a/docs/completed-item-compaction.md b/docs/completed-item-compaction.md index 16850e1e8bb..9da40a7d4d2 100644 --- a/docs/completed-item-compaction.md +++ b/docs/completed-item-compaction.md @@ -41,8 +41,14 @@ Timeline queries select physical rows with their internal metadata, then reconst only the selected history for projection. The timeline event budget charges for the records inside each combined item, so removing physical rows does not cause a page to decode substantially more history. Reconstruction passes validated -payload objects directly to projection instead of parsing them again. Selected fork history recovers completion order before -applying the existing completed-turn and event-type rules. Forks still create +payload objects directly to projection instead of parsing them again. A per-connection +in-memory cache reuses reconstruction only when both stored metadata and the +completion payload match exactly. It evicts least-recently-used entries above +8 million accounted text characters or 10,000 reconstructed records, and does +not retain oversized entries. These are accounting limits, not a byte-exact +JavaScript heap limit. Current row scope and snapshot filtering are applied +after reuse; changed output or metadata forces reconstruction. Selected fork +history recovers completion order before applying the existing completed-turn and event-type rules. Forks still create new IDs and sequence numbers. No arbitrary deleted-ID lookup or virtual raw-event pagination is provided. @@ -70,5 +76,5 @@ The migrated SQLite regression tests cover eligibility, lossless reconstruction, command discard rules, output ownership, atomic rollback, late arrivals, highwater, boundary exclusions and one-row raw traversal. Server tests exercise the existing live wrapper, rewrite notifications and warmed timeline caches. -Full-copy measurements and UI evidence accompany the draft PR; prototype +Full-copy measurements and UI evidence accompany the PR; prototype measurements are not implementation guarantees.