diff --git a/apps/server/src/services/threads/stored-event-decode-cache.ts b/apps/server/src/services/threads/stored-event-decode-cache.ts index 8f116ed2dc2..75ac2ae0fe3 100644 --- a/apps/server/src/services/threads/stored-event-decode-cache.ts +++ b/apps/server/src/services/threads/stored-event-decode-cache.ts @@ -1,4 +1,8 @@ -import type { DbConnection, StoredEventRow } from "@bb/db"; +import type { + DbConnection, + StoredEventRow, + ProjectionStoredEventRow, +} from "@bb/db"; import type { ThreadEvent } from "@bb/domain"; import { parseStoredEvent } from "./thread-data.js"; @@ -119,7 +123,7 @@ function rememberEntry( export function decodeStoredEventRowCached( db: DbConnection, - row: StoredEventRow, + row: ProjectionStoredEventRow, ): ThreadEvent { const cache = getDecodeCache(db); const byRow = cache.byRow.get(row); diff --git a/apps/server/src/services/threads/thread-data.ts b/apps/server/src/services/threads/thread-data.ts index a9ee37f1486..855bc497da0 100644 --- a/apps/server/src/services/threads/thread-data.ts +++ b/apps/server/src/services/threads/thread-data.ts @@ -6,7 +6,11 @@ import { hydrateRetainedEventOutputRows, listStoredEventRows as listStoredEventRowRecords, } from "@bb/db"; -import type { DbConnection, StoredEventRow } from "@bb/db"; +import type { + DbConnection, + StoredEventRow, + ProjectionStoredEventRow, +} from "@bb/db"; import { toRecord } from "@bb/core-ui"; import { buildThreadEventRow, parseStoredThreadEvent } from "@bb/domain"; import { threadScope, turnScope } from "@bb/domain"; @@ -21,8 +25,8 @@ import { ApiError } from "../../errors.js"; const THREAD_EVENT_RESPONSE_DATA_BYTE_LIMIT = 8 * 1024 * 1024; type StoredEventPayloadRow = Pick< - StoredEventRow, - "data" | "sequence" | "threadId" | "type" + ProjectionStoredEventRow, + "data" | "sequence" | "threadId" | "type" | "parsedData" >; interface ListThreadEventRowsArgs { @@ -43,6 +47,7 @@ interface FindThreadEventArgs { export function parseStoredEventPayload( row: StoredEventPayloadRow, ): Record { + if (row.parsedData !== undefined) return row.parsedData; let data: unknown; try { data = JSON.parse(row.data); @@ -88,7 +93,7 @@ function parseStoredEventScope(row: StoredEventRow): ThreadEventScope { } } -export function parseStoredEvent(row: StoredEventRow): ThreadEvent { +export function parseStoredEvent(row: ProjectionStoredEventRow): ThreadEvent { return parseStoredThreadEvent({ type: row.type, data: parseStoredEventPayload(row), diff --git a/apps/server/src/services/threads/thread-fork-history.ts b/apps/server/src/services/threads/thread-fork-history.ts index 8e3742a8996..13461088e84 100644 --- a/apps/server/src/services/threads/thread-fork-history.ts +++ b/apps/server/src/services/threads/thread-fork-history.ts @@ -1,3 +1,4 @@ +import { expandSelectedCompletedItemRows } from "@bb/db"; import { copyStoredThreadEventsInTransaction, findLastCompletedRootStoredTurn, @@ -262,11 +263,15 @@ function selectInheritedForkEventRows( deps: Pick, args: { historyEndSequence: number; sourceThreadId: string }, ): StoredEventRow[] { - const rows = listStoredEventRows(deps.db, { - beforeSequence: args.historyEndSequence + 1, - threadId: args.sourceThreadId, - types: INHERITED_EVENT_TYPES, - }); + const rows = expandSelectedCompletedItemRows( + deps.db, + listStoredEventRows(deps.db, { + beforeSequence: args.historyEndSequence + 1, + threadId: args.sourceThreadId, + types: INHERITED_EVENT_TYPES, + }), + args.historyEndSequence, + ).filter((row) => INHERITED_EVENT_TYPES.some((type) => type === row.type)); const completedTurnIds = new Set(); const acceptedClientRequestIds = new Set(); for (const row of rows) { diff --git a/apps/server/src/services/threads/timeline-selection-memo.ts b/apps/server/src/services/threads/timeline-selection-memo.ts index 05acfd36619..03f5e8b0442 100644 --- a/apps/server/src/services/threads/timeline-selection-memo.ts +++ b/apps/server/src/services/threads/timeline-selection-memo.ts @@ -136,7 +136,7 @@ function getSelectionMemo(db: DbConnection): TimelineSelectionMemo { function dataCharsOfRows(rows: readonly StoredEventRow[]): number { let chars = 0; for (const row of rows) { - chars += row.data.length; + chars += row.data.length + (row.completedItemHistory?.length ?? 0); } return chars; } @@ -280,7 +280,9 @@ export function lookupLatestTimelineSelection( const budgetFloor: TimelineBudgetFloor = { sequence: findTimelineWindowBudgetFloorSequence(db, { threadId: args.threadId, - sequenceStart: args.epochSequenceStart, + sequenceStart: + entry.hints[args.page.segmentLimit]?.sequence ?? + args.epochSequenceStart, beforeSequence: args.maxSeq + 1, eventBudget: args.eventBudget, excludedTypes: THREAD_TIMELINE_EXCLUDED_EVENT_TYPES, diff --git a/apps/server/src/services/threads/timeline.ts b/apps/server/src/services/threads/timeline.ts index 46069a0d2a9..c5b4c2409d1 100644 --- a/apps/server/src/services/threads/timeline.ts +++ b/apps/server/src/services/threads/timeline.ts @@ -1,3 +1,4 @@ +import { expandSelectedCompletedItemRowsForProjection } from "@bb/db"; import { paginateTimelineContents } from "./timeline-content-pagination.js"; import { getTimelineGroupingContext, @@ -972,7 +973,8 @@ function selectStandardTimelineEventRows( knownBudgetFloor === null ? findTimelineWindowBudgetFloorSequence(db, { threadId: thread.id, - sequenceStart: epochSequenceStart, + sequenceStart: + hints[page.segmentLimit]?.sequence ?? epochSequenceStart, beforeSequence, eventBudget, excludedTypes: THREAD_TIMELINE_EXCLUDED_EVENT_TYPES, @@ -1357,7 +1359,11 @@ function buildThreadTimelineInternal( rows: hydrateRetainedEventOutputRows(db, storedEventSelection.rows), } : storedEventSelection; - const rawEventRows = eventSelection.rows; + const rawEventRows = expandSelectedCompletedItemRowsForProjection( + db, + eventSelection.rows, + snapshot.maxSeq, + ); profile.eventDataBytes = byteLengthOfStoredEventRows(rawEventRows); profile.eventRowCount = rawEventRows.length; profile.selectionStrategy = eventSelection.strategy; @@ -1624,9 +1630,10 @@ export function buildThreadConversationOutline( sequenceStart: contextBoundarySeq ?? 0, threadId: thread.id, }); - const decodedRawEvents = rawEventRows.map((row) => - toThreadEventWithMeta(row), - ); + const decodedRawEvents = expandSelectedCompletedItemRowsForProjection( + db, + rawEventRows, + ).map((row) => toThreadEventWithMeta(row)); const decodedEvents = compactThreadTimelineSummaryEvents(decodedRawEvents); const clientRequestContextRows = selectClientRequestContextRows(db, { rows: rawEventRows, @@ -1926,7 +1933,11 @@ function buildTimelineTurnSummaryDetailsPage( : sourceSeqStart, sourceRange.sourceSeqStart, ); - const projectionEvents = projectionEventRows + const projectionEvents = expandSelectedCompletedItemRowsForProjection( + db, + projectionEventRows, + snapshot.maxSeq, + ) .filter((row) => row.sequence <= snapshot.maxSeq) .map((row) => toThreadEventWithMeta(row)); const children = buildThreadTimelineTurnDetailsFromEvents({ diff --git a/apps/server/test/services/threads/timeline-event-budget.test.ts b/apps/server/test/services/threads/timeline-event-budget.test.ts index 7d4d807f713..6432d76c040 100644 --- a/apps/server/test/services/threads/timeline-event-budget.test.ts +++ b/apps/server/test/services/threads/timeline-event-budget.test.ts @@ -18,6 +18,8 @@ import { } from "@bb/domain"; import type { ClientTurnRequestId, Thread } from "@bb/domain"; import { + advanceThreadPruning, + listStoredEventRows, createConnection, createProject, createThread, @@ -1390,3 +1392,123 @@ it("resolves acceptance after the next conversation boundary", () => { expect(rows).toEqual(expected.rows); db.$client.close(); }); + +it.each([1, 2, 5, 20])( + "traverses compacted items completely with event budget %i and a small byte budget", + (eventBudget) => { + const { db, thread } = setup(); + try { + insertTurns(db, thread, 3, [2, 15, 2]); + db.$client + .prepare("UPDATE events SET sequence = -sequence WHERE thread_id = ?") + .run(thread.id); + db.$client + .prepare( + "UPDATE events SET sequence = -10 * sequence WHERE thread_id = ?", + ) + .run(thread.id); + const owners = listStoredEventRows(db, { + threadId: thread.id, + types: ["item/completed"], + }); + const additions: Parameters[2] = []; + const ends = new Map(); + for (const row of owners) { + if (!row.turnId || !row.itemId) + throw new Error("Missing fixture scope"); + const base = { + threadId: thread.id, + scope: turnScope(row.turnId), + providerThreadId, + itemId: row.itemId, + itemKind: row.itemKind, + parentToolCallId: null, + }; + const data = JSON.parse(row.data); + additions.push({ + ...base, + type: "item/started", + sequence: row.sequence - 2, + createdAt: row.createdAt - 2, + data: JSON.stringify({ item: { ...data.item, text: "" } }), + }); + additions.push({ + ...base, + type: "item/agentMessage/delta", + itemKind: null, + sequence: row.sequence - 1, + createdAt: row.createdAt - 1, + data: JSON.stringify({ itemId: row.itemId, delta: data.item.text }), + }); + ends.set(row.turnId, row.sequence + 1); + } + for (const [turnId, sequence] of ends) + additions.push({ + threadId: thread.id, + scope: turnScope(turnId), + providerThreadId, + itemId: null, + itemKind: null, + parentToolCallId: null, + type: "turn/completed", + sequence, + data: JSON.stringify({ status: "completed" }), + }); + insertEvents( + db, + noopNotifier, + additions.sort((a, b) => a.sequence - b.sequence), + ); + const options = { + completedTurnDisplay: "collapse", + includeDiagnosticOperations: false, + includeNestedRows: true, + maxInlineOutputChars: null, + maxSeq: 0, + } as const; + const canonical = buildThreadTimelineWithProfile(db, thread, { + ...options, + eventBudget: LARGE_BUDGET, + page: { kind: "latest", segmentLimit: 100 }, + }).response.rows; + let removed = 0; + for (let i = 0; i < 100; i++) { + const result = advanceThreadPruning(db, "completed-items"); + removed += result.removed; + if (result.action === "cycle-complete") break; + } + expect(removed).toBe(owners.length * 2); + let rows: TimelineRow[] = []; + let cursor: TimelinePaginationCursor | null = null; + const cursors = new Set(); + do { + const response: ThreadTimelineResponse = buildThreadTimelineWithProfile( + db, + thread, + { + ...options, + eventBudget, + responseByteBudget: 512, + page: cursor + ? { kind: "older", beforeCursor: cursor, segmentLimit: 2 } + : { kind: "latest", segmentLimit: 2 }, + }, + ).response; + rows = prependOlderTimelineRows({ + loadedRows: rows, + olderRows: response.rows, + }); + cursor = response.timelinePage.olderCursor; + if (cursor) { + const key = JSON.stringify(cursor); + expect(cursors.has(key)).toBe(false); + cursors.add(key); + expect(cursors.size).toBeLessThan(100); + } + } while (cursor); + expect(rows).toEqual(canonical); + } finally { + db.$client.close(); + } + }, +); diff --git a/apps/server/test/system/event-pruning.test.ts b/apps/server/test/system/event-pruning.test.ts index 0d19a7fed36..106af999242 100644 --- a/apps/server/test/system/event-pruning.test.ts +++ b/apps/server/test/system/event-pruning.test.ts @@ -1,4 +1,6 @@ -import { getThread, listEvents } from "@bb/db"; +import { getThread, listEvents, threads } from "@bb/db"; +import { eq } from "drizzle-orm"; +import { runThreadPruningSweep } from "../../src/services/system/thread-pruning-sweep.js"; import { turnScope } from "@bb/domain"; import { groupHostDaemonEvents } from "@bb/host-daemon-contract"; import { describe, expect, it, vi } from "vitest"; @@ -171,6 +173,93 @@ function seedResolvedAssistantMessage( } describe("thread event pruning", () => { + it("refreshes a cached visible timeline after an idle background completed-item rewrite and exposes combined raw records", async () => { + await withTestHarness(async (harness) => { + const host = seedHost(harness.deps); + const { project } = seedProjectWithSource(harness.deps, { + hostId: host.id, + }); + const environment = seedEnvironment(harness.deps, { + hostId: host.id, + projectId: project.id, + }); + const thread = seedThread(harness.deps, { + projectId: project.id, + environmentId: environment.id, + }); + seedStoredEvent(harness.deps, { + threadId: thread.id, + sequence: 1, + scope: turnScope("turn-1"), + type: "turn/started", + itemId: null, + itemKind: null, + data: {}, + }); + seedResolvedAssistantMessage(harness, { + threadId: thread.id, + itemId: "message", + deltaSequences: [2], + completedSequence: 3, + }); + seedStoredEvent(harness.deps, { + threadId: thread.id, + sequence: 4, + scope: turnScope("turn-1"), + type: "turn/completed", + itemId: null, + itemKind: null, + data: { status: "completed" }, + }); + harness.db.$client + .prepare("UPDATE events SET provider_thread_id = ? WHERE thread_id = ?") + .run("provider", thread.id); + const build = () => + buildThreadTimelineWithProfile(harness.db, thread, { + completedTurnDisplay: "collapse", + includeDiagnosticOperations: false, + includeNestedRows: true, + maxInlineOutputChars: 8000, + eventBudget: 10000, + maxSeq: 4, + page: { kind: "latest", segmentLimit: 20 }, + }).response; + const before = build(); + expect(JSON.stringify(before)).toContain("Final answer"); + expect(build()).toEqual(before); + const notify = vi.spyOn(harness.deps.hub, "notifyThread"); + for (let i = 0; i < 10; i++) { + const result = pruneThreadEventHistoryBestEffort(harness.deps, { + threadId: thread.id, + mode: "active", + }); + expect(result?.policy).not.toBe("completed-items"); + } + expect(listEvents(harness.db, { threadId: thread.id })).toHaveLength(4); + expect(notify).not.toHaveBeenCalled(); + harness.db + .update(threads) + .set({ status: "idle" }) + .where(eq(threads.id, thread.id)) + .run(); + for (let i = 0; i < 5; i++) await runThreadPruningSweep(harness.deps); + expect(notify).toHaveBeenCalledWith(thread.id, ["history-rewritten"]); + expect(build()).toEqual(before); + const raw = await harness.app.request( + `/api/v1/threads/${thread.id}/events?limit=1&afterSeq=1`, + ); + expect(raw.status).toBe(200); + const text = await raw.text(); + expect(text).toContain("item/completed"); + expect(text).not.toContain("completed_item_history"); + expect( + listEvents(harness.db, { threadId: thread.id }).map( + (row) => row.sequence, + ), + ).toEqual([1, 2, 4]); + }); + }); + it("prunes idle-thread noise rows and resolved item deltas", async () => { await withTestHarness(async (harness) => { const host = seedHost(harness.deps); diff --git a/apps/server/test/system/thread-pruning-sweep.test.ts b/apps/server/test/system/thread-pruning-sweep.test.ts index e6582d29933..981850d6295 100644 --- a/apps/server/test/system/thread-pruning-sweep.test.ts +++ b/apps/server/test/system/thread-pruning-sweep.test.ts @@ -7,6 +7,7 @@ import { getNextThreadPruningPolicy, getThreadEventRewriteGeneration, threadPruningCursors, + THREAD_PRUNING_POLICIES, threads, } from "@bb/db"; import { runThreadPruningSweep } from "../../src/services/system/thread-pruning-sweep.js"; @@ -182,7 +183,7 @@ describe("thread pruning sweep", () => { ); expect(steps.length).toBeLessThanOrEqual(64); expect(harness.db.select().from(threadPruningCursors).all()).toHaveLength( - 4, + THREAD_PRUNING_POLICIES.length, ); const before = getNextThreadPruningPolicy(harness.db, new Set()); expect(before).not.toBeNull(); diff --git a/docs/completed-item-compaction-verification.md b/docs/completed-item-compaction-verification.md new file mode 100644 index 00000000000..8da87374d0e --- /dev/null +++ b/docs/completed-item-compaction-verification.md @@ -0,0 +1,191 @@ +# Completed-item compaction verification + +Storage and maintenance measured on 2026-09-16 in an isolated worktree based on `5aca5733a5`, +including PR1 (`c663ff1911`, #3766). The final measurements include the +output-order safeguard and exclusion of compaction from synchronous per-thread +cleanup. These are observations from private sanitized SQLite copies. The reader section below supersedes the earlier 30-page timing samples and compares the rebased implementation with main. + +## Correctness and storage + +| Check | Result | +| ---------------------------------- | ------------------------------------------------------------------------------------------------------------------------- | +| Physical events | 2,082,639 → 1,080,655 (1,001,984 removed) | +| All table rows | 2,571,187 → 1,569,205 (net 1,001,982 removed) | +| Combined owners | 793,715 | +| Internal metadata | 182,920,304 bytes | +| Allocated SQLite btree bytes | 5,634,584,576 → 4,411,314,176 (1,223,270,400 saved) | +| Database file bytes | 6,006,726,656 before and after; freed pages are reusable | +| Independent logical reconstruction | All 2,082,639 records match across 2,326 threads, except 119,716 eligible empty command-delta text markers | +| Complete visible timelines | All 2,326 match, including all 2,223 non-null context values | +| Highwater and provider recovery | All 2,326 match | +| Existing side tables | All 46 non-event/non-cursor/non-migration tables match, including output, search and attachment ownership | +| Complete large-thread pagination | Three largest threads: all 3,816 rendered rows match after complete traversal at a 1,000-row budget; baseline used 10,000 | +| Other consumers | 2,326 latest-output checks, ten large outlines and 100 rollback-only output mutations match | + +There are no added tables or indexes and no full VACUUM. The two-row difference +between physical and total removals is migration/cursor bookkeeping. The worker +performs actual discovery and mutation on a fresh clone. + +The output-order safeguard leaves 93 more owners ordinary, retaining 175 rows +that the first revision removed. Assistant items crossing another assistant +completion or manager output must stay ordinary: otherwise moving the completion +can make the output API, plugin summaries and child notifications choose an older +message. Both cases were reproduced through the actual server output helper and +now pass regression tests. Existing user-message/context-clear exclusions and +bounded support/input discovery remain in place. + +## Timing and catch-up + +Heavy measurements ran sequentially. The host was shared; these observations do +not establish cold-I/O bounds or a maximum event-loop stall. + +| Measurement | Result | +| ------------------------------------------------------- | ------------------------------------------------- | +| Full completed-item worker pass | 120,866 advances; 304.50 seconds | +| Background advance median / p99 / maximum | 1.90 / 15.18 / 372.18 ms | +| Final live wrapper, 2,500 calls | median 0.288 ms; p99 5.46 ms; maximum 322.25 ms | +| PR1 live wrapper, same 2,500-call workload | median 0.243 ms; p99 5.28 ms; maximum 87.34 ms | +| Compaction calls / event deletions in final live sample | zero / zero | +| Actual background sweep, 100 calls | 1,565 total advances; 313 completed-item advances | +| Maximum sweep / advance in that sample | 216.46 / 177.78 ms | + +The original 270 ms live compaction stall was independently reproduced at 249 ms. +An isolated replay attributed the delay to transaction commit. Disabling automatic +checkpointing on a disposable copy reduced that compaction call to 1.53 ms; +an explicit checkpoint immediately afterwards took 333.05 ms for 1,002 WAL frames. +That experiment moved the cost; it was not a fix. No checkpoint settings change +in this PR. + +Completed-item compaction now runs only in the existing idle background sweep. +The synchronous per-thread rotation retains PR1's four policies. This removes +compaction writes from ingestion and turn-completion handlers, but does not +eliminate SQLite checkpoint stalls from existing writes. The final live maximum +occurred during the existing usage policy with zero event deletions; cursor +transactions still write. The PR1 comparison also exceeded 50 ms. These samples +do not prove identical worst-case latency or quantify a regression from the +individual maxima. No sub-50 ms guarantee is made. + +The live harness includes the real wrapper, transaction, generation checks and +in-process notification/logging calls. The background sample explicitly marks +its private copy idle. No server or provider runs on a research copy. + +At 3.13 completed-item advances per sweep and the existing ten-second cadence, +the latest sample extrapolates to about **107 idle hours** for first-pass catch-up. +The initial revision's sample estimated 84 hours. Actual processing time remains +about five minutes; scheduling, activity and I/O determine elapsed catch-up. +These are short-sample estimates, not an end-to-end scheduled run. Checkpoint +stalls and multi-day idle catch-up remain limitations. Reader performance was +subsequently fixed and measured separately below. + +## Automated and UI verification + +Earlier maintenance follow-up checks (reader-pass checks follow below): + +- Turbo DB tests: 44 files, 591 tests pass, using migrated real SQLite. +- Turbo affected server tests: 14 tests pass, including the previously failing + scheduler test and cached timeline refresh through the actual idle sweep. +- Live cleanup leaves the completion ordinary; idle background cleanup combines + it and invalidates the warmed timeline. Raw output still excludes metadata. +- Turbo DB/server typechecks pass. +- Full-copy comparisons and pagination listed above were rerun after the fixes. + +The initial revision additionally passed 200 selected server tests and 73 Plugin +Guide tests, small-budget traversal tests at budgets 1/2/5/20 with 512-byte +responses, 34 real fork cases (2,638 copied records), and 24 real edit-boundary +rewinds. Fresh synthetic Chromium verification proved command output, file diffs, +reasoning, answers and durations survive background rewriting and reload. A +rewrite with unchanged highwater refreshed the open browser automatically. Those +UI/fork/edit checks were not rerun in the follow-up; renderer, fork and edit code +are unchanged, and compaction eligibility is narrower. + +The initial verification inventory reported pre-existing unmapped `browser` +CLI-family drift. No iOS or provider-resume claim is made. Dev processes/browser +from the initial verification were stopped. + +Initial evidence remains under implementing thread `thr_mwh5k9hti7` in +`pr2-start-position`. Follow-up scripts, full comparisons, timings and logs are +under parent thread `thr_vmdgc3ke5y` in `pr2-review/final`. Copies are private and +mode 0600, with Connect plugin records verified absent. No live database or copied +Connect configuration was used; nothing was merged or deployed. + +## Full-copy reader performance (2026-09-17) + +Compared main `0bb64f3789` (including #3874 and #3876) with implementation +`820e4e5bfc`. Later main commits were checked for changes to the measured DB and +timeline paths. The source fix makes the timeline budget count reconstructed +records, selects metadata with the physical rows, passes already parsed payloads +to projection, and reuses reconstruction in a bounded per-connection cache. +Cache hits require identical stored metadata and completion payloads. Limits are +8 million accounted text characters and 10,000 reconstructed records; this is +not a byte-exact heap cap. There are still no new tables or indexes. + +The benchmark uses the complete sanitized pre-compaction database and its +compacted counterpart: all 2,326 threads, with every older cursor followed to the +end. Each request runs both versions and compares the complete response, +including pagination and context. Three alternating warm samples per version +follow the first call. The measured interval is the complete server timeline +builder, including selection, reconstruction and projection; it excludes HTTP, +network and browser rendering. SQLite uses production cache/mmap settings. + +| Workload | Matching pages | Main / PR2 median warm page time | Median paired change | Sum of warm page medians | +| --- | ---: | ---: | ---: | ---: | +| Product settings | 3,027 | 9.73 / 8.78 ms | −1.14 ms | 43.49 / 38.22 s (12.1% lower) | +| Expanded stress settings | 2,733 | 11.77 / 10.87 ms | −1.13 ms | 58.45 / 51.78 s (11.4% lower) | + +Product settings use the 1,500-event budget, 20 segments, lazy nested rows, +32,000 inline output characters, and saved provider display/diagnostic settings. +The stress workload uses a 10,000-event budget, expanded nested rows, collapsed +completed turns and 8,000 inline characters. These are two complete traversals +of the same database, not 5,760 distinct stored pages. Every response matches. +Neither census has a warm elapsed-time case more than both 5 ms and 10% slower. +Warm process CPU totals are 12.5% lower for product settings and 11.7% lower for +the stress workload. Among the 93 product-setting requests taking at least 50 ms +on main, the median paired improvement is 11.75 ms. + +First visits are noisier: product-setting median paired elapsed change is ++0.15 ms, with summed first-call time 7.2% higher but CPU 3.0% lower. Stress +first-call elapsed and CPU totals are lower. This does not establish that every +first read is faster. Cases crossing the 5 ms / 10% threshold in elapsed or CPU, +plus controls, were selected for repeated cold-application-cache and warm checks. +A cold application cache means a fresh DB wrapper and empty per-connection JS +caches, not a cold OS disk cache. + +All 457 selected product-setting requests and 78 stress requests match in six +alternating samples per version and cache mode. None exceeds both elapsed-time +thresholds in either mode. Median paired changes are −1.39 / −2.37 ms for product +cold/warm checks and −1.05 / −1.85 ms for stress cold/warm checks. Three CPU-only +outliers were repeated with twelve samples per version and mode; all three then +have lower PR2 elapsed and CPU medians. All original samples remain in the +artifacts rather than being discarded. + +The two originally blocking large-page regressions now measure 214.11 → 200.16 ms +and 191.56 → 151.21 ms in the repeated warm stress checks. Fresh application-cache +medians also improve: 214.22 → 193.46 ms and 204.57 → 185.67 ms. Their responses +remain identical. + +Before backfill, both versions were also run against the same uncompacted copy +for 45 requests from large threads. All responses match across nine alternating +samples per version and cache mode. Median paired elapsed overhead is +0.88 ms +with fresh application caches and +0.76 ms warm; no elapsed case exceeds both +5 ms and 10%. Summed warm elapsed medians increase 2.7%, and CPU medians 3.4%. +Two CPU-only cases cross the threshold in different cache modes; this is a small +pre-backfill cost, not a claim of zero overhead before any rows are compacted. + +The inherited benchmark mixed a CommonJS Drizzle driver with the app's ES-module +schema. Profiling exposed extra generic row-mapping overhead. The final harness +uses the same ES-module driver as `createConnection` and asserts matching driver +and native SQLite constructors. Only the corrected `production-*` artifacts +support these timing results; earlier timing samples are superseded. Earlier +response-equality checks are supplemented by the complete corrected traversals. + +Final source validation: 611 DB tests, 198 affected server tests, and DB/server +typechecks pass through Turbo. Added tests cover logical work budgets, metadata +selection, parsed payloads, malformed JSON, cache invalidation, scope/snapshot +filtering, eviction and oversized-entry bypass. Migration 0127 adds only the +metadata column; events indexes match the preceding main snapshot. + +These measurements address the observed reader regression. They do not establish +a maximum event-loop stall, a cold-disk latency bound, or a new catch-up estimate. +The historical maintenance/checkpoint measurements above remain separate. +Scripts, raw samples and the final report are in parent thread storage +`pr2-perf-fix/`. diff --git a/docs/completed-item-compaction.md b/docs/completed-item-compaction.md new file mode 100644 index 00000000000..9da40a7d4d2 --- /dev/null +++ b/docs/completed-item-compaction.md @@ -0,0 +1,80 @@ +# Completed item history compaction + +The existing idle background event-maintenance rotation combines eligible settled item history +into its `item/completed` row. The completion keeps its ID, payload, original +creation timestamp and retained-output ownership. Its physical sequence becomes +the first retained lifecycle sequence. One versioned internal column stores the +original completion sequence and lossless start/delta information. No table, +index, allocator, scheduler or daemon protocol change is involved. + +Eligibility requires a unique compatible completion, at most one start, a later +retained completion of the turn, and no late lifecycle events or reopened turn. +Supported kinds are command executions, file changes, assistant messages and +reasoning. Unpruned repeated delta streams, file-change output deltas, malformed +payloads, incompatible envelopes and oversized discovery/payload windows remain +ordinary. A lifetime crossing a user request or completed context clear remains +ordinary so message editing can keep its existing suffix-deletion behavior. +Assistant lifetimes crossing another assistant completion or manager output also +remain ordinary, preserving the output API and child completion notifications. + +Assistant and reasoning delta text is retained. A command delta can become an +empty timing marker only when the command has completed, failed or been +interrupted and has nonempty aggregated output. Without a retained start, the +aggregated output must contain that delta. Otherwise the entire lifecycle stays +ordinary. Starts never derive mutable output fields from a completion, so later +output truncation or expiry cannot change reconstructed starts. + +Discovery uses the existing physical sequence/type/turn/item indexes. Each +advance bounds candidate/support rows and input bytes; ambiguous or larger +lifecycles are skipped, not partially rewritten. Source deletion, completion +movement and cursor progress commit together. PR1's generation increment and +`history-rewritten` notification invalidate cached views after the commit. +Completed-item compaction is excluded from the synchronous per-thread cleanup +rotation used by ingestion and turn-completion handlers. Existing PR1 live +cleanup remains unchanged; the idle sweep performs compaction and refreshes +open clients. This avoids adding compaction writes to the live path. SQLite +checkpointing can still stall a background transaction (and unrelated live +writes); limits bound work rather than guaranteeing a maximum elapsed time. +No checkpoint setting is changed. + +Timeline queries select physical rows with their internal metadata, then reconstruct +only the selected history for projection. The timeline event budget charges for +the records inside each combined item, so removing physical rows does not cause +a page to decode substantially more history. Reconstruction passes validated +payload objects directly to projection instead of parsing them again. A per-connection +in-memory cache reuses reconstruction only when both stored metadata and the +completion payload match exactly. It evicts least-recently-used entries above +8 million accounted text characters or 10,000 reconstructed records, and does +not retain oversized entries. These are accounting limits, not a byte-exact +JavaScript heap limit. Current row scope and snapshot filtering are applied +after reuse; changed output or metadata forces reconstruction. Selected fork +history recovers completion order before applying the existing completed-turn and event-type rules. Forks still create +new IDs and sequence numbers. No arbitrary deleted-ID lookup or virtual raw-event +pagination is provided. + +## SDK, CLI and exports + +`threads.events.list`, the raw events HTTP route and `bb thread log --json` +(including `--all` JSON exports) return physical records. A combined row is still +`item/completed`; it has its original completion ID/payload/timestamp and its new +first lifecycle sequence. Internal metadata is not returned. Deleted start/delta +IDs, original raw counts and old completion positions are not retained as a raw +API contract. Consumers must not treat these exports as an exact provider wire +transcript. + +Physical-row limits and sequence cursors count combined rows. Restart a traversal +after a history rewrite rather than continuing an old cursor against changed +history. Human CLI formats and the UI reconstruct original ordering, timing, +text and edit cards. Raw export limits count physical rows; the timeline work +budget counts reconstructed records. Exact page boundaries are not promised. +Response byte limits remain independent of row limits, and stored timeline byte +accounting includes metadata. + +## Verification + +The migrated SQLite regression tests cover eligibility, lossless reconstruction, +command discard rules, output ownership, atomic rollback, late arrivals, +highwater, boundary exclusions and one-row raw traversal. Server tests exercise +the existing live wrapper, rewrite notifications and warmed timeline caches. +Full-copy measurements and UI evidence accompany the PR; prototype +measurements are not implementation guarantees. diff --git a/packages/db/drizzle/0127_plain_cammi.sql b/packages/db/drizzle/0127_plain_cammi.sql new file mode 100644 index 00000000000..04d4ca8405f --- /dev/null +++ b/packages/db/drizzle/0127_plain_cammi.sql @@ -0,0 +1 @@ +ALTER TABLE `events` ADD `completed_item_history` text; \ No newline at end of file diff --git a/packages/db/drizzle/meta/0127_snapshot.json b/packages/db/drizzle/meta/0127_snapshot.json new file mode 100644 index 00000000000..d58783f160f --- /dev/null +++ b/packages/db/drizzle/meta/0127_snapshot.json @@ -0,0 +1,5008 @@ +{ + "version": "6", + "dialect": "sqlite", + "id": "7d1de893-ff6d-445e-bcd2-6fbbe311d502", + "prevId": "48979c46-bc2b-411d-a6cc-18dfcb7cd457", + "tables": { + "app_settings": { + "name": "app_settings", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "caffeinate": { + "name": "caffeinate", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "show_keyboard_hints": { + "name": "show_keyboard_hints", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": true + }, + "steer_active_thread_on_enter": { + "name": "steer_active_thread_on_enter", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "show_unhandled_provider_events": { + "name": "show_unhandled_provider_events", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "codex_memory_enabled": { + "name": "codex_memory_enabled", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": true + }, + "claude_code_memory_enabled": { + "name": "claude_code_memory_enabled", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": true + }, + "codex_subagents_disabled": { + "name": "codex_subagents_disabled", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "claude_code_subagents_disabled": { + "name": "claude_code_subagents_disabled", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "claude_code_workflows_disabled": { + "name": "claude_code_workflows_disabled", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "keybinding_overrides": { + "name": "keybinding_overrides", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'[]'" + }, + "onboarding_completed_at": { + "name": "onboarding_completed_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "app_settings_values": { + "name": "app_settings_values", + "columns": { + "key": { + "name": "key", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "value": { + "name": "value", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "app_theme": { + "name": "app_theme", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "theme_id": { + "name": "theme_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "favicon_color": { + "name": "favicon_color", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'default'" + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "apikey": { + "name": "apikey", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "start": { + "name": "start", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "prefix": { + "name": "prefix", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "key": { + "name": "key", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "referenceId": { + "name": "referenceId", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "refillInterval": { + "name": "refillInterval", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "refillAmount": { + "name": "refillAmount", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "lastRefillAt": { + "name": "lastRefillAt", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "enabled": { + "name": "enabled", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "rateLimitEnabled": { + "name": "rateLimitEnabled", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "rateLimitTimeWindow": { + "name": "rateLimitTimeWindow", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "rateLimitMax": { + "name": "rateLimitMax", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "requestCount": { + "name": "requestCount", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "remaining": { + "name": "remaining", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "lastRequest": { + "name": "lastRequest", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "expiresAt": { + "name": "expiresAt", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updatedAt": { + "name": "updatedAt", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "permissions": { + "name": "permissions", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "metadata": { + "name": "metadata", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "configId": { + "name": "configId", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "apikey_key_unique": { + "name": "apikey_key_unique", + "columns": [ + "key" + ], + "isUnique": true + }, + "apikey_reference_id_idx": { + "name": "apikey_reference_id_idx", + "columns": [ + "referenceId" + ], + "isUnique": false + }, + "apikey_config_id_idx": { + "name": "apikey_config_id_idx", + "columns": [ + "configId" + ], + "isUnique": false + } + }, + "foreignKeys": { + "apikey_referenceId_user_id_fk": { + "name": "apikey_referenceId_user_id_fk", + "tableFrom": "apikey", + "tableTo": "user", + "columnsFrom": [ + "referenceId" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "user": { + "name": "user", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "email": { + "name": "email", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "emailVerified": { + "name": "emailVerified", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "image": { + "name": "image", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updatedAt": { + "name": "updatedAt", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "user_email_unique": { + "name": "user_email_unique", + "columns": [ + "email" + ], + "isUnique": true + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "environment_hook_operations": { + "name": "environment_hook_operations", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "operation_id": { + "name": "operation_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "host_id": { + "name": "host_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "path": { + "name": "path", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "started_at": { + "name": "started_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "finished_at": { + "name": "finished_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "error": { + "name": "error", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "environment_variables": { + "name": "environment_variables", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": true + }, + "project_id": { + "name": "project_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "ciphertext": { + "name": "ciphertext", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "encryption_version": { + "name": "encryption_version", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "note": { + "name": "note", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "environment_variables_global_name": { + "name": "environment_variables_global_name", + "columns": [ + "name" + ], + "isUnique": true, + "where": "\"environment_variables\".\"project_id\" IS NULL" + }, + "environment_variables_project_name": { + "name": "environment_variables_project_name", + "columns": [ + "project_id", + "name" + ], + "isUnique": true, + "where": "\"environment_variables\".\"project_id\" IS NOT NULL" + } + }, + "foreignKeys": { + "environment_variables_project_id_projects_id_fk": { + "name": "environment_variables_project_id_projects_id_fk", + "tableFrom": "environment_variables", + "tableTo": "projects", + "columnsFrom": [ + "project_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "environments": { + "name": "environments", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "project_id": { + "name": "project_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "host_id": { + "name": "host_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "path": { + "name": "path", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "is_git_repo": { + "name": "is_git_repo", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "is_worktree": { + "name": "is_worktree", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "branch_name": { + "name": "branch_name", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "base_branch": { + "name": "base_branch", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "default_branch": { + "name": "default_branch", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "merge_base_branch": { + "name": "merge_base_branch", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "environment_provider_id": { + "name": "environment_provider_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "environment_provider_plugin_id": { + "name": "environment_provider_plugin_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "provider_owns_path": { + "name": "provider_owns_path", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "environment_provider_selection": { + "name": "environment_provider_selection", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "environment_provider_instance_key": { + "name": "environment_provider_instance_key", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "retire_at": { + "name": "retire_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "teardown_attempt": { + "name": "teardown_attempt", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "teardown_status": { + "name": "teardown_status", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "teardown_message": { + "name": "teardown_message", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "resource": { + "name": "resource", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "owner_thread_id": { + "name": "owner_thread_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "attempt": { + "name": "attempt", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "status_message": { + "name": "status_message", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "pending_log": { + "name": "pending_log", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "''" + }, + "claim_path": { + "name": "claim_path", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'provisioning'" + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "environments_project_host_path_idx": { + "name": "environments_project_host_path_idx", + "columns": [ + "project_id", + "host_id", + "path" + ], + "isUnique": true + }, + "environments_host_path_lookup_idx": { + "name": "environments_host_path_lookup_idx", + "columns": [ + "host_id", + "path" + ], + "isUnique": false + }, + "environments_owner_thread_idx": { + "name": "environments_owner_thread_idx", + "columns": [ + "owner_thread_id" + ], + "isUnique": true + }, + "environments_claim_idx": { + "name": "environments_claim_idx", + "columns": [ + "host_id", + "claim_path" + ], + "isUnique": false + }, + "environments_project_idx": { + "name": "environments_project_idx", + "columns": [ + "project_id" + ], + "isUnique": false + }, + "environments_status_idx": { + "name": "environments_status_idx", + "columns": [ + "status" + ], + "isUnique": false + }, + "environments_provider_instance_idx": { + "name": "environments_provider_instance_idx", + "columns": [ + "environment_provider_id", + "environment_provider_instance_key" + ], + "isUnique": false + } + }, + "foreignKeys": { + "environments_project_id_projects_id_fk": { + "name": "environments_project_id_projects_id_fk", + "tableFrom": "environments", + "tableTo": "projects", + "columnsFrom": [ + "project_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "environments_host_id_hosts_id_fk": { + "name": "environments_host_id_hosts_id_fk", + "tableFrom": "environments", + "tableTo": "hosts", + "columnsFrom": [ + "host_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "events": { + "name": "events", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "thread_id": { + "name": "thread_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "environment_id": { + "name": "environment_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "scope_kind": { + "name": "scope_kind", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "turn_id": { + "name": "turn_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "provider_thread_id": { + "name": "provider_thread_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "sequence": { + "name": "sequence", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "item_id": { + "name": "item_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "item_kind": { + "name": "item_kind", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "parent_tool_call_id": { + "name": "parent_tool_call_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "data": { + "name": "data", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'{}'" + }, + "completed_item_history": { + "name": "completed_item_history", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "events_thread_sequence_idx": { + "name": "events_thread_sequence_idx", + "columns": [ + "thread_id", + "sequence" + ], + "isUnique": true + }, + "events_delegating_item_lookup_idx": { + "name": "events_delegating_item_lookup_idx", + "columns": [ + "thread_id", + "item_id", + "sequence", + "item_kind" + ], + "isUnique": false, + "where": "\"events\".\"item_kind\" IN ('toolCall', 'delegation')" + }, + "events_plan_steps_thread_sequence_idx": { + "name": "events_plan_steps_thread_sequence_idx", + "columns": [ + "thread_id", + "sequence" + ], + "isUnique": false, + "where": "(\"events\".\"item_kind\" = 'planSteps' AND \"events\".\"type\" = 'item/completed') OR \"events\".\"type\" = 'turn/plan/updated'" + }, + "events_parent_tool_call_thread_parent_sequence_idx": { + "name": "events_parent_tool_call_thread_parent_sequence_idx", + "columns": [ + "thread_id", + "parent_tool_call_id", + "sequence" + ], + "isUnique": false, + "where": "\"events\".\"parent_tool_call_id\" IS NOT NULL" + }, + "events_thread_type_item_kind_sequence_idx": { + "name": "events_thread_type_item_kind_sequence_idx", + "columns": [ + "thread_id", + "type", + "item_kind", + "sequence" + ], + "isUnique": false + }, + "events_background_task_thread_type_item_sequence_idx": { + "name": "events_background_task_thread_type_item_sequence_idx", + "columns": [ + "thread_id", + "type", + "item_id", + "sequence" + ], + "isUnique": false, + "where": "\"events\".\"item_kind\" = 'backgroundTask'" + }, + "events_thread_type_sequence_idx": { + "name": "events_thread_type_sequence_idx", + "columns": [ + "thread_id", + "type", + "sequence" + ], + "isUnique": false + }, + "events_thread_turn_type_item_sequence_idx": { + "name": "events_thread_turn_type_item_sequence_idx", + "columns": [ + "thread_id", + "turn_id", + "type", + "item_id", + "sequence" + ], + "isUnique": false + }, + "events_item_lifecycle_thread_item_sequence_idx": { + "name": "events_item_lifecycle_thread_item_sequence_idx", + "columns": [ + "thread_id", + "item_id", + "sequence" + ], + "isUnique": false, + "where": "\"events\".\"type\" IN ('item/started', 'item/completed', 'item/backgroundTask/completed')" + }, + "events_environment_idx": { + "name": "events_environment_idx", + "columns": [ + "environment_id" + ], + "isUnique": false + }, + "events_provider_identity_idx": { + "name": "events_provider_identity_idx", + "columns": [ + "provider_thread_id", + "created_at" + ], + "isUnique": false, + "where": "\"events\".\"type\" = 'thread/identity'" + }, + "events_completed_item_truncation_idx": { + "name": "events_completed_item_truncation_idx", + "columns": [ + "item_kind", + "created_at", + "id" + ], + "isUnique": false, + "where": "\"events\".\"type\" = 'item/completed'" + }, + "events_thread_state_thread_sequence_idx": { + "name": "events_thread_state_thread_sequence_idx", + "columns": [ + "thread_id", + "sequence" + ], + "isUnique": false, + "where": "\"events\".\"type\" IN ('thread/goal/updated', 'thread/goal/cleared', 'thread/extensionState/updated')" + } + }, + "foreignKeys": { + "events_thread_id_threads_id_fk": { + "name": "events_thread_id_threads_id_fk", + "tableFrom": "events", + "tableTo": "threads", + "columnsFrom": [ + "thread_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "events_environment_id_environments_id_fk": { + "name": "events_environment_id_environments_id_fk", + "tableFrom": "events", + "tableTo": "environments", + "columnsFrom": [ + "environment_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": { + "events_scope_shape_check": { + "name": "events_scope_shape_check", + "value": "(\n (\"events\".\"scope_kind\" = 'turn' AND \"events\".\"turn_id\" IS NOT NULL)\n OR\n (\"events\".\"scope_kind\" = 'thread' AND \"events\".\"turn_id\" IS NULL)\n )" + } + } + }, + "host_daemon_sessions": { + "name": "host_daemon_sessions", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "host_id": { + "name": "host_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "instance_id": { + "name": "instance_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "host_name": { + "name": "host_name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "data_dir": { + "name": "data_dir", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "protocol_version": { + "name": "protocol_version", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "heartbeat_interval_ms": { + "name": "heartbeat_interval_ms", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "lease_timeout_ms": { + "name": "lease_timeout_ms", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "lease_expires_at": { + "name": "lease_expires_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "closed_at": { + "name": "closed_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "close_reason": { + "name": "close_reason", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "host_daemon_sessions_host_status_idx": { + "name": "host_daemon_sessions_host_status_idx", + "columns": [ + "host_id", + "status" + ], + "isUnique": false + }, + "host_daemon_sessions_host_latest_idx": { + "name": "host_daemon_sessions_host_latest_idx", + "columns": [ + "host_id", + "updated_at", + "created_at", + "id" + ], + "isUnique": false + }, + "host_daemon_sessions_closed_prune_idx": { + "name": "host_daemon_sessions_closed_prune_idx", + "columns": [ + "status", + "closed_at", + "id" + ], + "isUnique": false + } + }, + "foreignKeys": { + "host_daemon_sessions_host_id_hosts_id_fk": { + "name": "host_daemon_sessions_host_id_hosts_id_fk", + "tableFrom": "host_daemon_sessions", + "tableTo": "hosts", + "columnsFrom": [ + "host_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "hosts": { + "name": "hosts", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "connect_machine_id": { + "name": "connect_machine_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "machine_provider_id": { + "name": "machine_provider_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "launch_key": { + "name": "launch_key", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "machine_inputs": { + "name": "machine_inputs", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "machine_attempt": { + "name": "machine_attempt", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "pending_log": { + "name": "pending_log", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "''" + }, + "machine_operation_id": { + "name": "machine_operation_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "server_access_provider_id": { + "name": "server_access_provider_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "server_access_grant_id": { + "name": "server_access_grant_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "resource": { + "name": "resource", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "phase": { + "name": "phase", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'active'" + }, + "suspended_at": { + "name": "suspended_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "status_message": { + "name": "status_message", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "suspend_retry_at": { + "name": "suspend_retry_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "remove_retry_at": { + "name": "remove_retry_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "teardown_attempt": { + "name": "teardown_attempt", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "teardown_status": { + "name": "teardown_status", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "max_permission_mode": { + "name": "max_permission_mode", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'full'" + }, + "destroyed_at": { + "name": "destroyed_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "last_seen_at": { + "name": "last_seen_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "last_rejected_protocol_version": { + "name": "last_rejected_protocol_version", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "hosts_last_seen_idx": { + "name": "hosts_last_seen_idx", + "columns": [ + "last_seen_at" + ], + "isUnique": false + }, + "hosts_live_launch_key_idx": { + "name": "hosts_live_launch_key_idx", + "columns": [ + "launch_key" + ], + "isUnique": true, + "where": "\"hosts\".\"destroyed_at\" is null" + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "plugins": { + "name": "plugins", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "source": { + "name": "source", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "provenance": { + "name": "provenance", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'direct'" + }, + "catalog_entry_id": { + "name": "catalog_entry_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "catalog_marketplace_name": { + "name": "catalog_marketplace_name", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "source_kind": { + "name": "source_kind", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'path'" + }, + "source_path": { + "name": "source_path", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "source_builtin_name": { + "name": "source_builtin_name", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "source_npm_package": { + "name": "source_npm_package", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "source_npm_registry": { + "name": "source_npm_registry", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "source_npm_requested_spec": { + "name": "source_npm_requested_spec", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "source_npm_spec_kind": { + "name": "source_npm_spec_kind", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "source_git_url": { + "name": "source_git_url", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "source_git_subdirectory": { + "name": "source_git_subdirectory", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "source_git_requested_ref": { + "name": "source_git_requested_ref", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "source_git_ref_kind": { + "name": "source_git_ref_kind", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "source_git_range": { + "name": "source_git_range", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "source_git_tag_prefix": { + "name": "source_git_tag_prefix", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "source_git_resolved_tag": { + "name": "source_git_resolved_tag", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "npm_resolved_version": { + "name": "npm_resolved_version", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "npm_integrity": { + "name": "npm_integrity", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "git_resolved_commit": { + "name": "git_resolved_commit", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "last_update_check_at": { + "name": "last_update_check_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "available_compatible_version": { + "name": "available_compatible_version", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "newest_incompatible_version": { + "name": "newest_incompatible_version", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "update_status_detail": { + "name": "update_status_detail", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "last_failure_version": { + "name": "last_failure_version", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "last_failure_at": { + "name": "last_failure_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "last_failure_detail": { + "name": "last_failure_detail", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "active_artifact_id": { + "name": "active_artifact_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "normalization_version": { + "name": "normalization_version", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "root_dir": { + "name": "root_dir", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "version": { + "name": "version", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "enabled": { + "name": "enabled", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": true + }, + "removed_at": { + "name": "removed_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "installed_at": { + "name": "installed_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": { + "plugins_active_artifact_id_plugin_artifacts_id_fk": { + "name": "plugins_active_artifact_id_plugin_artifacts_id_fk", + "tableFrom": "plugins", + "tableTo": "plugin_artifacts", + "columnsFrom": [ + "active_artifact_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "maintenance_scan_cursors": { + "name": "maintenance_scan_cursors", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "policy": { + "name": "policy", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "version": { + "name": "version", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "item_kind": { + "name": "item_kind", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "output_path": { + "name": "output_path", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "last_created_at": { + "name": "last_created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "last_event_id": { + "name": "last_event_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "''" + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "maintenance_scan_cursors_path_idx": { + "name": "maintenance_scan_cursors_path_idx", + "columns": [ + "policy", + "version", + "item_kind", + "output_path" + ], + "isUnique": true + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "pending_interactions": { + "name": "pending_interactions", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "thread_id": { + "name": "thread_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "origin_kind": { + "name": "origin_kind", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'provider'" + }, + "turn_id": { + "name": "turn_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "provider_id": { + "name": "provider_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "provider_thread_id": { + "name": "provider_thread_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "provider_request_id": { + "name": "provider_request_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "plugin_id": { + "name": "plugin_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "renderer_id": { + "name": "renderer_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "payload": { + "name": "payload", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "resolution": { + "name": "resolution", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "status_reason": { + "name": "status_reason", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "expires_at": { + "name": "expires_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "resolved_at": { + "name": "resolved_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "pending_interactions_provider_request_idx": { + "name": "pending_interactions_provider_request_idx", + "columns": [ + "provider_id", + "provider_thread_id", + "provider_request_id" + ], + "isUnique": true + }, + "pending_interactions_thread_created_idx": { + "name": "pending_interactions_thread_created_idx", + "columns": [ + "thread_id", + "created_at" + ], + "isUnique": false + }, + "pending_interactions_thread_status_created_idx": { + "name": "pending_interactions_thread_status_created_idx", + "columns": [ + "thread_id", + "status", + "created_at" + ], + "isUnique": false + }, + "pending_interactions_status_created_idx": { + "name": "pending_interactions_status_created_idx", + "columns": [ + "status", + "created_at" + ], + "isUnique": false + }, + "pending_interactions_plugin_status_created_idx": { + "name": "pending_interactions_plugin_status_created_idx", + "columns": [ + "plugin_id", + "status", + "created_at" + ], + "isUnique": false + } + }, + "foreignKeys": { + "pending_interactions_thread_id_threads_id_fk": { + "name": "pending_interactions_thread_id_threads_id_fk", + "tableFrom": "pending_interactions", + "tableTo": "threads", + "columnsFrom": [ + "thread_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "plugin_artifacts": { + "name": "plugin_artifacts", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "plugin_id": { + "name": "plugin_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "source_kind": { + "name": "source_kind", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "npm_resolved_version": { + "name": "npm_resolved_version", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "git_resolved_commit": { + "name": "git_resolved_commit", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "git_checkout_root": { + "name": "git_checkout_root", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "path": { + "name": "path", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "integrity": { + "name": "integrity", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "content_hash": { + "name": "content_hash", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "validation_result": { + "name": "validation_result", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "validated_at": { + "name": "validated_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "plugin_artifacts_plugin_idx": { + "name": "plugin_artifacts_plugin_idx", + "columns": [ + "plugin_id" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "plugin_kv": { + "name": "plugin_kv", + "columns": { + "plugin_id": { + "name": "plugin_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "key": { + "name": "key", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "value": { + "name": "value", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": { + "plugin_kv_plugin_id_key_pk": { + "columns": [ + "plugin_id", + "key" + ], + "name": "plugin_kv_plugin_id_key_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "plugin_marketplace_icons": { + "name": "plugin_marketplace_icons", + "columns": { + "marketplace_name": { + "name": "marketplace_name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "entry_id": { + "name": "entry_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "source_url": { + "name": "source_url", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "content_type": { + "name": "content_type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "etag": { + "name": "etag", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "content_hash": { + "name": "content_hash", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "bytes": { + "name": "bytes", + "type": "blob", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": { + "plugin_marketplace_icons_marketplace_name_entry_id_pk": { + "columns": [ + "marketplace_name", + "entry_id" + ], + "name": "plugin_marketplace_icons_marketplace_name_entry_id_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "plugin_marketplaces": { + "name": "plugin_marketplaces", + "columns": { + "name": { + "name": "name", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "source_kind": { + "name": "source_kind", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'https'" + }, + "manifest_url": { + "name": "manifest_url", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "source_git_ref": { + "name": "source_git_ref", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "source_git_commit": { + "name": "source_git_commit", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "manifest_json": { + "name": "manifest_json", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "stats_json": { + "name": "stats_json", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "etag": { + "name": "etag", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "last_modified": { + "name": "last_modified", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "last_successful_refresh_at": { + "name": "last_successful_refresh_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "last_attempted_refresh_at": { + "name": "last_attempted_refresh_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "last_error": { + "name": "last_error", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "plugin_schedules": { + "name": "plugin_schedules", + "columns": { + "plugin_id": { + "name": "plugin_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "cron": { + "name": "cron", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "next_run_at": { + "name": "next_run_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "last_run_at": { + "name": "last_run_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "last_status": { + "name": "last_status", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "last_error": { + "name": "last_error", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": { + "plugin_schedules_plugin_id_name_pk": { + "columns": [ + "plugin_id", + "name" + ], + "name": "plugin_schedules_plugin_id_name_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "plugin_settings": { + "name": "plugin_settings", + "columns": { + "plugin_id": { + "name": "plugin_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "key": { + "name": "key", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "value": { + "name": "value", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": { + "plugin_settings_plugin_id_key_pk": { + "columns": [ + "plugin_id", + "key" + ], + "name": "plugin_settings_plugin_id_key_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "plugin_state_snapshots": { + "name": "plugin_state_snapshots", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "plugin_id": { + "name": "plugin_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "from_artifact_id": { + "name": "from_artifact_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "to_artifact_id": { + "name": "to_artifact_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "snapshot_path": { + "name": "snapshot_path", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "database_path": { + "name": "database_path", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "state_path": { + "name": "state_path", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "secrets_path": { + "name": "secrets_path", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "registration_path": { + "name": "registration_path", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "rollback_candidate_version": { + "name": "rollback_candidate_version", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "rollback_source_fingerprint": { + "name": "rollback_source_fingerprint", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "rollback_bb_version": { + "name": "rollback_bb_version", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "rollback_sdk_version": { + "name": "rollback_sdk_version", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "rollback_detail": { + "name": "rollback_detail", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "retained_until": { + "name": "retained_until", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "plugin_state_snapshots_plugin_idx": { + "name": "plugin_state_snapshots_plugin_idx", + "columns": [ + "plugin_id" + ], + "isUnique": false + }, + "plugin_state_snapshots_retention_idx": { + "name": "plugin_state_snapshots_retention_idx", + "columns": [ + "retained_until" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "project_attachment_backfills": { + "name": "project_attachment_backfills", + "columns": { + "project_id": { + "name": "project_id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "phase": { + "name": "phase", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "thread_cursor": { + "name": "thread_cursor", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "input_cursor": { + "name": "input_cursor", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "input_id": { + "name": "input_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "input_sequence": { + "name": "input_sequence", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "attempted_at": { + "name": "attempted_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "error": { + "name": "error", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": { + "project_attachment_backfills_project_id_projects_id_fk": { + "name": "project_attachment_backfills_project_id_projects_id_fk", + "tableFrom": "project_attachment_backfills", + "tableTo": "projects", + "columnsFrom": [ + "project_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "project_attachment_threads": { + "name": "project_attachment_threads", + "columns": { + "attachment_id": { + "name": "attachment_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "thread_id": { + "name": "thread_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "project_attachment_threads_thread_idx": { + "name": "project_attachment_threads_thread_idx", + "columns": [ + "thread_id" + ], + "isUnique": false + } + }, + "foreignKeys": { + "project_attachment_threads_attachment_id_project_attachments_id_fk": { + "name": "project_attachment_threads_attachment_id_project_attachments_id_fk", + "tableFrom": "project_attachment_threads", + "tableTo": "project_attachments", + "columnsFrom": [ + "attachment_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "project_attachment_threads_thread_id_threads_id_fk": { + "name": "project_attachment_threads_thread_id_threads_id_fk", + "tableFrom": "project_attachment_threads", + "tableTo": "threads", + "columnsFrom": [ + "thread_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "project_attachment_threads_attachment_id_thread_id_pk": { + "columns": [ + "attachment_id", + "thread_id" + ], + "name": "project_attachment_threads_attachment_id_thread_id_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "project_attachments": { + "name": "project_attachments", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "project_id": { + "name": "project_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "stored_path": { + "name": "stored_path", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "original_name": { + "name": "original_name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "mime_type": { + "name": "mime_type", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "size_bytes": { + "name": "size_bytes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "ready_at": { + "name": "ready_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "deletion_claimed_at": { + "name": "deletion_claimed_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "project_attachments_project_path_idx": { + "name": "project_attachments_project_path_idx", + "columns": [ + "project_id", + "stored_path" + ], + "isUnique": true + }, + "project_attachments_project_created_idx": { + "name": "project_attachments_project_created_idx", + "columns": [ + "project_id", + "created_at" + ], + "isUnique": false + }, + "project_attachments_deletion_idx": { + "name": "project_attachments_deletion_idx", + "columns": [ + "project_id", + "deletion_claimed_at", + "id" + ], + "isUnique": false, + "where": "\"project_attachments\".\"deletion_claimed_at\" IS NOT NULL" + } + }, + "foreignKeys": { + "project_attachments_project_id_projects_id_fk": { + "name": "project_attachments_project_id_projects_id_fk", + "tableFrom": "project_attachments", + "tableTo": "projects", + "columnsFrom": [ + "project_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": { + "project_attachments_size_check": { + "name": "project_attachments_size_check", + "value": "\"project_attachments\".\"size_bytes\" >= 0" + } + } + }, + "project_execution_defaults": { + "name": "project_execution_defaults", + "columns": { + "project_id": { + "name": "project_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "provider_id": { + "name": "provider_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "model": { + "name": "model", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "service_tier": { + "name": "service_tier", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "reasoning_level": { + "name": "reasoning_level", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "permission_mode": { + "name": "permission_mode", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "project_execution_defaults_project_idx": { + "name": "project_execution_defaults_project_idx", + "columns": [ + "project_id" + ], + "isUnique": true + } + }, + "foreignKeys": { + "project_execution_defaults_project_id_projects_id_fk": { + "name": "project_execution_defaults_project_id_projects_id_fk", + "tableFrom": "project_execution_defaults", + "tableTo": "projects", + "columnsFrom": [ + "project_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "project_sources": { + "name": "project_sources", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "project_id": { + "name": "project_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "host_id": { + "name": "host_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "path": { + "name": "path", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "owns_path": { + "name": "owns_path", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "is_default": { + "name": "is_default", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "project_sources_project_idx": { + "name": "project_sources_project_idx", + "columns": [ + "project_id" + ], + "isUnique": false + }, + "project_sources_host_idx": { + "name": "project_sources_host_idx", + "columns": [ + "host_id" + ], + "isUnique": false + }, + "project_sources_project_host_idx": { + "name": "project_sources_project_host_idx", + "columns": [ + "project_id", + "host_id" + ], + "isUnique": true + } + }, + "foreignKeys": { + "project_sources_project_id_projects_id_fk": { + "name": "project_sources_project_id_projects_id_fk", + "tableFrom": "project_sources", + "tableTo": "projects", + "columnsFrom": [ + "project_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "project_sources_host_id_hosts_id_fk": { + "name": "project_sources_host_id_hosts_id_fk", + "tableFrom": "project_sources", + "tableTo": "hosts", + "columnsFrom": [ + "host_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": { + "project_sources_shape_check": { + "name": "project_sources_shape_check", + "value": "(\n \"project_sources\".\"type\" = 'local_path' AND \"project_sources\".\"host_id\" IS NOT NULL AND \"project_sources\".\"path\" IS NOT NULL\n )" + } + } + }, + "projects": { + "name": "projects", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'standard'" + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "git_remote_url": { + "name": "git_remote_url", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "sort_key": { + "name": "sort_key", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'V'" + }, + "deleted_at": { + "name": "deleted_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "projects_updated_idx": { + "name": "projects_updated_idx", + "columns": [ + "updated_at" + ], + "isUnique": false + }, + "projects_deleted_idx": { + "name": "projects_deleted_idx", + "columns": [ + "deleted_at" + ], + "isUnique": false + }, + "projects_sort_idx": { + "name": "projects_sort_idx", + "columns": [ + "sort_key", + "id" + ], + "isUnique": false + }, + "projects_personal_singleton_idx": { + "name": "projects_personal_singleton_idx", + "columns": [ + "kind" + ], + "isUnique": true, + "where": "\"projects\".\"kind\" = 'personal'" + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "prompt_history_entries": { + "name": "prompt_history_entries", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "project_id": { + "name": "project_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "thread_id": { + "name": "thread_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "scope": { + "name": "scope", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "request_sequence": { + "name": "request_sequence", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "input": { + "name": "input", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "prompt_history_entries_thread_request_idx": { + "name": "prompt_history_entries_thread_request_idx", + "columns": [ + "thread_id", + "request_sequence" + ], + "isUnique": true + }, + "prompt_history_entries_project_scope_created_idx": { + "name": "prompt_history_entries_project_scope_created_idx", + "columns": [ + "project_id", + "scope", + "created_at", + "request_sequence", + "id" + ], + "isUnique": false + }, + "prompt_history_entries_thread_scope_created_idx": { + "name": "prompt_history_entries_thread_scope_created_idx", + "columns": [ + "thread_id", + "scope", + "created_at", + "request_sequence", + "id" + ], + "isUnique": false + } + }, + "foreignKeys": { + "prompt_history_entries_project_id_projects_id_fk": { + "name": "prompt_history_entries_project_id_projects_id_fk", + "tableFrom": "prompt_history_entries", + "tableTo": "projects", + "columnsFrom": [ + "project_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "prompt_history_entries_thread_id_threads_id_fk": { + "name": "prompt_history_entries_thread_id_threads_id_fk", + "tableFrom": "prompt_history_entries", + "tableTo": "threads", + "columnsFrom": [ + "thread_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "provider_model_catalogs": { + "name": "provider_model_catalogs", + "columns": { + "host_id": { + "name": "host_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "provider_id": { + "name": "provider_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "scope_key": { + "name": "scope_key", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "fingerprint": { + "name": "fingerprint", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "models_json": { + "name": "models_json", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "selected_only_models_json": { + "name": "selected_only_models_json", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "fetched_at": { + "name": "fetched_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": { + "provider_model_catalogs_host_id_hosts_id_fk": { + "name": "provider_model_catalogs_host_id_hosts_id_fk", + "tableFrom": "provider_model_catalogs", + "tableTo": "hosts", + "columnsFrom": [ + "host_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "provider_model_catalogs_host_id_provider_id_scope_key_pk": { + "columns": [ + "host_id", + "provider_id", + "scope_key" + ], + "name": "provider_model_catalogs_host_id_provider_id_scope_key_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "queued_thread_messages": { + "name": "queued_thread_messages", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "system_notice": { + "name": "system_notice", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "thread_id": { + "name": "thread_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "content": { + "name": "content", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "sender_thread_id": { + "name": "sender_thread_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "origin": { + "name": "origin", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "origin_plugin_id": { + "name": "origin_plugin_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "requested_by_initiator": { + "name": "requested_by_initiator", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "requested_by_thread_id": { + "name": "requested_by_thread_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "model": { + "name": "model", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "reasoning_level": { + "name": "reasoning_level", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "permission_mode": { + "name": "permission_mode", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "service_tier": { + "name": "service_tier", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "group_with_next": { + "name": "group_with_next", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "send_at": { + "name": "send_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "waiting_on": { + "name": "waiting_on", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "wait_holder": { + "name": "wait_holder", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "failure_reason": { + "name": "failure_reason", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "payload_kind": { + "name": "payload_kind", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'inline'" + }, + "retry_of_turn_request_id": { + "name": "retry_of_turn_request_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "retry_attempt": { + "name": "retry_attempt", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "retry_reason": { + "name": "retry_reason", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "claimed_at": { + "name": "claimed_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "claim_token": { + "name": "claim_token", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "sort_key": { + "name": "sort_key", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "queued_thread_messages_thread_created_idx": { + "name": "queued_thread_messages_thread_created_idx", + "columns": [ + "thread_id", + "created_at", + "id" + ], + "isUnique": false + }, + "queued_thread_messages_thread_sort_idx": { + "name": "queued_thread_messages_thread_sort_idx", + "columns": [ + "thread_id", + "sort_key", + "id" + ], + "isUnique": false + }, + "queued_thread_messages_due_idx": { + "name": "queued_thread_messages_due_idx", + "columns": [ + "send_at", + "id" + ], + "isUnique": false, + "where": "\"queued_thread_messages\".\"send_at\" IS NOT NULL AND \"queued_thread_messages\".\"claimed_at\" IS NULL AND \"queued_thread_messages\".\"claim_token\" IS NULL" + }, + "queued_thread_messages_wait_holder_idx": { + "name": "queued_thread_messages_wait_holder_idx", + "columns": [ + "wait_holder", + "id" + ], + "isUnique": false, + "where": "\"queued_thread_messages\".\"wait_holder\" IS NOT NULL" + } + }, + "foreignKeys": { + "queued_thread_messages_thread_id_threads_id_fk": { + "name": "queued_thread_messages_thread_id_threads_id_fk", + "tableFrom": "queued_thread_messages", + "tableTo": "threads", + "columnsFrom": [ + "thread_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "retained_event_outputs": { + "name": "retained_event_outputs", + "columns": { + "event_id": { + "name": "event_id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "output_path": { + "name": "output_path", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "value": { + "name": "value", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "expires_at": { + "name": "expires_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "retained_event_outputs_expiry_idx": { + "name": "retained_event_outputs_expiry_idx", + "columns": [ + "expires_at", + "event_id" + ], + "isUnique": false + } + }, + "foreignKeys": { + "retained_event_outputs_event_id_events_id_fk": { + "name": "retained_event_outputs_event_id_events_id_fk", + "tableFrom": "retained_event_outputs", + "tableTo": "events", + "columnsFrom": [ + "event_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "system_experiments": { + "name": "system_experiments", + "columns": { + "key": { + "name": "key", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "value": { + "name": "value", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "terminal_sessions": { + "name": "terminal_sessions", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "thread_id": { + "name": "thread_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "environment_id": { + "name": "environment_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "host_id": { + "name": "host_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "daemon_session_id": { + "name": "daemon_session_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "initial_cwd": { + "name": "initial_cwd", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "cols": { + "name": "cols", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "rows": { + "name": "rows", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "exit_code": { + "name": "exit_code", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "close_reason": { + "name": "close_reason", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "last_user_input_at": { + "name": "last_user_input_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "terminal_sessions_thread_status_updated_idx": { + "name": "terminal_sessions_thread_status_updated_idx", + "columns": [ + "thread_id", + "status", + "updated_at" + ], + "isUnique": false + }, + "terminal_sessions_environment_status_idx": { + "name": "terminal_sessions_environment_status_idx", + "columns": [ + "environment_id", + "status" + ], + "isUnique": false + }, + "terminal_sessions_host_status_idx": { + "name": "terminal_sessions_host_status_idx", + "columns": [ + "host_id", + "status" + ], + "isUnique": false + }, + "terminal_sessions_daemon_session_idx": { + "name": "terminal_sessions_daemon_session_idx", + "columns": [ + "daemon_session_id" + ], + "isUnique": false + } + }, + "foreignKeys": { + "terminal_sessions_thread_id_threads_id_fk": { + "name": "terminal_sessions_thread_id_threads_id_fk", + "tableFrom": "terminal_sessions", + "tableTo": "threads", + "columnsFrom": [ + "thread_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "terminal_sessions_environment_id_environments_id_fk": { + "name": "terminal_sessions_environment_id_environments_id_fk", + "tableFrom": "terminal_sessions", + "tableTo": "environments", + "columnsFrom": [ + "environment_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "terminal_sessions_host_id_hosts_id_fk": { + "name": "terminal_sessions_host_id_hosts_id_fk", + "tableFrom": "terminal_sessions", + "tableTo": "hosts", + "columnsFrom": [ + "host_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "terminal_sessions_daemon_session_id_host_daemon_sessions_id_fk": { + "name": "terminal_sessions_daemon_session_id_host_daemon_sessions_id_fk", + "tableFrom": "terminal_sessions", + "tableTo": "host_daemon_sessions", + "columnsFrom": [ + "daemon_session_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "thread_conversation_outlines": { + "name": "thread_conversation_outlines", + "columns": { + "thread_id": { + "name": "thread_id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "projection_key": { + "name": "projection_key", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "items_json": { + "name": "items_json", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": { + "thread_conversation_outlines_thread_id_threads_id_fk": { + "name": "thread_conversation_outlines_thread_id_threads_id_fk", + "tableFrom": "thread_conversation_outlines", + "tableTo": "threads", + "columnsFrom": [ + "thread_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "thread_dynamic_context_file_states": { + "name": "thread_dynamic_context_file_states", + "columns": { + "thread_id": { + "name": "thread_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "file_key": { + "name": "file_key", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "content_status": { + "name": "content_status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "content_hash": { + "name": "content_hash", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "shown_at": { + "name": "shown_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "thread_dynamic_context_file_states_thread_file_idx": { + "name": "thread_dynamic_context_file_states_thread_file_idx", + "columns": [ + "thread_id", + "file_key" + ], + "isUnique": true + } + }, + "foreignKeys": { + "thread_dynamic_context_file_states_thread_id_threads_id_fk": { + "name": "thread_dynamic_context_file_states_thread_id_threads_id_fk", + "tableFrom": "thread_dynamic_context_file_states", + "tableTo": "threads", + "columnsFrom": [ + "thread_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "thread_plugin_metadata": { + "name": "thread_plugin_metadata", + "columns": { + "thread_id": { + "name": "thread_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "plugin_id": { + "name": "plugin_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "metadata_json": { + "name": "metadata_json", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": { + "thread_plugin_metadata_thread_id_threads_id_fk": { + "name": "thread_plugin_metadata_thread_id_threads_id_fk", + "tableFrom": "thread_plugin_metadata", + "tableTo": "threads", + "columnsFrom": [ + "thread_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "thread_plugin_metadata_thread_id_plugin_id_pk": { + "columns": [ + "thread_id", + "plugin_id" + ], + "name": "thread_plugin_metadata_thread_id_plugin_id_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "thread_pruning_cursors": { + "name": "thread_pruning_cursors", + "columns": { + "policy": { + "name": "policy", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "scope": { + "name": "scope", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "''" + }, + "thread_id": { + "name": "thread_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "version": { + "name": "version", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "last_thread_id": { + "name": "last_thread_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "''" + }, + "current_thread_id": { + "name": "current_thread_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "step": { + "name": "step", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "sequence": { + "name": "sequence", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "upper_sequence": { + "name": "upper_sequence", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "cycle": { + "name": "cycle", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "latest_root_sequence": { + "name": "latest_root_sequence", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "latest_context_sequence": { + "name": "latest_context_sequence", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "probe_event_id": { + "name": "probe_event_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "probe_phase": { + "name": "probe_phase", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "probe_sequence": { + "name": "probe_sequence", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "probe_witness_id": { + "name": "probe_witness_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "thread_pruning_cursors_thread_idx": { + "name": "thread_pruning_cursors_thread_idx", + "columns": [ + "thread_id" + ], + "isUnique": false + } + }, + "foreignKeys": { + "thread_pruning_cursors_thread_id_threads_id_fk": { + "name": "thread_pruning_cursors_thread_id_threads_id_fk", + "tableFrom": "thread_pruning_cursors", + "tableTo": "threads", + "columnsFrom": [ + "thread_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "thread_pruning_cursors_policy_scope_pk": { + "columns": [ + "policy", + "scope" + ], + "name": "thread_pruning_cursors_policy_scope_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": { + "thread_pruning_cursors_scope_check": { + "name": "thread_pruning_cursors_scope_check", + "value": "\"thread_pruning_cursors\".\"scope\" = coalesce(\"thread_pruning_cursors\".\"thread_id\", '')" + } + } + }, + "thread_search_segments": { + "name": "thread_search_segments", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "thread_id": { + "name": "thread_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "source_kind": { + "name": "source_kind", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "source_key": { + "name": "source_key", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "source_seq": { + "name": "source_seq", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "text": { + "name": "text", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "thread_search_segments_source_idx": { + "name": "thread_search_segments_source_idx", + "columns": [ + "thread_id", + "source_kind", + "source_key" + ], + "isUnique": true + }, + "thread_search_segments_thread_source_seq_idx": { + "name": "thread_search_segments_thread_source_seq_idx", + "columns": [ + "thread_id", + "source_seq" + ], + "isUnique": false + } + }, + "foreignKeys": { + "thread_search_segments_thread_id_threads_id_fk": { + "name": "thread_search_segments_thread_id_threads_id_fk", + "tableFrom": "thread_search_segments", + "tableTo": "threads", + "columnsFrom": [ + "thread_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "thread_sections": { + "name": "thread_sections", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "thread_sections_name_idx": { + "name": "thread_sections_name_idx", + "columns": [ + "name" + ], + "isUnique": true + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "thread_tabs": { + "name": "thread_tabs", + "columns": { + "thread_id": { + "name": "thread_id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "tabs_json": { + "name": "tabs_json", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "revision": { + "name": "revision", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": { + "thread_tabs_thread_id_threads_id_fk": { + "name": "thread_tabs_thread_id_threads_id_fk", + "tableFrom": "thread_tabs", + "tableTo": "threads", + "columnsFrom": [ + "thread_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "threads": { + "name": "threads", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "project_id": { + "name": "project_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "environment_id": { + "name": "environment_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "provider_id": { + "name": "provider_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "model_override": { + "name": "model_override", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "reasoning_level_override": { + "name": "reasoning_level_override", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "title_fallback": { + "name": "title_fallback", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "section_id": { + "name": "section_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'starting'" + }, + "startup_context": { + "name": "startup_context", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "parent_thread_id": { + "name": "parent_thread_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "lifecycle_owner_thread_id": { + "name": "lifecycle_owner_thread_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "source_thread_id": { + "name": "source_thread_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "origin_kind": { + "name": "origin_kind", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "origin_plugin_id": { + "name": "origin_plugin_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "visibility": { + "name": "visibility", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'visible'" + }, + "archived_at": { + "name": "archived_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "pinned_at": { + "name": "pinned_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "pin_sort_key": { + "name": "pin_sort_key", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "deleted_at": { + "name": "deleted_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "storage_deleted_at": { + "name": "storage_deleted_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "last_read_at": { + "name": "last_read_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "latest_attention_at": { + "name": "latest_attention_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "threads_project_id_idx": { + "name": "threads_project_id_idx", + "columns": [ + "project_id", + "id" + ], + "isUnique": false + }, + "threads_project_updated_idx": { + "name": "threads_project_updated_idx", + "columns": [ + "project_id", + "updated_at" + ], + "isUnique": false + }, + "threads_project_archived_deleted_idx": { + "name": "threads_project_archived_deleted_idx", + "columns": [ + "project_id", + "archived_at", + "deleted_at", + "id" + ], + "isUnique": false + }, + "threads_pin_sort_idx": { + "name": "threads_pin_sort_idx", + "columns": [ + "archived_at", + "deleted_at", + "pin_sort_key", + "id" + ], + "isUnique": false, + "where": "\"threads\".\"pinned_at\" IS NOT NULL" + }, + "threads_environment_idx": { + "name": "threads_environment_idx", + "columns": [ + "environment_id" + ], + "isUnique": false + }, + "threads_lifecycle_owner_idx": { + "name": "threads_lifecycle_owner_idx", + "columns": [ + "lifecycle_owner_thread_id" + ], + "isUnique": false + }, + "threads_parent_idx": { + "name": "threads_parent_idx", + "columns": [ + "parent_thread_id" + ], + "isUnique": false + }, + "threads_source_origin_idx": { + "name": "threads_source_origin_idx", + "columns": [ + "source_thread_id", + "origin_kind" + ], + "isUnique": false + }, + "threads_origin_plugin_archived_idx": { + "name": "threads_origin_plugin_archived_idx", + "columns": [ + "origin_plugin_id", + "archived_at" + ], + "isUnique": false + }, + "threads_section_archived_deleted_idx": { + "name": "threads_section_archived_deleted_idx", + "columns": [ + "section_id", + "archived_at", + "deleted_at", + "id" + ], + "isUnique": false + }, + "threads_archived_status_idx": { + "name": "threads_archived_status_idx", + "columns": [ + "archived_at", + "status" + ], + "isUnique": false + }, + "threads_environment_archived_deleted_idx": { + "name": "threads_environment_archived_deleted_idx", + "columns": [ + "environment_id", + "archived_at", + "deleted_at" + ], + "isUnique": false + }, + "threads_active_maintenance_idx": { + "name": "threads_active_maintenance_idx", + "columns": [ + "status" + ], + "isUnique": false, + "where": "\"threads\".\"deleted_at\" IS NULL" + } + }, + "foreignKeys": { + "threads_project_id_projects_id_fk": { + "name": "threads_project_id_projects_id_fk", + "tableFrom": "threads", + "tableTo": "projects", + "columnsFrom": [ + "project_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "threads_environment_id_environments_id_fk": { + "name": "threads_environment_id_environments_id_fk", + "tableFrom": "threads", + "tableTo": "environments", + "columnsFrom": [ + "environment_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + }, + "threads_section_id_thread_sections_id_fk": { + "name": "threads_section_id_thread_sections_id_fk", + "tableFrom": "threads", + "tableTo": "thread_sections", + "columnsFrom": [ + "section_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + }, + "threads_parent_thread_id_threads_id_fk": { + "name": "threads_parent_thread_id_threads_id_fk", + "tableFrom": "threads", + "tableTo": "threads", + "columnsFrom": [ + "parent_thread_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + }, + "threads_lifecycle_owner_thread_id_threads_id_fk": { + "name": "threads_lifecycle_owner_thread_id_threads_id_fk", + "tableFrom": "threads", + "tableTo": "threads", + "columnsFrom": [ + "lifecycle_owner_thread_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "restrict", + "onUpdate": "no action" + }, + "threads_source_thread_id_threads_id_fk": { + "name": "threads_source_thread_id_threads_id_fk", + "tableFrom": "threads", + "tableTo": "threads", + "columnsFrom": [ + "source_thread_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "ui_preferences": { + "name": "ui_preferences", + "columns": { + "key": { + "name": "key", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "value_json": { + "name": "value_json", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "revision": { + "name": "revision", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + } + }, + "views": {}, + "enums": {}, + "_meta": { + "schemas": {}, + "tables": {}, + "columns": {} + }, + "internal": { + "indexes": {} + } +} \ No newline at end of file diff --git a/packages/db/drizzle/meta/_journal.json b/packages/db/drizzle/meta/_journal.json index 6a30ce673b3..c2e5d65b131 100644 --- a/packages/db/drizzle/meta/_journal.json +++ b/packages/db/drizzle/meta/_journal.json @@ -890,6 +890,13 @@ "when": 1789607725218, "tag": "0126_overconfident_vin_gonzales", "breakpoints": true + }, + { + "idx": 127, + "version": "6", + "when": 1789686861487, + "tag": "0127_plain_cammi", + "breakpoints": true } ] } \ No newline at end of file diff --git a/packages/db/src/completed-item-history.ts b/packages/db/src/completed-item-history.ts new file mode 100644 index 00000000000..aff48d4106b --- /dev/null +++ b/packages/db/src/completed-item-history.ts @@ -0,0 +1,246 @@ +import { isDeepStrictEqual } from "node:util"; +import { + threadEventTypeSchema, + type JsonObject, + type JsonValue, + type ThreadEventType, +} from "@bb/domain"; + +export const COMPACTED_ITEM_KINDS = [ + "commandExecution", + "fileChange", + "reasoning", + "agentMessage", +] as const; +export const COMPACTED_HISTORY_TYPES: readonly ThreadEventType[] = [ + "item/started", + "item/agentMessage/delta", + "item/commandExecution/outputDelta", + "item/reasoning/textDelta", + "item/reasoning/summaryTextDelta", +]; +type ItemKind = (typeof COMPACTED_ITEM_KINDS)[number]; +export function isCompactedItemKind(value: string | null): value is ItemKind { + return COMPACTED_ITEM_KINDS.some((kind) => kind === value); +} +interface HistoryRecord { + id: string; + sequence: number; + createdAt: number; + type: ThreadEventType; + itemKind: ItemKind | null; + payload: JsonObject; + sharedFields: string[]; + sharedItemFields: string[]; +} +const mutableOutputFields = new Set([ + "aggregatedOutput", + "result", + "resultText", + "truncation", +]); + +function isObject(value: JsonValue | undefined): value is JsonObject { + return value !== null && typeof value === "object" && !Array.isArray(value); +} + +function assertFiniteJsonNumbers(value: unknown): void { + if (typeof value === "number" && !Number.isFinite(value)) + throw new Error("Non-finite completed item history number"); + if (value !== null && typeof value === "object") { + for (const child of Object.values(value)) assertFiniteJsonNumbers(child); + } +} + +function parseHistoryJson(data: string): JsonValue { + const value: unknown = JSON.parse(data); + assertFiniteJsonNumbers(value); + return value as JsonValue; +} + +export function parseHistoryPayload(data: string): JsonObject { + const parsed = parseHistoryJson(data); + if (!isObject(parsed)) + throw new Error("Completed item history payload must be an object"); + return parsed; +} + +function fieldNames(value: JsonValue | undefined): string[] { + if ( + !Array.isArray(value) || + !value.every((key): key is string => typeof key === "string") + ) + throw new Error("Invalid completed item history field names"); + return value; +} + +export function decodeHistory(data: string): HistoryRecord[] { + return decodeHistoryValue(parseHistoryJson(data)); +} + +function decodeHistoryValue(value: JsonValue): HistoryRecord[] { + if ( + !Array.isArray(value) || + value.length !== 2 || + value[0] !== 1 || + !Array.isArray(value[1]) || + value[1].length < 1 || + value[1].length > 3 + ) + throw new Error("Invalid completed item history version or records"); + const records = value[1].map((record): HistoryRecord => { + if (!Array.isArray(record) || record.length !== 6) + throw new Error("Invalid completed item history record"); + const [id, sequence, createdAt, type, itemKind, encoding] = record; + if ( + typeof id !== "string" || + id.length === 0 || + typeof sequence !== "number" || + !Number.isSafeInteger(sequence) || + sequence < 1 || + typeof createdAt !== "number" || + !Number.isFinite(createdAt) + ) + throw new Error("Invalid completed item history identity"); + if ( + itemKind !== null && + itemKind !== "commandExecution" && + itemKind !== "fileChange" && + itemKind !== "reasoning" && + itemKind !== "agentMessage" + ) + throw new Error("Invalid completed item history kind"); + if ( + !Array.isArray(encoding) || + encoding.length !== 3 || + !isObject(encoding[0]) + ) + throw new Error("Invalid completed item history payload"); + const sharedFields = fieldNames(encoding[1]); + const sharedItemFields = fieldNames(encoding[2]); + if ( + sharedFields.includes("item") || + sharedItemFields.some((key) => mutableOutputFields.has(key)) + ) + throw new Error( + "Mutable completed output cannot own reconstructed history", + ); + const eventType = threadEventTypeSchema.parse(type); + if (!COMPACTED_HISTORY_TYPES.includes(eventType)) + throw new Error("Unsupported completed item history event type"); + return { + id, + sequence, + createdAt, + type: eventType, + itemKind, + payload: encoding[0], + sharedFields, + sharedItemFields, + }; + }); + if ( + new Set(records.map((record) => record.id)).size !== records.length || + new Set(records.map((record) => record.sequence)).size !== records.length + ) + throw new Error("Duplicate completed item history identity"); + return records; +} + +export function encodeHistory(records: readonly HistoryRecord[]): string { + return JSON.stringify([ + 1, + records.map((record) => [ + record.id, + record.sequence, + record.createdAt, + record.type, + record.itemKind, + [record.payload, record.sharedFields, record.sharedItemFields], + ]), + ]); +} + +export function compactHistoryPayload(data: JsonObject, owner: JsonObject) { + const payload = { ...data }; + const sharedFields: string[] = []; + const sharedItemFields: string[] = []; + for (const key of Object.keys(payload)) { + if ( + key !== "item" && + Object.hasOwn(owner, key) && + isDeepStrictEqual(payload[key], owner[key]) + ) { + sharedFields.push(key); + delete payload[key]; + } + } + if (isObject(payload.item) && isObject(owner.item)) { + const item = { ...payload.item }; + for (const key of Object.keys(item)) { + if ( + !mutableOutputFields.has(key) && + Object.hasOwn(owner.item, key) && + isDeepStrictEqual(item[key], owner.item[key]) + ) { + sharedItemFields.push(key); + delete item[key]; + } + } + payload.item = item; + } + return { payload, sharedFields, sharedItemFields }; +} + +export function restoreHistoryPayloadObject( + record: HistoryRecord, + owner: JsonObject, +): JsonObject { + const payload = { ...record.payload }; + for (const key of record.sharedFields) { + const value = owner[key]; + if (!Object.hasOwn(owner, key) || value === undefined) + throw new Error("Missing completed item history field"); + Object.defineProperty(payload, key, { value, enumerable: true }); + } + if (record.sharedItemFields.length > 0) { + if (!isObject(payload.item) || !isObject(owner.item)) + throw new Error("Missing completed item history item"); + const item = { ...payload.item }; + for (const key of record.sharedItemFields) { + const value = owner.item[key]; + if (!Object.hasOwn(owner.item, key) || value === undefined) + throw new Error("Missing completed item history item field"); + Object.defineProperty(item, key, { value, enumerable: true }); + } + payload.item = item; + } + return payload; +} + +export function restoreHistoryPayload( + record: HistoryRecord, + owner: JsonObject, +): string { + return JSON.stringify(restoreHistoryPayloadObject(record, owner)); +} + +export function decodeCompletedItemHistory(data: string) { + const value = parseHistoryJson(data); + if ( + !Array.isArray(value) || + value.length !== 4 || + value[0] !== 1 || + typeof value[1] !== "number" || + !Number.isSafeInteger(value[1]) || + value[1] < 1 || + typeof value[2] !== "number" || + !Number.isFinite(value[2]) + ) + throw new Error("Invalid combined completed item history"); + const records = decodeHistoryValue(value[3]); + const sequence = value[1]; + if (records.some((row) => row.sequence >= sequence)) + throw new Error("Completed item history exceeds completion sequence"); + return { sequence: value[1], createdAt: value[2], records }; +} diff --git a/packages/db/src/data/completed-item-compaction.ts b/packages/db/src/data/completed-item-compaction.ts new file mode 100644 index 00000000000..1a0fa42743a --- /dev/null +++ b/packages/db/src/data/completed-item-compaction.ts @@ -0,0 +1,415 @@ +import { eq, inArray, sql } from "drizzle-orm"; +import { parseStoredThreadEvent, type ThreadEventType } from "@bb/domain"; +import type { DbQueryConnection } from "../connection.js"; +import { events } from "../schema.js"; +import { + compactHistoryPayload, + encodeHistory, + isCompactedItemKind, + parseHistoryPayload, +} from "../completed-item-history.js"; + +const INPUT_BYTE_BUDGET = 1024 * 1024; +const supportTypesByKind = { + commandExecution: [ + "item/started", + "item/completed", + "item/commandExecution/outputDelta", + ], + fileChange: ["item/started", "item/completed", "item/fileChange/outputDelta"], + agentMessage: ["item/started", "item/completed", "item/agentMessage/delta"], + reasoning: [ + "item/started", + "item/completed", + "item/reasoning/textDelta", + "item/reasoning/summaryTextDelta", + ], +} satisfies Record; +interface Candidate { + id: string; + sequence: number; + type: ThreadEventType; + itemId: string | null; + itemKind: typeof events.$inferSelect.itemKind; + parentToolCallId: string | null; + turnId: string | null; + providerThreadId: string | null; + environmentId: string | null; + dataBytes: number; + hasHistory: number; +} +const candidateFields = sql`id, sequence, type, item_id AS itemId, item_kind AS itemKind, + parent_tool_call_id AS parentToolCallId, turn_id AS turnId, provider_thread_id AS providerThreadId, + environment_id AS environmentId, octet_length(data) AS dataBytes, completed_item_history IS NOT NULL AS hasHistory`; + +export function advanceCompletedItemCompaction( + db: DbQueryConnection, + args: { + threadId: string; + afterSequence: number; + throughSequence: number; + limit: number; + }, +) { + const pageSize = Math.min(8, Math.max(1, Math.floor(args.limit / 64))); + const candidates = db.all(sql`SELECT ${candidateFields} + FROM events INDEXED BY events_thread_type_sequence_idx + WHERE thread_id = ${args.threadId} AND type = 'item/completed' + AND sequence > ${args.afterSequence} AND sequence <= ${args.throughSequence} + ORDER BY sequence LIMIT ${pageSize}`); + let scanned = candidates.length; + let removed = 0; + let removedBytes = 0; + let processedBytes = 0; + let nextSequence = args.afterSequence; + const skipped: Record = {}; + const skip = (kind: string, reason: string) => { + const key = `${kind}:${reason}`; + skipped[key] = (skipped[key] ?? 0) + 1; + }; + for (const candidate of candidates) { + if (processedBytes >= INPUT_BYTE_BUDGET) break; + nextSequence = candidate.sequence; + const kind = candidate.itemKind; + if (!isCompactedItemKind(kind)) { + skip(kind ?? "unknown", "unsupported-kind"); + continue; + } + if (candidate.turnId === null || candidate.itemId === null) { + skip(kind, "missing-scope"); + continue; + } + if (candidate.hasHistory) continue; + const support: Candidate[] = []; + let exhausted = true; + for (const type of [ + ...supportTypesByKind[kind], + "turn/started", + "turn/completed", + ] as const) { + const limit = Math.min(8, args.limit - scanned - 2); + if (limit <= 0) { + exhausted = false; + break; + } + const rows = db.all(sql`SELECT ${candidateFields} + FROM events INDEXED BY events_thread_turn_type_item_sequence_idx + WHERE thread_id = ${args.threadId} AND turn_id = ${candidate.turnId} + AND type = ${type} AND item_id IS ${type.startsWith("turn/") ? null : candidate.itemId} + ORDER BY sequence LIMIT ${limit}`); + scanned += Math.max(1, rows.length); + support.push(...rows); + if (rows.length === limit) { + exhausted = false; + break; + } + } + if (!exhausted) { + skip(kind, "support-budget"); + continue; + } + if ( + support.some( + (row) => + !row.type.startsWith("turn/") && + (row.parentToolCallId !== candidate.parentToolCallId || + ((row.type === "item/started" || row.type === "item/completed") && + row.itemKind !== kind)), + ) + ) { + skip(kind, "incompatible-lifecycle"); + continue; + } + const peers = support.filter( + (row) => + !row.type.startsWith("turn/") && + row.parentToolCallId === candidate.parentToolCallId && + (row.itemKind === kind || + (row.type !== "item/started" && row.type !== "item/completed")), + ); + const starts = peers.filter((row) => row.type === "item/started"); + const completions = peers.filter((row) => row.type === "item/completed"); + const deltas = peers.filter( + (row) => row.type !== "item/started" && row.type !== "item/completed", + ); + if (completions.length !== 1 || completions[0]?.id !== candidate.id) { + skip(kind, "ambiguous-completion"); + continue; + } + if (starts.length > 1) { + skip(kind, "ambiguous-start"); + continue; + } + if (peers.some((row) => row.sequence > candidate.sequence)) { + skip(kind, "late-item-event"); + continue; + } + if ( + peers.some( + (row) => + row.environmentId !== candidate.environmentId || + row.providerThreadId !== candidate.providerThreadId, + ) + ) { + skip(kind, "incompatible-envelope"); + continue; + } + const turnRows = support.filter( + (row) => + row.type.startsWith("turn/") && + (row.parentToolCallId === null || + row.parentToolCallId === candidate.parentToolCallId) && + row.providerThreadId === candidate.providerThreadId, + ); + const settled = turnRows.find( + (row) => + row.type === "turn/completed" && row.sequence > candidate.sequence, + ); + if ( + settled === undefined || + turnRows.some( + (row) => + row.type === "turn/started" && row.sequence > candidate.sequence, + ) + ) { + skip(kind, "unsettled"); + continue; + } + if ( + deltas.some((row) => row.type === "item/fileChange/outputDelta") || + new Set(deltas.map((row) => row.type)).size !== deltas.length || + deltas.length > 2 + ) { + skip(kind, "unpruned-or-unsupported-deltas"); + continue; + } + const source = [...starts, ...deltas].sort( + (a, b) => a.sequence - b.sequence, + ); + if (source.length === 0) continue; + const firstSequence = source[0]!.sequence; + const boundaryRows: { id: string; dataBytes: number }[] = []; + let boundaryBudgetExhausted = false; + let crossesOutputBoundary = false; + const boundaryTypes = ["client/turn/requested", "system/operation"]; + if (kind === "agentMessage") + boundaryTypes.push("item/completed", "system/manager/user_message"); + for (const type of boundaryTypes) { + const limit = Math.min(8, args.limit - scanned); + if (limit <= 0) { + boundaryBudgetExhausted = true; + break; + } + const found = db.all<{ + id: string; + dataBytes: number; + itemKind: string | null; + }>(sql` + SELECT id, octet_length(data) AS dataBytes, item_kind AS itemKind + FROM events INDEXED BY events_thread_type_sequence_idx + WHERE thread_id = ${args.threadId} AND type = ${type} + AND sequence > ${firstSequence} AND sequence < ${candidate.sequence} + ORDER BY sequence LIMIT ${limit} + `); + scanned += Math.max(1, found.length); + if (type === "item/completed") { + crossesOutputBoundary ||= found.some( + (row) => row.itemKind === "agentMessage", + ); + } else if (type === "system/manager/user_message") { + crossesOutputBoundary ||= found.length > 0; + } else { + boundaryRows.push(...found); + } + if (found.length === limit) { + boundaryBudgetExhausted = true; + break; + } + } + if (boundaryBudgetExhausted) { + skip(kind, "boundary-budget"); + continue; + } + if (crossesOutputBoundary) { + skip(kind, "output-order-boundary"); + continue; + } + const boundaryBytes = boundaryRows.reduce( + (total, row) => total + row.dataBytes, + 0, + ); + if (boundaryBytes > INPUT_BYTE_BUDGET - processedBytes) { + skip(kind, "boundary-payload-budget"); + continue; + } + processedBytes += boundaryBytes; + let crossesBoundary = false; + for (const boundary of boundaryRows) { + const row = db + .select({ data: events.data }) + .from(events) + .where(eq(events.id, boundary.id)) + .get(); + try { + if (!row) throw new Error("Missing boundary"); + const payload = parseHistoryPayload(row.data); + if ( + payload.initiator === "user" || + (payload.operation === "context_clear" && + payload.status === "completed") + ) + crossesBoundary = true; + } catch { + crossesBoundary = true; + } + } + if (crossesBoundary) { + skip(kind, "edit-or-context-boundary"); + continue; + } + const bytes = peers.reduce( + (total, row) => total + row.dataBytes, + settled.dataBytes, + ); + if (bytes > INPUT_BYTE_BUDGET - processedBytes) { + skip(kind, "payload-budget"); + continue; + } + const rows = db + .select() + .from(events) + .where(inArray(events.id, [...peers.map((row) => row.id), settled.id])) + .all(); + processedBytes += bytes; + const owner = rows.find((row) => row.id === candidate.id); + if (owner === undefined) + throw new Error("Missing completed item compaction owner"); + let payloads: Map>; + try { + payloads = new Map( + rows.map((row) => { + const payload = parseHistoryPayload(row.data); + const event = parseStoredThreadEvent({ + threadId: args.threadId, + providerThreadId: row.providerThreadId, + type: row.type, + scope: { kind: "turn", turnId: candidate.turnId! }, + data: payload, + }); + if ( + (event.type === "item/started" || + event.type === "item/completed") && + (event.item.id !== candidate.itemId || event.item.type !== kind) + ) + throw new Error("Mismatched item identity"); + return [row.id, payload]; + }), + ); + } catch { + skip(kind, "malformed-payload"); + continue; + } + const ownerPayload = payloads.get(candidate.id); + if (ownerPayload === undefined) + throw new Error("Missing completed item payload"); + const item = ownerPayload.item; + if (item === null || typeof item !== "object" || Array.isArray(item)) { + skip(kind, "malformed-completion"); + continue; + } + if ( + (kind === "commandExecution" || kind === "fileChange") && + item.status !== "completed" && + item.status !== "failed" && + item.status !== "interrupted" + ) { + skip(kind, "completion-status"); + continue; + } + const compacted = source.filter((row) => { + if (row.type !== "item/commandExecution/outputDelta") return true; + const payload = payloads.get(row.id); + if (payload === undefined || typeof payload.delta !== "string") { + skip(kind, "malformed-delta"); + return false; + } + if ( + item.status !== "completed" && + item.status !== "failed" && + item.status !== "interrupted" + ) { + skip(kind, "command-status"); + return false; + } + if ( + typeof item.aggregatedOutput !== "string" || + item.aggregatedOutput.length === 0 + ) { + skip(kind, "empty-output"); + return false; + } + if ( + starts.length === 0 && + !item.aggregatedOutput.includes(payload.delta) + ) { + skip(kind, "missing-start-uncontained-text"); + return false; + } + payload.delta = ""; + return true; + }); + if (compacted.length !== source.length) continue; + const records = compacted.map((row) => { + const payload = payloads.get(row.id); + const stored = rows.find((entry) => entry.id === row.id); + if (payload === undefined || stored === undefined) + throw new Error("Missing completed item source"); + if (stored.itemKind !== null && !isCompactedItemKind(stored.itemKind)) + throw new Error("Unsupported compacted item kind"); + return { + id: row.id, + sequence: row.sequence, + createdAt: stored.createdAt, + type: row.type, + itemKind: stored.itemKind, + ...compactHistoryPayload(payload, ownerPayload), + }; + }); + const data = JSON.stringify([ + 1, + candidate.sequence, + owner.createdAt, + JSON.parse(encodeHistory(records)), + ]); + const result = db + .delete(events) + .where( + inArray( + events.id, + records.map((row) => row.id), + ), + ) + .run(); + if (result.changes !== records.length) + throw new Error("Completed item compaction lost a source row"); + db.update(events) + .set({ sequence: firstSequence, completedItemHistory: data }) + .where(eq(events.id, candidate.id)) + .run(); + removed += result.changes; + removedBytes += + compacted.reduce((total, row) => total + row.dataBytes, 0) - + Buffer.byteLength(data); + } + return { + scanned, + removed, + removedBytes, + processedBytes, + skipped, + nextSequence, + complete: + candidates.length < pageSize && + nextSequence === (candidates.at(-1)?.sequence ?? args.afterSequence), + }; +} diff --git a/packages/db/src/data/completed-item-history.ts b/packages/db/src/data/completed-item-history.ts new file mode 100644 index 00000000000..16014e8d347 --- /dev/null +++ b/packages/db/src/data/completed-item-history.ts @@ -0,0 +1,210 @@ +import type { JsonObject } from "@bb/domain"; +import { and, inArray, isNotNull, sql } from "drizzle-orm"; +import type { DbQueryConnection } from "../connection.js"; +import { events } from "../schema.js"; +import { + decodeCompletedItemHistory, + parseHistoryPayload, + restoreHistoryPayloadObject, +} from "../completed-item-history.js"; +import type { StoredEventRow } from "./events.js"; + +export type ProjectionStoredEventRow = StoredEventRow & { + parsedData?: JsonObject; +}; + +function reconstructCompletedItemHistory(metadata: string, ownerData: string) { + const history = decodeCompletedItemHistory(metadata); + const payload = parseHistoryPayload(ownerData); + return { + sequence: history.sequence, + createdAt: history.createdAt, + payload, + records: history.records.map((record) => { + const data = restoreHistoryPayloadObject(record, payload); + return { + id: record.id, + sequence: record.sequence, + createdAt: record.createdAt, + type: record.type, + itemKind: record.itemKind, + data: JSON.stringify(data), + payload: data, + }; + }), + }; +} + +interface ReconstructionEntry { + metadata: string; + ownerData: string; + history: ReturnType; + chars: number; + records: number; +} + +interface ReconstructionCache { + entries: Map; + chars: number; + records: number; +} + +const reconstructionCaches = new WeakMap< + DbQueryConnection, + ReconstructionCache +>(); +const RECONSTRUCTION_CACHE_MAX_RECORDS = 10_000; +const RECONSTRUCTION_CACHE_MAX_CHARS = 8_000_000; + +function reconstructCached( + db: DbQueryConnection, + id: string, + metadata: string, + ownerData: string, +) { + let cache = reconstructionCaches.get(db); + if (cache === undefined) { + cache = { entries: new Map(), chars: 0, records: 0 }; + reconstructionCaches.set(db, cache); + } + const previous = cache.entries.get(id); + if (previous !== undefined) { + cache.entries.delete(id); + cache.chars -= previous.chars; + cache.records -= previous.records; + if (previous.metadata === metadata && previous.ownerData === ownerData) { + cache.entries.set(id, previous); + cache.chars += previous.chars; + cache.records += previous.records; + return previous.history; + } + } + const history = reconstructCompletedItemHistory(metadata, ownerData); + const chars = + metadata.length + + ownerData.length + + history.records.reduce((sum, record) => sum + record.data.length, 0); + const records = history.records.length + 1; + if ( + chars > RECONSTRUCTION_CACHE_MAX_CHARS || + records > RECONSTRUCTION_CACHE_MAX_RECORDS + ) + return history; + while ( + cache.records + records > RECONSTRUCTION_CACHE_MAX_RECORDS || + cache.chars + chars > RECONSTRUCTION_CACHE_MAX_CHARS + ) { + const oldest = cache.entries.entries().next().value; + if (oldest === undefined) break; + cache.entries.delete(oldest[0]); + cache.chars -= oldest[1].chars; + cache.records -= oldest[1].records; + } + cache.entries.set(id, { metadata, ownerData, history, chars, records }); + cache.chars += chars; + cache.records += records; + return history; +} + +function expandSelectedCompletedItemRowsInternal( + db: DbQueryConnection, + rows: readonly StoredEventRow[], + throughSequence: number, + includeParsedData: boolean, +): ProjectionStoredEventRow[] { + const ids = [ + ...new Set( + rows + .filter( + (row) => + row.type === "item/completed" && + row.completedItemHistory === undefined, + ) + .map((row) => row.id), + ), + ]; + const histories = new Map(); + for (const row of rows) { + if (row.type === "item/completed" && row.completedItemHistory != null) + histories.set(row.id, row.completedItemHistory); + } + if (ids.length > 0) { + const selected = db + .select({ id: events.id, history: events.completedItemHistory }) + .from(events) + .where( + and( + inArray( + events.id, + sql`(select value from json_each(${JSON.stringify(ids)}))`, + ), + isNotNull(events.completedItemHistory), + ), + ) + .all(); + for (const row of selected) { + if (row.history !== null) histories.set(row.id, row.history); + } + } + const expanded: ProjectionStoredEventRow[] = []; + for (const selectedRow of rows) { + const metadata = histories.get(selectedRow.id); + if (metadata === undefined && includeParsedData) { + if (selectedRow.sequence <= throughSequence) expanded.push(selectedRow); + continue; + } + const { completedItemHistory: _history, ...row } = selectedRow; + if (metadata === undefined) { + if (row.sequence <= throughSequence) expanded.push(row); + continue; + } + const history = reconstructCached(db, row.id, metadata, row.data); + if (history.sequence <= throughSequence) + expanded.push({ + ...row, + sequence: history.sequence, + createdAt: history.createdAt, + ...(includeParsedData ? { parsedData: history.payload } : {}), + }); + for (const record of history.records) { + if (record.sequence <= throughSequence) + expanded.push({ + ...row, + id: record.id, + sequence: record.sequence, + createdAt: record.createdAt, + type: record.type, + itemKind: record.itemKind, + data: record.data, + ...(includeParsedData ? { parsedData: record.payload } : {}), + }); + } + } + return expanded.sort((a, b) => a.sequence - b.sequence); +} + +export function expandSelectedCompletedItemRows( + db: DbQueryConnection, + rows: readonly StoredEventRow[], + throughSequence = Number.MAX_SAFE_INTEGER, +): StoredEventRow[] { + return expandSelectedCompletedItemRowsInternal( + db, + rows, + throughSequence, + false, + ); +} + +export function expandSelectedCompletedItemRowsForProjection( + db: DbQueryConnection, + rows: readonly StoredEventRow[], + throughSequence = Number.MAX_SAFE_INTEGER, +): ProjectionStoredEventRow[] { + return expandSelectedCompletedItemRowsInternal( + db, + rows, + throughSequence, + true, + ); +} diff --git a/packages/db/src/data/events.ts b/packages/db/src/data/events.ts index c53cbed1c51..1e4d1de6825 100644 --- a/packages/db/src/data/events.ts +++ b/packages/db/src/data/events.ts @@ -1145,7 +1145,7 @@ const storedEventRowFields = { export type StoredEventRow = Pick< typeof events.$inferSelect, keyof typeof storedEventRowFields ->; +> & { completedItemHistory?: string | null }; export type InlineOutputCharLimit = number | null; @@ -2876,15 +2876,29 @@ export function findTimelineWindowBudgetFloorSequence( conditions.push(lt(events.sequence, args.beforeSequence)); } - const row = db - .select({ sequence: events.sequence }) + const query = db + .select({ + sequence: events.sequence, + count: + sql`1 + coalesce(json_array_length(${events.completedItemHistory}, '$[3][1]'), 0)`.as( + "count", + ), + }) .from(events) .where(and(...conditions)) .orderBy(desc(events.sequence)) - .limit(1) - .offset(args.eventBudget) - .get(); - return row?.sequence; + .limit(args.eventBudget + 1) + .toSQL(); + const statement = db.$client.prepare< + unknown[], + { sequence: number; count: number } + >(query.sql); + let count = 0; + for (const row of statement.iterate(...query.params)) { + count += row.count; + if (count > args.eventBudget) return row.sequence; + } + return undefined; } export function listTimelineInterruptionRows( @@ -3103,7 +3117,10 @@ export function findStoredTimelineWindowByteBudgetFloor( const query = db .select({ createdAt: events.createdAt, - dataBytes: sql`length(CAST(${data} AS BLOB))`.as("data_bytes"), + dataBytes: + sql`length(CAST(${data} AS BLOB)) + COALESCE(length(CAST(${events.completedItemHistory} AS BLOB)), 0)`.as( + "data_bytes", + ), sequence: events.sequence, turnId: events.turnId, }) @@ -3178,7 +3195,12 @@ export function listStoredTimelineTurnEventRows( fixedVariableCount: 32, queryBatch: (turnIds) => db - .select(storedEventRowSqlFields(args.maxInlineOutputChars)) + .select({ + ...storedEventRowSqlFields(args.maxInlineOutputChars), + completedItemHistory: sql< + string | null + >`${events.completedItemHistory}`, + }) .from( sql`${events} INDEXED BY events_thread_turn_type_item_sequence_idx`, ) @@ -3220,9 +3242,10 @@ export function listStoredTimelineThreadWindowEventRows( args: ListStoredTimelineWindowEventRowsArgs, ): StoredEventRow[] { return db - .select( - storedEventRowFieldsWithInlineOutputLimit(args.maxInlineOutputChars), - ) + .select({ + ...storedEventRowFieldsWithInlineOutputLimit(args.maxInlineOutputChars), + completedItemHistory: events.completedItemHistory, + }) .from(events) .where(and(...storedTimelineWindowConditions(args), isNull(events.turnId))) .orderBy(events.sequence) @@ -3234,9 +3257,10 @@ export function listStoredTimelineWindowEventRows( args: ListStoredTimelineWindowEventRowsArgs, ): StoredEventRow[] { return db - .select( - storedEventRowFieldsWithInlineOutputLimit(args.maxInlineOutputChars), - ) + .select({ + ...storedEventRowFieldsWithInlineOutputLimit(args.maxInlineOutputChars), + completedItemHistory: events.completedItemHistory, + }) .from(events) .where(and(...storedTimelineWindowConditions(args))) .orderBy(events.sequence) diff --git a/packages/db/src/data/thread-pruning.ts b/packages/db/src/data/thread-pruning.ts index bf5d21884eb..42a8ade5b1a 100644 --- a/packages/db/src/data/thread-pruning.ts +++ b/packages/db/src/data/thread-pruning.ts @@ -1,3 +1,4 @@ +import { advanceCompletedItemCompaction } from "./completed-item-compaction.js"; import { isBeforeLatestThreadEvent } from "./event-pruning-guards.js"; import { advanceLiveEventPruning, @@ -19,6 +20,7 @@ export const THREAD_PRUNING_POLICIES = [ "usage", "turn-diffs", "resolved-items", + "completed-items", ] as const; export type ThreadPruningPolicy = (typeof THREAD_PRUNING_POLICIES)[number]; const VERSION_BY_POLICY: Record = { @@ -26,6 +28,7 @@ const VERSION_BY_POLICY: Record = { usage: 1, "turn-diffs": 1, "resolved-items": 1, + "completed-items": 1, }; const BATCH_SIZE = 500; const LIVE_BATCH_SIZE = 32; @@ -50,9 +53,10 @@ export function getNextThreadPruningPolicy( .all(); const updated = new Map(rows.map((row) => [row.policy, row.updatedAt])); return ( - THREAD_PRUNING_POLICIES.filter((policy) => !excluded.has(policy)).sort( - (a, b) => (updated.get(a) ?? 0) - (updated.get(b) ?? 0), - )[0] ?? null + THREAD_PRUNING_POLICIES.filter( + (policy) => + !excluded.has(policy) && (scope === "" || policy !== "completed-items"), + ).sort((a, b) => (updated.get(a) ?? 0) - (updated.get(b) ?? 0))[0] ?? null ); } @@ -173,6 +177,19 @@ function advanceThreadPruningTransaction( cursor.sequence = batch.nextSequence; if (batch.complete || cursor.sequence >= cursor.upperSequence) action = "thread-complete"; + } else if (policy === "completed-items") { + const batch = advanceCompletedItemCompaction(tx, { + threadId, + afterSequence: cursor.sequence, + throughSequence: cursor.upperSequence, + limit: batchSize, + }); + scanned = batch.scanned; + removed = batch.removed; + removedBytes = batch.removedBytes; + cursor.sequence = batch.nextSequence; + if (batch.complete || cursor.sequence >= cursor.upperSequence) + action = "thread-complete"; } else if (policy === "resolved-items") { const batch = advanceLiveEventPruning(tx, { threadId, diff --git a/packages/db/src/index.ts b/packages/db/src/index.ts index 84f1f95ee68..b9d7dfb445f 100644 --- a/packages/db/src/index.ts +++ b/packages/db/src/index.ts @@ -26,3 +26,9 @@ export { noopNotifier } from "./notifier.js"; export type { DbNotifier } from "./notifier.js"; export * from "./data/index.js"; + +export { + expandSelectedCompletedItemRows, + expandSelectedCompletedItemRowsForProjection, + type ProjectionStoredEventRow, +} from "./data/completed-item-history.js"; diff --git a/packages/db/src/schema.ts b/packages/db/src/schema.ts index 68c9b152330..a98a4adf88b 100644 --- a/packages/db/src/schema.ts +++ b/packages/db/src/schema.ts @@ -775,6 +775,7 @@ export const events = sqliteTable( itemKind: text("item_kind").$type(), parentToolCallId: text("parent_tool_call_id"), data: text("data").notNull().default("{}"), + completedItemHistory: text("completed_item_history"), createdAt: integer("created_at").notNull(), }, (table) => [ diff --git a/packages/db/test/completed-item-history.test.ts b/packages/db/test/completed-item-history.test.ts new file mode 100644 index 00000000000..24a1a145f0a --- /dev/null +++ b/packages/db/test/completed-item-history.test.ts @@ -0,0 +1,82 @@ +import { describe, expect, it } from "vitest"; +import { + decodeCompletedItemHistory, + decodeHistory, + parseHistoryPayload, +} from "../src/completed-item-history.js"; + +const record = [ + "start", + 1, + 100, + "item/started", + "agentMessage", + [{ item: { text: "" } }, [], ["id", "type"]], +]; + +describe("completed item history decoding", () => { + it("decodes nested records with the standalone history contract", () => { + const history = [1, [record]]; + expect( + decodeCompletedItemHistory(JSON.stringify([1, 3, 300, history])), + ).toEqual({ + sequence: 3, + createdAt: 300, + records: decodeHistory(JSON.stringify(history)), + }); + }); + + it("rejects non-finite numbers nested inside parsed JSON", () => { + expect(() => parseHistoryPayload('{"nested":[{"value":1e400}]}')).toThrow( + "Non-finite", + ); + const metadata = JSON.stringify([ + 1, + 3, + 300, + [ + 1, + [ + [ + "start", + 1, + 100, + "item/started", + "agentMessage", + [{ nested: [0] }, [], []], + ], + ], + ], + ]); + expect(() => + decodeCompletedItemHistory( + metadata.replace('"nested":[0]', '"nested":[1e400]'), + ), + ).toThrow("Non-finite"); + }); + + it.each([ + [2, [record]], + [1, []], + [1, [record, record]], + [1, [["start", 1, 100, "turn/completed", "agentMessage", [{}, [], []]]]], + [ + 1, + [ + [ + "start", + 1, + 100, + "item/started", + "agentMessage", + [{}, [], ["resultText"]], + ], + ], + ], + [1, [["start", 3, 100, "item/started", "agentMessage", [{}, [], []]]]], + ])("rejects invalid nested history %j", (...history) => { + expect(() => + decodeCompletedItemHistory(JSON.stringify([1, 3, 300, history])), + ).toThrow(); + }); +}); diff --git a/packages/db/test/data/completed-item-compaction.test.ts b/packages/db/test/data/completed-item-compaction.test.ts new file mode 100644 index 00000000000..6b5ff1c848a --- /dev/null +++ b/packages/db/test/data/completed-item-compaction.test.ts @@ -0,0 +1,676 @@ +import { describe, expect, it, vi } from "vitest"; +import { eq, sql } from "drizzle-orm"; +import { + events, + retainedEventOutputs, + threadPruningCursors, +} from "../../src/schema.js"; +import { noopNotifier } from "../../src/notifier.js"; +import { upsertHost } from "../../src/data/hosts.js"; +import { createProject } from "../../src/data/projects.js"; +import { createThread } from "../../src/data/threads.js"; +import { advanceThreadPruning } from "../../src/data/thread-pruning.js"; +import { advanceCompletedItemCompaction } from "../../src/data/completed-item-compaction.js"; +import { + expandSelectedCompletedItemRows, + expandSelectedCompletedItemRowsForProjection, +} from "../../src/data/completed-item-history.js"; +import { + listStoredEventRows, + listStoredTimelineTurnEventRows, + findTimelineWindowBudgetFloorSequence, + getHighWaterMarks, + getLatestThreadOutputEventRow, + deleteThreadEventSuffixInTransaction, +} from "../../src/data/events.js"; +import { getThreadEventRewriteGeneration } from "../../src/data/event-rewrite-generation.js"; +import { createMigratedConnection } from "../helpers/migrated-connection.js"; + +function setup() { + const db = createMigratedConnection(); + const host = upsertHost(db, noopNotifier, { name: "compaction" }); + const { project } = createProject(db, noopNotifier, { + name: "compaction", + source: { type: "local_path", hostId: host.id, path: "/tmp/compaction" }, + }); + const thread = createThread(db, noopNotifier, { + projectId: project.id, + providerId: "codex", + }); + const seed = ( + sequence: number, + values: Partial, + ) => + db + .insert(events) + .values({ + id: `event-${sequence}`, + threadId: thread.id, + sequence, + scopeKind: "turn", + turnId: "turn", + providerThreadId: "provider", + type: "item/started", + data: "{}", + createdAt: sequence * 100, + ...values, + }) + .run(); + const item = { + id: "message", + type: "agentMessage", + text: "lossless assistant text", + }; + seed(1, { type: "turn/started", data: "{}" }); + seed(2, { + itemId: item.id, + itemKind: "agentMessage", + data: JSON.stringify({ item: { ...item, text: "" } }), + }); + seed(3, { + type: "item/agentMessage/delta", + itemId: item.id, + data: JSON.stringify({ itemId: item.id, delta: "lossless assistant text" }), + }); + seed(5, { + type: "item/completed", + itemId: item.id, + itemKind: "agentMessage", + data: JSON.stringify({ item }), + }); + seed(6, { + type: "turn/completed", + data: JSON.stringify({ status: "completed" }), + }); + const rows = () => listStoredEventRows(db, { threadId: thread.id }); + const advance = () => advanceThreadPruning(db, "completed-items"); + return { db, thread, seed, rows, advance }; +} + +function normalized(rows: ReturnType["rows"]>) { + return rows.map((row) => ({ ...row, data: JSON.parse(row.data) })); +} + +describe("completed items at first lifecycle position", () => { + it("expands selected history alongside more than a thousand ordinary completions", () => { + const f = setup(); + try { + for (let sequence = 10; sequence < 1011; sequence++) { + f.seed(sequence, { type: "item/completed" }); + } + const before = f.rows(); + for (let index = 0; index < 20; index++) f.advance(); + expect(f.rows().length).toBe(before.length - 2); + expect( + normalized(expandSelectedCompletedItemRows(f.db, f.rows())), + ).toEqual(normalized(before)); + } finally { + f.db.$client.close(); + } + }); + + it("loads history with the selected timeline rows and keeps raw rows private", () => { + const f = setup(); + try { + const before = f.rows(); + f.advance(); + const selected = listStoredTimelineTurnEventRows(f.db, { + threadId: f.thread.id, + turnIds: ["turn"], + sequenceStart: 0, + maxInlineOutputChars: 8000, + }); + const prepare = vi.spyOn(f.db.$client, "prepare"); + try { + const projected = expandSelectedCompletedItemRowsForProjection( + f.db, + selected, + ); + const raw = expandSelectedCompletedItemRows(f.db, selected); + expect(prepare).not.toHaveBeenCalled(); + expect(normalized(raw)).toEqual(normalized(before)); + for (const row of projected) { + if (row.parsedData !== undefined) + expect(row.parsedData).toEqual(JSON.parse(row.data)); + } + expect(projected.some((row) => row.parsedData !== undefined)).toBe( + true, + ); + } finally { + prepare.mockRestore(); + } + } finally { + f.db.$client.close(); + } + }); + + it("reuses identical history while applying each snapshot and row scope", () => { + const f = setup(); + try { + f.advance(); + const owner = listStoredTimelineTurnEventRows(f.db, { + threadId: f.thread.id, + turnIds: ["turn"], + sequenceStart: 0, + maxInlineOutputChars: 8000, + }).find((row) => row.completedItemHistory != null)!; + const full = expandSelectedCompletedItemRowsForProjection(f.db, [owner]); + const again = expandSelectedCompletedItemRowsForProjection(f.db, [ + { ...owner }, + ]); + expect(again).toEqual(full); + expect(again[0]!.parsedData).toBe(full[0]!.parsedData); + const prefix = expandSelectedCompletedItemRowsForProjection( + f.db, + [owner], + 3, + ); + expect(prefix).toEqual(full.filter((row) => row.sequence <= 3)); + const scoped = expandSelectedCompletedItemRowsForProjection(f.db, [ + { + ...owner, + providerThreadId: "changed-provider", + parentToolCallId: "changed-parent", + }, + ]); + expect( + scoped.every( + (row) => + row.providerThreadId === "changed-provider" && + row.parentToolCallId === "changed-parent", + ), + ).toBe(true); + expect(scoped.map((row) => row.data)).toEqual( + full.map((row) => row.data), + ); + } finally { + f.db.$client.close(); + } + }); + + it.each(["characters", "records"] as const)( + "evicts reconstructed history at its %s bound", + (bound) => { + const f = setup(); + try { + f.advance(); + const owner = listStoredTimelineTurnEventRows(f.db, { + threadId: f.thread.id, + turnIds: ["turn"], + sequenceStart: 0, + maxInlineOutputChars: 8000, + }).find((row) => row.completedItemHistory != null)!; + const first = expandSelectedCompletedItemRowsForProjection(f.db, [ + owner, + ])[0]!.parsedData; + const copies = bound === "characters" ? 500 : 5000; + const data = + bound === "characters" + ? JSON.stringify({ + item: { + id: "message", + type: "agentMessage", + text: "x".repeat(20_000), + }, + }) + : owner.data; + let last = owner; + let lastPayload = first; + for (let index = 0; index < copies; index++) { + last = { ...owner, id: `cache-owner-${index}`, data }; + lastPayload = expandSelectedCompletedItemRowsForProjection(f.db, [ + last, + ])[0]!.parsedData; + } + expect( + expandSelectedCompletedItemRowsForProjection(f.db, [last])[0]! + .parsedData, + ).toBe(lastPayload); + expect( + expandSelectedCompletedItemRowsForProjection(f.db, [owner])[0]! + .parsedData, + ).not.toBe(first); + } finally { + f.db.$client.close(); + } + }, + ); + + it("does not retain an oversized reconstruction or evict a smaller one for it", () => { + const f = setup(); + try { + f.advance(); + const owner = listStoredTimelineTurnEventRows(f.db, { + threadId: f.thread.id, + turnIds: ["turn"], + sequenceStart: 0, + maxInlineOutputChars: 8000, + }).find((row) => row.completedItemHistory != null)!; + const first = expandSelectedCompletedItemRowsForProjection(f.db, [ + owner, + ])[0]!.parsedData; + const oversized = { + ...owner, + id: "oversized", + data: JSON.stringify({ + item: { + id: "message", + type: "agentMessage", + text: "x".repeat(8_000_001), + }, + }), + }; + const big = expandSelectedCompletedItemRowsForProjection(f.db, [ + oversized, + ])[0]!.parsedData; + expect( + expandSelectedCompletedItemRowsForProjection(f.db, [oversized])[0]! + .parsedData, + ).not.toBe(big); + expect( + expandSelectedCompletedItemRowsForProjection(f.db, [owner])[0]! + .parsedData, + ).toBe(first); + } finally { + f.db.$client.close(); + } + }); + + it("charges the timeline budget for records inside a combined item", () => { + const f = setup(); + try { + const args = { + threadId: f.thread.id, + sequenceStart: 0, + excludedTypes: [], + eventBudget: 4, + }; + expect(findTimelineWindowBudgetFloorSequence(f.db, args)).toBe(1); + f.advance(); + expect(findTimelineWindowBudgetFloorSequence(f.db, args)).toBe(1); + expect( + findTimelineWindowBudgetFloorSequence(f.db, { + ...args, + eventBudget: 0, + }), + ).toBe(6); + expect( + findTimelineWindowBudgetFloorSequence(f.db, { + ...args, + eventBudget: 5, + }), + ).toBeUndefined(); + } finally { + f.db.$client.close(); + } + }); + + it("reads changed owner payloads and metadata without reusing stale history", () => { + const f = setup(); + try { + f.advance(); + expandSelectedCompletedItemRows(f.db, f.rows()); + f.db + .update(events) + .set({ + data: JSON.stringify({ + item: { id: "changed", type: "agentMessage", text: "new output" }, + }), + }) + .where(eq(events.id, "event-5")) + .run(); + const changedOwner = expandSelectedCompletedItemRows(f.db, f.rows()); + expect( + JSON.parse(changedOwner.find((row) => row.id === "event-2")!.data).item + .id, + ).toBe("changed"); + f.db + .update(events) + .set({ + completedItemHistory: sql`json_set(${events.completedItemHistory}, '$[3][1][0][5][0].item.text', 'changed start')`, + }) + .where(eq(events.id, "event-5")) + .run(); + const changedHistory = expandSelectedCompletedItemRows(f.db, f.rows()); + expect( + JSON.parse(changedHistory.find((row) => row.id === "event-2")!.data) + .item.text, + ).toBe("changed start"); + f.db + .update(events) + .set({ completedItemHistory: "broken" }) + .where(eq(events.id, "event-5")) + .run(); + expect(() => expandSelectedCompletedItemRows(f.db, f.rows())).toThrow(); + f.db + .update(events) + .set({ completedItemHistory: null }) + .where(eq(events.id, "event-5")) + .run(); + expect(expandSelectedCompletedItemRows(f.db, f.rows())).toEqual(f.rows()); + } finally { + f.db.$client.close(); + } + }); + + it("preserves completion ownership, timestamps, lossless history and highwater through atomic progress", () => { + const f = setup(); + try { + const before = f.rows(); + const generation = getThreadEventRewriteGeneration(f.thread.id); + f.db + .insert(retainedEventOutputs) + .values({ + eventId: "event-5", + outputPath: "resultText", + value: "retained", + expiresAt: 999999, + }) + .run(); + const result = f.advance(); + expect(result.removed).toBe(2); + const owner = f.rows().find((row) => row.id === "event-5"); + expect(owner).toMatchObject({ + sequence: 2, + createdAt: 500, + type: "item/completed", + data: before.find((row) => row.id === "event-5")!.data, + }); + expect( + normalized(expandSelectedCompletedItemRows(f.db, f.rows())), + ).toEqual(normalized(before)); + expect(getHighWaterMarks(f.db, [f.thread.id])[f.thread.id]).toBe(6); + expect(f.db.select().from(retainedEventOutputs).all()).toHaveLength(1); + expect(getThreadEventRewriteGeneration(f.thread.id)).toBe(generation + 1); + expect(f.db.select().from(threadPruningCursors).all()).toHaveLength(1); + expect(f.advance().removed).toBe(0); + } finally { + f.db.$client.close(); + } + }); + + it.each(["user", "context"])( + "leaves %s boundary crossings ordinary so edit suffix deletion remains valid", + (boundary) => { + const f = setup(); + try { + f.seed(4, { + type: + boundary === "user" ? "client/turn/requested" : "system/operation", + scopeKind: "thread", + turnId: null, + data: JSON.stringify( + boundary === "user" + ? { initiator: "user" } + : { operation: "context_clear", status: "completed" }, + ), + }); + expect(f.advance().removed).toBe(0); + f.db.transaction((tx) => + deleteThreadEventSuffixInTransaction(tx, { + threadId: f.thread.id, + cutoffSequence: 4, + oldMaxSequence: 6, + }), + ); + expect(f.rows().map((row) => row.sequence)).toEqual([1, 2, 3]); + } finally { + f.db.$client.close(); + } + }, + ); + + it.each(["unsettled", "duplicate", "late", "malformed", "oversized"])( + "skips %s lifecycles", + (reason) => { + const f = setup(); + try { + if (reason === "unsettled") + f.db.delete(events).where(eq(events.id, "event-6")).run(); + if (reason === "duplicate") + f.seed(4, { + type: "item/completed", + itemId: "message", + itemKind: "agentMessage", + }); + if (reason === "late") + f.seed(7, { type: "item/agentMessage/delta", itemId: "message" }); + if (reason === "malformed") + f.db + .update(events) + .set({ data: "invalid" }) + .where(eq(events.id, "event-3")) + .run(); + if (reason === "oversized") + f.db + .update(events) + .set({ + data: JSON.stringify({ + itemId: "message", + delta: "x".repeat(1024 * 1024), + }), + }) + .where(eq(events.id, "event-3")) + .run(); + const before = f.rows(); + expect(f.advance().removed).toBe(0); + expect(f.rows()).toEqual(before); + } finally { + f.db.$client.close(); + } + }, + ); + + it.each(["assistant", "manager"])( + "preserves latest output when a completion crosses another %s output", + (kind) => { + const f = setup(); + try { + f.seed( + 4, + kind === "manager" + ? { + type: "system/manager/user_message", + scopeKind: "thread", + turnId: null, + data: JSON.stringify({ text: "earlier manager output" }), + } + : { + type: "item/completed", + itemId: "earlier-message", + itemKind: "agentMessage", + data: JSON.stringify({ + item: { + id: "earlier-message", + type: "agentMessage", + text: "earlier assistant output", + }, + }), + }, + ); + const before = f.rows(); + const output = getLatestThreadOutputEventRow(f.db, { + threadId: f.thread.id, + }); + expect(output?.id).toBe("event-5"); + expect(f.advance().removed).toBe(0); + expect(f.rows()).toEqual(before); + expect( + getLatestThreadOutputEventRow(f.db, { threadId: f.thread.id }), + ).toEqual(output); + } finally { + f.db.$client.close(); + } + }, + ); + + it("rolls back moved owners and source deletion together", () => { + const f = setup(); + try { + const before = f.rows(); + expect(() => + f.db.transaction((tx) => { + expect( + advanceCompletedItemCompaction(tx, { + threadId: f.thread.id, + afterSequence: 0, + throughSequence: 6, + limit: 32, + }).removed, + ).toBe(2); + throw new Error("rollback"); + }), + ).toThrow("rollback"); + expect(f.rows()).toEqual(before); + } finally { + f.db.$client.close(); + } + }); + + it("keeps later arrivals ordinary and reconstructs history after completion output mutation", () => { + const f = setup(); + try { + const before = f.rows(); + expect(f.advance().removed).toBe(2); + f.db + .update(events) + .set({ + data: JSON.stringify({ + item: { + id: "message", + type: "agentMessage", + text: "lossless assistant text", + resultText: "expired", + }, + }), + }) + .where(eq(events.id, "event-5")) + .run(); + f.seed(7, { + type: "item/agentMessage/delta", + itemId: "message", + data: JSON.stringify({ itemId: "message", delta: "late text" }), + }); + for (let i = 0; i < 10; i++) + advanceThreadPruning(f.db, { threadId: f.thread.id }); + const expanded = expandSelectedCompletedItemRows(f.db, f.rows()); + expect(normalized(expanded.filter((row) => row.sequence <= 3))).toEqual( + normalized(before.filter((row) => row.sequence <= 3)), + ); + expect(expanded.find((row) => row.sequence === 7)).toBeDefined(); + } finally { + f.db.$client.close(); + } + }); + it.each([ + { output: "kept output", delta: "kept", start: true, removed: 2 }, + { output: "kept output", delta: "uncontained", start: true, removed: 2 }, + { output: "kept output", delta: "kept", start: false, removed: 1 }, + { output: "kept output", delta: "uncontained", start: false, removed: 0 }, + { output: "", delta: "keep this", start: true, removed: 0 }, + ])( + "only discards eligible command delta text: %j", + ({ output, delta, start, removed }) => { + const f = setup(); + try { + const item = { + id: "message", + type: "commandExecution", + command: "echo output", + cwd: "/tmp", + status: "completed", + approvalStatus: null, + aggregatedOutput: output, + }; + f.db + .update(events) + .set({ itemKind: "commandExecution", data: JSON.stringify({ item }) }) + .where(eq(events.id, "event-5")) + .run(); + f.db + .update(events) + .set({ + itemKind: "commandExecution", + data: JSON.stringify({ + item: { ...item, status: "pending", aggregatedOutput: "" }, + }), + }) + .where(eq(events.id, "event-2")) + .run(); + f.db + .update(events) + .set({ + type: "item/commandExecution/outputDelta", + data: JSON.stringify({ itemId: "message", delta }), + }) + .where(eq(events.id, "event-3")) + .run(); + if (!start) f.db.delete(events).where(eq(events.id, "event-2")).run(); + expect(f.advance().removed).toBe(removed); + const expanded = expandSelectedCompletedItemRows(f.db, f.rows()); + expect( + JSON.parse(expanded.find((row) => row.id === "event-3")!.data).delta, + ).toBe(removed ? "" : delta); + } finally { + f.db.$client.close(); + } + }, + ); + + it("retains both reasoning streams and traverses combined physical records at one-row budgets", () => { + const f = setup(); + try { + const item = { + id: "message", + type: "reasoning", + content: ["full reasoning"], + summary: ["summary"], + }; + f.db + .update(events) + .set({ itemKind: "reasoning", data: JSON.stringify({ item }) }) + .where(eq(events.id, "event-5")) + .run(); + f.db + .update(events) + .set({ + itemKind: "reasoning", + data: JSON.stringify({ item: { ...item, content: [], summary: [] } }), + }) + .where(eq(events.id, "event-2")) + .run(); + f.db + .update(events) + .set({ + type: "item/reasoning/textDelta", + data: JSON.stringify({ itemId: "message", delta: "full reasoning" }), + }) + .where(eq(events.id, "event-3")) + .run(); + f.seed(4, { + type: "item/reasoning/summaryTextDelta", + itemId: "message", + data: JSON.stringify({ itemId: "message", delta: "summary" }), + }); + const before = f.rows(); + expect(f.advance().removed).toBe(3); + const seen = []; + let afterSequence = 0; + for (;;) { + const page = listStoredEventRows(f.db, { + threadId: f.thread.id, + afterSequence, + limit: 1, + }); + if (!page.length) break; + seen.push(...page); + afterSequence = page[0]!.sequence; + } + expect(normalized(expandSelectedCompletedItemRows(f.db, seen))).toEqual( + normalized(before), + ); + } finally { + f.db.$client.close(); + } + }); +}); diff --git a/packages/db/test/migrate.test.ts b/packages/db/test/migrate.test.ts index 4da166327e3..f45452edab8 100644 --- a/packages/db/test/migrate.test.ts +++ b/packages/db/test/migrate.test.ts @@ -868,9 +868,15 @@ function rewindMachineProvidersMigration(db: DbConnection): void { "requested_by_thread_id", ]) { if (!queuedDispatchOrigin.some((column) => column.name === name)) continue; - db.$client.exec( - `ALTER TABLE queued_thread_messages DROP COLUMN ${name}`, - ); + db.$client.exec(`ALTER TABLE queued_thread_messages DROP COLUMN ${name}`); + } + if ( + db.$client + .prepare<[], TableInfoRow>("PRAGMA table_info(events)") + .all() + .some((column) => column.name === "completed_item_history") + ) { + db.$client.exec("ALTER TABLE events DROP COLUMN completed_item_history"); } db.$client.exec("DROP TABLE IF EXISTS thread_pruning_cursors"); db.$client.exec("DROP TABLE IF EXISTS project_attachment_threads"); @@ -4567,6 +4573,7 @@ describe("migrate", () => { "data", "created_at", "parent_tool_call_id", + "completed_item_history", ]); const eventIndexNames = readIndexNames({ db, diff --git a/packages/plugin-api-map/src/surfaces.ts b/packages/plugin-api-map/src/surfaces.ts index 3b13c06fd48..ac715037e87 100644 --- a/packages/plugin-api-map/src/surfaces.ts +++ b/packages/plugin-api-map/src/surfaces.ts @@ -995,6 +995,7 @@ export const SURFACE_GROUPS: SurfaceGroup[] = [ "Spawn or fork with lifecycleOwnerThreadId to archive/delete a dependent with a live owner across projects; ownership is immutable, independent of sidebar parents and supports different hosts/environments. Thread responses return the owner or null. Unarchive owner first; Stop does not cascade", "List machines and suspend, resume, or remove provider-managed machines", "Read recorded context usage with sdk.threads.context({ threadId }); usage is null when unavailable, and its snapshot is present only when the latest measurement includes a breakdown", + "Read physical records through sdk.threads.events.list: settled items may combine at their first lifecycle sequence, retaining completion ID/payload/timestamp; raw exports omit internal start/delta history and budgets count physical rows", "Reach the same operations the [bb CLI](cli) and the bb UI use", "Have the threads it creates attributed back to the plugin", "Read the server's loopback URL, public app URL, and data directory when it needs server facts", diff --git a/packages/templates/src/templates/bb-guide-threads.md b/packages/templates/src/templates/bb-guide-threads.md index 4e9483fc6e3..1034e6d8e21 100644 --- a/packages/templates/src/templates/bb-guide-threads.md +++ b/packages/templates/src/templates/bb-guide-threads.md @@ -209,6 +209,13 @@ Inspecting: walks a consistent history snapshot and joins paginated group contents. Appends stay outside that walk; rerun the command if a history edit invalidates it. + Raw JSON (including JSON exports and SDK threads.events.list) contains physical + records. Settled items may be one item/completed at their first lifecycle + sequence, keeping the completion ID, payload and completion timestamp. Separate + start/delta IDs disappear; internal timing metadata is not part of raw output. + Limits and sequence cursors count these physical rows. Human timelines retain + original ordering and timing; the same row budget can include older history. + bb thread output [id] Get the final output of a thread --self Target current thread diff --git a/plugins/bb-guide/skills/bb-cli/references/thread-operation.md b/plugins/bb-guide/skills/bb-cli/references/thread-operation.md index 774043f029a..e7acd33b475 100644 --- a/plugins/bb-guide/skills/bb-cli/references/thread-operation.md +++ b/plugins/bb-guide/skills/bb-cli/references/thread-operation.md @@ -106,7 +106,11 @@ hostId, providerId, projectId, parentThreadId, groupBy })`. snapshot and join paginated group contents. Appends remain outside that walk; rerun after a cursor-invalidated error from a history edit. JSON accepts any positive limit. It defaults to the oldest 100 raw events and warns when more exist. Page with - `--after-seq ` or pass `--all`. + `--after-seq ` or pass `--all`. Raw JSON exports use physical records: + a settled item can appear as one `item/completed` at its first lifecycle + sequence, retaining its completion ID, payload and timestamp. Start/delta IDs + are no longer separate rows. Human timelines preserve original timing/order; + smaller physical-row budgets can include more history. Grep the `--all` output, not the default page, when checking whether a thread ever received a message. - Use `bb thread output ` to read the latest final output, or