From 23a51d7393d8472af7ff6ad9417bec8170266dfb Mon Sep 17 00:00:00 2001 From: angela-helios Date: Fri, 4 Sep 2026 22:14:33 -0400 Subject: [PATCH 1/3] perf(sqlite): raise the per-connection page cache to 64 MiB MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit SQLite's default page cache is 2 MiB per connection. Once the search_index B-trees outgrow it — immediately, on any real dataset — every index INSERT and batch commit evicts and rewrites hot pages, so bulk ingest spends its time shuffling the cache instead of building the trees. Measured on the real 31 GB bulk-submit manifest (single worker, identical 7-minute windows on a fresh database): 232/s -> 289/s (+25%), with index-row INSERT cost falling 1.71 -> 1.24 ms per entry and batch commit 1.95 -> 1.72 ms. A larger batch size was also tried and measured slower (205/s at 5,000 entries): bigger transactions overflow the cache and grow the commit-time checkpoint, which is exactly what the larger cache absorbs. The limit is per pooled connection (pool of 10), but a connection's cache only grows with the pages it actually touches, so idle and read-only connections stay small. --- crates/persistence/src/backends/sqlite/backend.rs | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/crates/persistence/src/backends/sqlite/backend.rs b/crates/persistence/src/backends/sqlite/backend.rs index d5159c419..d324fcce6 100644 --- a/crates/persistence/src/backends/sqlite/backend.rs +++ b/crates/persistence/src/backends/sqlite/backend.rs @@ -327,6 +327,15 @@ impl SqliteBackend { if enable_foreign_keys { conn.execute_batch("PRAGMA foreign_keys = ON;")?; } + // 64 MiB page cache (negative = KiB). SQLite's 2 MiB default + // thrashes once the search_index B-trees outgrow it: during a + // bulk import every index INSERT and batch commit evicts and + // rewrites hot pages. Measured on a real 31 GB import, raising it + // took ingest from 232/s to 289/s (+25%), cutting index-row + // INSERT cost 1.71→1.24 ms and commit cost 1.95→1.72 ms per + // entry. The limit is per pooled connection (pool of 10), but + // only connections that touch that many pages grow their cache. + conn.execute_batch("PRAGMA cache_size = -65536;")?; crate::sof::sqlite_udfs::register(conn).map_err(|e| { rusqlite::Error::SqliteFailure( rusqlite::ffi::Error::new(rusqlite::ffi::SQLITE_ERROR), From 10a1d382a3e69f77c13a5368115331088da43399 Mon Sep 17 00:00:00 2001 From: angela-helios Date: Fri, 4 Sep 2026 23:53:24 -0400 Subject: [PATCH 2/3] perf(sqlite): checkpoint the WAL every 10,000 pages instead of 1,000 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A bulk-ingest batch writes far more than the default 4 MiB wal_autocheckpoint threshold, so every batch commit also ran a checkpoint and paid the WAL-to-database copy inline — commit cost was 1.72 ms per entry, ~50% of the remaining ingest budget. Fewer, larger checkpoints move the same bytes sequentially: measured on the same real-manifest window, 289/s -> 382/s (+32%), with batch-commit cost falling to 0.96 ms per entry. The -wal file now grows to ~40 MiB between checkpoints. --- crates/persistence/src/backends/sqlite/backend.rs | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/crates/persistence/src/backends/sqlite/backend.rs b/crates/persistence/src/backends/sqlite/backend.rs index d324fcce6..606b0bf50 100644 --- a/crates/persistence/src/backends/sqlite/backend.rs +++ b/crates/persistence/src/backends/sqlite/backend.rs @@ -336,6 +336,15 @@ impl SqliteBackend { // entry. The limit is per pooled connection (pool of 10), but // only connections that touch that many pages grow their cache. conn.execute_batch("PRAGMA cache_size = -65536;")?; + // Checkpoint every ~10,000 WAL pages (~40 MiB) instead of every + // 1,000 (~4 MiB). A bulk-ingest batch writes far more than 4 MiB + // of WAL, so with the default every batch commit also ran a + // checkpoint and paid the WAL->database copy inline; fewer, + // larger checkpoints move the same bytes sequentially. Measured + // on the same real-manifest window: 289/s -> 382/s (+32%), with + // batch-commit cost falling 1.72 -> 0.96 ms per entry. Cost: the + // -wal file grows to ~40 MiB between checkpoints. + conn.execute_batch("PRAGMA wal_autocheckpoint = 10000;")?; crate::sof::sqlite_udfs::register(conn).map_err(|e| { rusqlite::Error::SqliteFailure( rusqlite::ffi::Error::new(rusqlite::ffi::SQLITE_ERROR), From c128d28129c72a7b8add0f081c2f2bc102ed6a94 Mon Sep 17 00:00:00 2001 From: angela-helios Date: Sat, 5 Sep 2026 05:50:06 -0400 Subject: [PATCH 3/3] perf(sqlite): raise the WAL autocheckpoint threshold to 50,000 pages Measured stepwise on the same real-manifest window: 10,000 pages took ingest from 289/s to 382/s (batch commit 1.72 -> 0.96 ms per entry); 50,000 pages takes the multi-row branch from 417/s to 501/s (commit -> 0.55 ms per entry). The -wal file grows to ~200 MiB under sustained writes, which a server ingesting tens of gigabytes can afford; light workloads rarely reach the threshold between their own commits. --- crates/persistence/src/backends/sqlite/backend.rs | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/crates/persistence/src/backends/sqlite/backend.rs b/crates/persistence/src/backends/sqlite/backend.rs index 606b0bf50..21c3951b9 100644 --- a/crates/persistence/src/backends/sqlite/backend.rs +++ b/crates/persistence/src/backends/sqlite/backend.rs @@ -336,15 +336,16 @@ impl SqliteBackend { // entry. The limit is per pooled connection (pool of 10), but // only connections that touch that many pages grow their cache. conn.execute_batch("PRAGMA cache_size = -65536;")?; - // Checkpoint every ~10,000 WAL pages (~40 MiB) instead of every + // Checkpoint every ~50,000 WAL pages (~200 MiB) instead of every // 1,000 (~4 MiB). A bulk-ingest batch writes far more than 4 MiB // of WAL, so with the default every batch commit also ran a // checkpoint and paid the WAL->database copy inline; fewer, // larger checkpoints move the same bytes sequentially. Measured - // on the same real-manifest window: 289/s -> 382/s (+32%), with - // batch-commit cost falling 1.72 -> 0.96 ms per entry. Cost: the - // -wal file grows to ~40 MiB between checkpoints. - conn.execute_batch("PRAGMA wal_autocheckpoint = 10000;")?; + // stepwise on the same real-manifest window: 10,000 pages took + // 289/s -> 382/s (commit 1.72 -> 0.96 ms per entry) and 50,000 + // took 417/s -> 501/s on the multi-row branch (commit -> 0.55 ms). + // Cost: the -wal file grows to ~200 MiB under sustained writes. + conn.execute_batch("PRAGMA wal_autocheckpoint = 50000;")?; crate::sof::sqlite_udfs::register(conn).map_err(|e| { rusqlite::Error::SqliteFailure( rusqlite::ffi::Error::new(rusqlite::ffi::SQLITE_ERROR),