diff --git a/crates/persistence/src/backends/sqlite/backend.rs b/crates/persistence/src/backends/sqlite/backend.rs index d5159c419..21c3951b9 100644 --- a/crates/persistence/src/backends/sqlite/backend.rs +++ b/crates/persistence/src/backends/sqlite/backend.rs @@ -327,6 +327,25 @@ impl SqliteBackend { if enable_foreign_keys { conn.execute_batch("PRAGMA foreign_keys = ON;")?; } + // 64 MiB page cache (negative = KiB). SQLite's 2 MiB default + // thrashes once the search_index B-trees outgrow it: during a + // bulk import every index INSERT and batch commit evicts and + // rewrites hot pages. Measured on a real 31 GB import, raising it + // took ingest from 232/s to 289/s (+25%), cutting index-row + // INSERT cost 1.71→1.24 ms and commit cost 1.95→1.72 ms per + // entry. The limit is per pooled connection (pool of 10), but + // only connections that touch that many pages grow their cache. + conn.execute_batch("PRAGMA cache_size = -65536;")?; + // Checkpoint every ~50,000 WAL pages (~200 MiB) instead of every + // 1,000 (~4 MiB). A bulk-ingest batch writes far more than 4 MiB + // of WAL, so with the default every batch commit also ran a + // checkpoint and paid the WAL->database copy inline; fewer, + // larger checkpoints move the same bytes sequentially. Measured + // stepwise on the same real-manifest window: 10,000 pages took + // 289/s -> 382/s (commit 1.72 -> 0.96 ms per entry) and 50,000 + // took 417/s -> 501/s on the multi-row branch (commit -> 0.55 ms). + // Cost: the -wal file grows to ~200 MiB under sustained writes. + conn.execute_batch("PRAGMA wal_autocheckpoint = 50000;")?; crate::sof::sqlite_udfs::register(conn).map_err(|e| { rusqlite::Error::SqliteFailure( rusqlite::ffi::Error::new(rusqlite::ffi::SQLITE_ERROR),