From 9c8a8113b20281ee74755355e212481d4e59b256 Mon Sep 17 00:00:00 2001 From: angela-helios Date: Sat, 5 Sep 2026 01:41:33 -0400 Subject: [PATCH 1/2] perf(sqlite): write search_index rows eight per INSERT A resource writes ~10-30 search_index rows, each via a single-row prepared INSERT stepped once per row. The B-tree maintenance is the irreducible cost, but entering and leaving the statement per row is not: batching eight rows per INSERT (with a single-row remainder) amortizes it. Measured on the real 31 GB bulk-submit manifest, identical 7-minute windows, on top of the FTS-scan fix and the write-path pragma defaults: 382/s -> 417/s (+9%), index-row INSERT cost 1.11 -> 0.85 ms per entry, identical index-row counts per resource. File fan-out at 2 was also measured on the optimized write path and came out neutral-to-negative (388/s vs 417/s, no lock errors): with extraction and fetch already cheap, a second in-flight file only adds write-lock handoffs. SQLite's default file concurrency of 1 stands. --- .../src/backends/sqlite/search/writer.rs | 13 +++++ .../src/backends/sqlite/storage.rs | 53 +++++++++++++++++-- 2 files changed, 63 insertions(+), 3 deletions(-) diff --git a/crates/persistence/src/backends/sqlite/search/writer.rs b/crates/persistence/src/backends/sqlite/search/writer.rs index 1e2128226..03f0f7cfb 100644 --- a/crates/persistence/src/backends/sqlite/search/writer.rs +++ b/crates/persistence/src/backends/sqlite/search/writer.rs @@ -86,6 +86,19 @@ impl SqliteSearchIndexWriter { /// Converts an ExtractedValue to SQL parameters. /// /// Returns a tuple of (column_values) where each value corresponds to a column. + /// Multi-row variant of [`Self::insert_sql`]: one INSERT carrying eight + /// rows (8 x 24 positional parameters). Bulk indexing executes this once + /// per chunk instead of stepping the single-row statement eight times. + pub fn insert_sql_rows8() -> &'static str { + static SQL: std::sync::OnceLock = std::sync::OnceLock::new(); + SQL.get_or_init(|| { + let base = Self::insert_sql(); + let cols = &base[..base.find("VALUES").expect("insert_sql has VALUES")]; + let group = format!("({})", ["?"; 24].join(", ")); + format!("{cols}VALUES {}", vec![group; 8].join(", ")) + }) + } + pub fn to_sql_params( tenant_id: &str, resource_type: &str, diff --git a/crates/persistence/src/backends/sqlite/storage.rs b/crates/persistence/src/backends/sqlite/storage.rs index 840e0b852..2d1ca013e 100644 --- a/crates/persistence/src/backends/sqlite/storage.rs +++ b/crates/persistence/src/backends/sqlite/storage.rs @@ -1249,10 +1249,57 @@ impl SqliteBackend { .extract(resource, resource_type) .map_err(|e| internal_error(format!("Search parameter extraction failed: {}", e)))?; + // Rows are written eight at a time: a single-row INSERT stepped once + // per row spends a measurable share of its time entering and leaving + // the statement, and every resource writes ~10-30 rows. The B-tree + // work is unchanged; only the per-statement overhead is amortized + // (measured +9% bulk-ingest throughput on the real 31 GB manifest). let mut count = 0; - for value in values { - self.write_index_entry(conn, tenant_id, resource_type, resource_id, &value)?; - count += 1; + { + use crate::search::converters::IndexValue; + let owned: Vec<_> = values + .into_iter() + .map(|v| match &v.value { + IndexValue::Date { + value: d, + precision, + } => { + let mut n = v.clone(); + n.value = IndexValue::Date { + value: Self::normalize_date_for_sqlite(d), + precision: *precision, + }; + n + } + _ => v, + }) + .collect(); + let rows: Vec> = owned + .iter() + .map(|v| { + SqliteSearchIndexWriter::to_sql_params(tenant_id, resource_type, resource_id, v) + }) + .collect(); + let mut i = 0; + while i + 8 <= rows.len() { + let refs: Vec<&dyn ToSql> = rows[i..i + 8] + .iter() + .flatten() + .map(|p| self.sql_value_to_ref(p)) + .collect(); + conn.prepare_cached(SqliteSearchIndexWriter::insert_sql_rows8()) + .and_then(|mut s| s.execute(refs.as_slice())) + .map_err(|e| internal_error(format!("multi-row index insert: {e}")))?; + i += 8; + count += 8; + } + for row in &rows[i..] { + let refs: Vec<&dyn ToSql> = row.iter().map(|p| self.sql_value_to_ref(p)).collect(); + conn.prepare_cached(SqliteSearchIndexWriter::insert_sql()) + .and_then(|mut s| s.execute(refs.as_slice())) + .map_err(|e| internal_error(format!("index insert: {e}")))?; + count += 1; + } } // Also index any contained resources for `_contained` search. From d245967ee72eaa6425b76d8f4534afb645cd866d Mon Sep 17 00:00:00 2001 From: smunini Date: Sat, 5 Sep 2026 09:39:44 -0400 Subject: [PATCH 2/2] fix(clippy): restore to_sql_params doc comment insert_sql_rows8 was inserted between to_sql_params' doc comment and its signature, so the new function absorbed those docs and to_sql_params was left undocumented, failing the workspace `-D missing-docs` lint. Move the new function above the block and give each its own docs. Claude-Session: https://claude.ai/code/session_016kNZJTXf86QoMqcdWqKCbR --- crates/persistence/src/backends/sqlite/search/writer.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/crates/persistence/src/backends/sqlite/search/writer.rs b/crates/persistence/src/backends/sqlite/search/writer.rs index 03f0f7cfb..536f19adb 100644 --- a/crates/persistence/src/backends/sqlite/search/writer.rs +++ b/crates/persistence/src/backends/sqlite/search/writer.rs @@ -83,9 +83,6 @@ impl SqliteSearchIndexWriter { "DELETE FROM search_index WHERE tenant_id = ?1 AND resource_type = ?2 AND resource_id = ?3 AND param_name = ?4" } - /// Converts an ExtractedValue to SQL parameters. - /// - /// Returns a tuple of (column_values) where each value corresponds to a column. /// Multi-row variant of [`Self::insert_sql`]: one INSERT carrying eight /// rows (8 x 24 positional parameters). Bulk indexing executes this once /// per chunk instead of stepping the single-row statement eight times. @@ -99,6 +96,9 @@ impl SqliteSearchIndexWriter { }) } + /// Converts an ExtractedValue to SQL parameters. + /// + /// Returns a tuple of (column_values) where each value corresponds to a column. pub fn to_sql_params( tenant_id: &str, resource_type: &str,