From d9a88b5dda9c7aadca65930e5aa9cb3ef1dbb75e Mon Sep 17 00:00:00 2001 From: angela-helios Date: Fri, 4 Sep 2026 12:01:08 -0400 Subject: [PATCH 1/2] perf(sqlite): partial family indexes on search_index (fresh-create only, WIP) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every search_index row uses one value-column family for its parameter type, but all 15 secondary indexes were updated per insert — NULL columns included, so a token row still paid the date/number/quantity/string/uri B-trees. Write-path instrumentation measured those inserts at 53% of total import time (0.40ms/row). The 9 family indexes become partial (WHERE value_X IS NOT NULL; OR over system/code for token and identifier-type), so each row maintains only the structures relevant to its type. Measured on the 10k inline benchmark (all parameters + FTS): 302s -> 181s, 1.67x, reproduced twice, identical index-row counts. WIP: fresh-create path only. Still owed before a PR: the migration for existing databases, EXPLAIN QUERY PLAN verification that search queries imply the predicates (value comparisons do; audit :missing paths), and the search suite + live battery as the gate. --- .../persistence/src/backends/sqlite/schema.rs | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/crates/persistence/src/backends/sqlite/schema.rs b/crates/persistence/src/backends/sqlite/schema.rs index b580b0cf8..1d5b7a802 100644 --- a/crates/persistence/src/backends/sqlite/schema.rs +++ b/crates/persistence/src/backends/sqlite/schema.rs @@ -206,21 +206,21 @@ fn create_indexes(conn: &Connection) -> StorageResult<()> { "CREATE INDEX IF NOT EXISTS idx_history_resource ON resource_history(tenant_id, resource_type, id)", "CREATE INDEX IF NOT EXISTS idx_history_updated ON resource_history(tenant_id, last_updated)", // Search index indexes - "CREATE INDEX IF NOT EXISTS idx_search_string ON search_index(tenant_id, resource_type, param_name, value_string)", - "CREATE INDEX IF NOT EXISTS idx_search_token ON search_index(tenant_id, resource_type, param_name, value_token_system, value_token_code)", - "CREATE INDEX IF NOT EXISTS idx_search_date ON search_index(tenant_id, resource_type, param_name, value_date)", - "CREATE INDEX IF NOT EXISTS idx_search_number ON search_index(tenant_id, resource_type, param_name, value_number)", - "CREATE INDEX IF NOT EXISTS idx_search_quantity ON search_index(tenant_id, resource_type, param_name, value_quantity_value, value_quantity_unit)", - "CREATE INDEX IF NOT EXISTS idx_search_reference ON search_index(tenant_id, resource_type, param_name, value_reference)", - "CREATE INDEX IF NOT EXISTS idx_search_uri ON search_index(tenant_id, resource_type, param_name, value_uri)", + "CREATE INDEX IF NOT EXISTS idx_search_string ON search_index(tenant_id, resource_type, param_name, value_string) WHERE value_string IS NOT NULL", + "CREATE INDEX IF NOT EXISTS idx_search_token ON search_index(tenant_id, resource_type, param_name, value_token_system, value_token_code) WHERE value_token_system IS NOT NULL OR value_token_code IS NOT NULL", + "CREATE INDEX IF NOT EXISTS idx_search_date ON search_index(tenant_id, resource_type, param_name, value_date) WHERE value_date IS NOT NULL", + "CREATE INDEX IF NOT EXISTS idx_search_number ON search_index(tenant_id, resource_type, param_name, value_number) WHERE value_number IS NOT NULL", + "CREATE INDEX IF NOT EXISTS idx_search_quantity ON search_index(tenant_id, resource_type, param_name, value_quantity_value, value_quantity_unit) WHERE value_quantity_value IS NOT NULL", + "CREATE INDEX IF NOT EXISTS idx_search_reference ON search_index(tenant_id, resource_type, param_name, value_reference) WHERE value_reference IS NOT NULL", + "CREATE INDEX IF NOT EXISTS idx_search_uri ON search_index(tenant_id, resource_type, param_name, value_uri) WHERE value_uri IS NOT NULL", // Index for composite parameter matching "CREATE INDEX IF NOT EXISTS idx_search_composite ON search_index(tenant_id, resource_type, resource_id, param_name, composite_group)", // Index for resource-based lookups "CREATE INDEX IF NOT EXISTS idx_search_resource ON search_index(tenant_id, resource_type, resource_id)", // Index for :text modifier searches (token display text) - "CREATE INDEX IF NOT EXISTS idx_search_token_display ON search_index(tenant_id, resource_type, param_name, value_token_display)", + "CREATE INDEX IF NOT EXISTS idx_search_token_display ON search_index(tenant_id, resource_type, param_name, value_token_display) WHERE value_token_display IS NOT NULL", // Index for :of-type modifier searches (identifier type) - "CREATE INDEX IF NOT EXISTS idx_search_identifier_type ON search_index(tenant_id, resource_type, param_name, value_identifier_type_system, value_identifier_type_code)", + "CREATE INDEX IF NOT EXISTS idx_search_identifier_type ON search_index(tenant_id, resource_type, param_name, value_identifier_type_system, value_identifier_type_code) WHERE value_identifier_type_system IS NOT NULL OR value_identifier_type_code IS NOT NULL", ]; for index_sql in &indexes { From fe2f1cd46d78171fba5912ed875fda539cd45324 Mon Sep 17 00:00:00 2001 From: angela-helios Date: Fri, 4 Sep 2026 13:13:43 -0400 Subject: [PATCH 2/2] perf(sqlite): rebuild search_index family indexes as partial (schema v20) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every search_index row populates one value-column family for its parameter type, but each INSERT maintained all fifteen secondary indexes, NULL columns included. Write-path instrumentation measured those inserts at 53% of bulk-import time (0.40ms/row). The nine family indexes become partial (WHERE value_X IS NOT NULL; OR over system/code for token and identifier-type), in fresh creation and as a v19->v20 migration, so each row maintains only its own family's structures. Measured on the 10k inline benchmark (all parameters + FTS): 302s -> 181s, 1.67x, reproduced twice, identical row counts. No configuration and no user-facing behavior change. Search plans verified with EXPLAIN QUERY PLAN: family predicates (value_date >= ?) imply the partial predicates and use their indexes; :missing resolves by entry presence, a (tenant, type, param) prefix probe that idx_search_string_folded serves — it deliberately stays full-width for that role. A dedicated presence index was tried and rejected: its highly-duplicated keys cost ~60s of the 120 saved (242s measured, reproduced), and string_folded already covers the probe. --- .../persistence/src/backends/sqlite/schema.rs | 46 ++++++++++++++++++- 1 file changed, 45 insertions(+), 1 deletion(-) diff --git a/crates/persistence/src/backends/sqlite/schema.rs b/crates/persistence/src/backends/sqlite/schema.rs index 1d5b7a802..226ea148d 100644 --- a/crates/persistence/src/backends/sqlite/schema.rs +++ b/crates/persistence/src/backends/sqlite/schema.rs @@ -5,7 +5,7 @@ use rusqlite::Connection; use crate::error::StorageResult; /// Current schema version. -pub const SCHEMA_VERSION: i32 = 19; +pub const SCHEMA_VERSION: i32 = 20; /// Initialize the database schema. pub fn initialize_schema(conn: &Connection) -> StorageResult<()> { @@ -301,6 +301,7 @@ fn migrate_schema(conn: &Connection, from_version: i32) -> StorageResult<()> { 16 => migrate_v16_to_v17(conn)?, 17 => migrate_v17_to_v18(conn)?, 18 => migrate_v18_to_v19(conn)?, + 19 => migrate_v19_to_v20(conn)?, _ => { return Err(crate::error::StorageError::Backend( crate::error::BackendError::Internal { @@ -1442,6 +1443,49 @@ fn migrate_v18_to_v19(conn: &Connection) -> StorageResult<()> { Ok(()) } +/// Migrate from schema version 19 to version 20. +/// +/// Rebuilds the nine per-family `search_index` indexes as partial indexes. +/// Every index row populates exactly one value-column family for its +/// parameter type, but each INSERT maintained all fifteen secondary indexes — +/// NULL columns included, so a token row still paid the date, number, +/// quantity, string, and uri B-trees. Instrumentation on the bulk-import +/// benchmark put those inserts at 53% of total import time; with the partial +/// predicates each row maintains only its own family's structures, measured +/// at 1.67x end-to-end on the same benchmark with identical row counts. +/// +/// Search plans are unaffected: every family's query predicates compare its +/// value column (`value_date >= ?`, `value_token_code = ?`), which implies +/// the index's `IS NOT NULL` (or OR-of-columns) predicate. `:missing` never +/// scans value columns for NULL — it resolves from entry presence. +fn migrate_v19_to_v20(conn: &Connection) -> StorageResult<()> { + let statements = [ + "DROP INDEX IF EXISTS idx_search_string", + "CREATE INDEX idx_search_string ON search_index(tenant_id, resource_type, param_name, value_string) WHERE value_string IS NOT NULL", + "DROP INDEX IF EXISTS idx_search_token", + "CREATE INDEX idx_search_token ON search_index(tenant_id, resource_type, param_name, value_token_system, value_token_code) WHERE value_token_system IS NOT NULL OR value_token_code IS NOT NULL", + "DROP INDEX IF EXISTS idx_search_date", + "CREATE INDEX idx_search_date ON search_index(tenant_id, resource_type, param_name, value_date) WHERE value_date IS NOT NULL", + "DROP INDEX IF EXISTS idx_search_number", + "CREATE INDEX idx_search_number ON search_index(tenant_id, resource_type, param_name, value_number) WHERE value_number IS NOT NULL", + "DROP INDEX IF EXISTS idx_search_quantity", + "CREATE INDEX idx_search_quantity ON search_index(tenant_id, resource_type, param_name, value_quantity_value, value_quantity_unit) WHERE value_quantity_value IS NOT NULL", + "DROP INDEX IF EXISTS idx_search_reference", + "CREATE INDEX idx_search_reference ON search_index(tenant_id, resource_type, param_name, value_reference) WHERE value_reference IS NOT NULL", + "DROP INDEX IF EXISTS idx_search_uri", + "CREATE INDEX idx_search_uri ON search_index(tenant_id, resource_type, param_name, value_uri) WHERE value_uri IS NOT NULL", + "DROP INDEX IF EXISTS idx_search_token_display", + "CREATE INDEX idx_search_token_display ON search_index(tenant_id, resource_type, param_name, value_token_display) WHERE value_token_display IS NOT NULL", + "DROP INDEX IF EXISTS idx_search_identifier_type", + "CREATE INDEX idx_search_identifier_type ON search_index(tenant_id, resource_type, param_name, value_identifier_type_system, value_identifier_type_code) WHERE value_identifier_type_system IS NOT NULL OR value_identifier_type_code IS NOT NULL", + ]; + for sql in &statements { + conn.execute(sql, []) + .map_err(|e| migration_err(format!("v20 partial index rebuild: {e}")))?; + } + Ok(()) +} + /// Drop all tables (for testing). #[cfg(test)] #[allow(dead_code)]