From ec42927b5d627beabe7c8a1010055c305e67baef Mon Sep 17 00:00:00 2001 From: "google-labs-jules[bot]" <161369871+google-labs-jules[bot]@users.noreply.github.com> Date: Fri, 12 Jun 2026 17:59:45 +0000 Subject: [PATCH] Optimize dynamic SQL generation in D1 targets Replaced intermediate `Vec` allocations and `format!` operations with pre-allocated `String::with_capacity` and `write!` macro calls. This significantly reduces heap allocations and string copies. Co-authored-by: bashandbone <89049923+bashandbone@users.noreply.github.com> --- .jules/bolt.md | 4 ++ crates/flow/src/targets/d1.rs | 104 +++++++++++++++++++++++----------- 2 files changed, 76 insertions(+), 32 deletions(-) diff --git a/.jules/bolt.md b/.jules/bolt.md index fb3e8f19..af82f9f6 100644 --- a/.jules/bolt.md +++ b/.jules/bolt.md @@ -2,3 +2,7 @@ ## 2026-04-08 - [Performance: Defer Allocation during Traversal] **Learning:** During DAG traversals, creating owned variants of identifiers (like `file.to_path_buf()`) *before* checking `visited` HashSets results in heap allocations (O(E)) for every edge instead of every visited node (O(V)). By moving the `&PathBuf` allocation strictly *after* all HashSet `contains` checks using the borrowed reference (`&Path`), we drastically reduce memory churn. **Action:** Always check `HashSet::contains` with a borrowed reference *before* creating the owned version required by `HashSet::insert`, especially in performance-critical graph traversal paths. + +## 2024-05-24 - Dynamic SQL Generation Optimization +**Learning:** For dynamic SQL generation (e.g., in `crates/flow/src/targets/d1.rs` for Cloudflare D1 targets), using intermediate `Vec` allocations and `format!` in loops is a significant performance bottleneck due to excessive heap allocations and string copies. +**Action:** Always use `String::with_capacity` and the `write!` macro (via `std::fmt::Write`) to construct queries directly to minimize heap allocations and string copies. diff --git a/crates/flow/src/targets/d1.rs b/crates/flow/src/targets/d1.rs index e45fd523..21544cc3 100644 --- a/crates/flow/src/targets/d1.rs +++ b/crates/flow/src/targets/d1.rs @@ -300,40 +300,75 @@ impl D1ExportContext { key: &KeyValue, values: &FieldValues, ) -> Result<(String, Vec), RecocoError> { - let mut columns = vec![]; - let mut placeholders = vec![]; - let mut params = vec![]; - let mut update_clauses = vec![]; + use std::fmt::Write; + + let mut params = + Vec::with_capacity(self.key_fields_schema.len() + self.value_fields_schema.len()); + + // Pre-calculate string capacity to avoid reallocations + // "INSERT INTO " (12) + table_name + " (" (2) + columns + ") VALUES (" (10) + placeholders + ") ON CONFLICT DO UPDATE SET " (28) + update_clauses + let estimated_len = 12 + + self.table_name.len() + + 2 + + (self.key_fields_schema.len() + self.value_fields_schema.len()) * 15 + + 10 + + (self.key_fields_schema.len() + self.value_fields_schema.len()) * 3 + + 28 + + self.value_fields_schema.len() * 25; + let mut sql = String::with_capacity(estimated_len); - // Extract key parts - KeyValue is a wrapper around Box<[KeyPart]> - for (idx, _key_field) in self.key_fields_schema.iter().enumerate() { + let _ = write!(sql, "INSERT INTO {} (", self.table_name); + + let mut first = true; + + // Extract key parts + for (idx, key_field) in self.key_fields_schema.iter().enumerate() { if let Some(key_part) = key.0.get(idx) { - columns.push(self.key_fields_schema[idx].name.clone()); - placeholders.push("?".to_string()); + if !first { + let _ = write!(sql, ", "); + } + let _ = write!(sql, "{}", key_field.name); + first = false; params.push(key_part_to_json(key_part)?); } } - // Add value fields + // Add value fields for columns for (idx, value) in values.fields.iter().enumerate() { if let Some(value_field) = self.value_fields_schema.get(idx) { - columns.push(value_field.name.clone()); - placeholders.push("?".to_string()); + if !first { + let _ = write!(sql, ", "); + } + let _ = write!(sql, "{}", value_field.name); + first = false; params.push(value_to_json(value)?); - update_clauses.push(format!( - "{} = excluded.{}", - value_field.name, value_field.name - )); } } - let sql = format!( - "INSERT INTO {} ({}) VALUES ({}) ON CONFLICT DO UPDATE SET {}", - self.table_name, - columns.join(", "), - placeholders.join(", "), - update_clauses.join(", ") - ); + let _ = write!(sql, ") VALUES ("); + + // Placeholders + for i in 0..params.len() { + if i > 0 { + let _ = write!(sql, ", ?"); + } else { + let _ = write!(sql, "?"); + } + } + + let _ = write!(sql, ") ON CONFLICT DO UPDATE SET "); + + // Update clauses + first = true; + for (idx, _value) in values.fields.iter().enumerate() { + if let Some(value_field) = self.value_fields_schema.get(idx) { + if !first { + let _ = write!(sql, ", "); + } + let _ = write!(sql, "{name} = excluded.{name}", name = value_field.name); + first = false; + } + } Ok((sql, params)) } @@ -342,22 +377,27 @@ impl D1ExportContext { &self, key: &KeyValue, ) -> Result<(String, Vec), RecocoError> { - let mut where_clauses = vec![]; - let mut params = vec![]; + use std::fmt::Write; + + let mut params = Vec::with_capacity(self.key_fields_schema.len()); - for (idx, _key_field) in self.key_fields_schema.iter().enumerate() { + let estimated_len = 12 + self.table_name.len() + 7 + self.key_fields_schema.len() * 20; + let mut sql = String::with_capacity(estimated_len); + + let _ = write!(sql, "DELETE FROM {} WHERE ", self.table_name); + + let mut first = true; + for (idx, key_field) in self.key_fields_schema.iter().enumerate() { if let Some(key_part) = key.0.get(idx) { - where_clauses.push(format!("{} = ?", self.key_fields_schema[idx].name)); + if !first { + let _ = write!(sql, " AND "); + } + let _ = write!(sql, "{} = ?", key_field.name); + first = false; params.push(key_part_to_json(key_part)?); } } - let sql = format!( - "DELETE FROM {} WHERE {}", - self.table_name, - where_clauses.join(" AND ") - ); - Ok((sql, params)) }