perf(v2.0): #6 in-place UPDATE for columnar/append-only storage - fixed-width updates no longer append new versions

Author: MPCoreDeveloper

src/SharpCoreDB/DataStructures/Table.CRUD.cs

@@ -1177,8 +1177,7 @@ byte[] SerializeFullRow()
                 {
                     var rowData = SerializeFullRow();
 
-                    // Columnar: Append new version (old ref becomes stale)
-                    // Get old position from primary key index
+                    // Get old position from primary key index.
                     long oldPosition = -1;
                     if (this.PrimaryKeyIndex >= 0)
                     {
@@ -1190,28 +1189,65 @@ byte[] SerializeFullRow()
                         }
                     }
 
-                    // Insert new version
-                    long newPosition = engine.Insert(Name, rowData);
-
-                    // Update indexes to point to new position
-                    if (this.PrimaryKeyIndex >= 0)
+                    // Issue #6: in-place UPDATE — overwrite the record in its existing slot when
+                    // the new record fits (fixed-width rows, or variable-width rows whose stored
+                    // length is unchanged). No new version is appended, the storage reference and
+                    // the PK index stay valid, and no stale version is left for compaction.
+                    if (oldPosition >= 0 && engine.TryUpdateInPlace(Name, oldPosition, rowData))
                     {
-                        var pkVal = row[this.Columns[this.PrimaryKeyIndex]]?.ToString() ?? string.Empty;
-                        this.Index.Insert(pkVal, newPosition);
-                    }
+                        // Position unchanged: move hash entries in place (values may have changed).
+                        foreach (var kvp in this.hashIndexes)
+                        {
+                            if (oldHashKeys != null && oldHashKeys.TryGetValue(kvp.Key, out var oldKey))
+                            {
+                                kvp.Value.Remove(oldKey, oldPosition);
+                            }
 
-                    // Update hash indexes (key-only removal of the old value)
-                    foreach (var kvp in this.hashIndexes)
-                    {
-                        if (oldPosition >= 0 && oldHashKeys != null && oldHashKeys.TryGetValue(kvp.Key, out var oldKey))
+                            kvp.Value.Add(row, oldPosition);
+                        }
+
+                        // Re-point the PK index only when the PK value itself changed.
+                        if (this.PrimaryKeyIndex >= 0)
                         {
-                            kvp.Value.Remove(oldKey, oldPosition); // Remove old ref
+                            var newPkVal = row[this.Columns[this.PrimaryKeyIndex]]?.ToString() ?? string.Empty;
+                            if (!string.Equals(newPkVal, oldPkValue, StringComparison.Ordinal))
+                            {
+                                if (!string.IsNullOrEmpty(oldPkValue))
+                                {
+                                    this.Index.Delete(oldPkValue);
+                                }
+
+                                if (!string.IsNullOrEmpty(newPkVal))
+                                {
+                                    this.Index.Insert(newPkVal, oldPosition);
+                                }
+                            }
                         }
-                        kvp.Value.Add(row, newPosition); // Add new ref
                     }
+                    else
+                    {
+                        // Columnar fallback: append new version (old ref becomes stale) + re-point indexes.
+                        long newPosition = engine.Insert(Name, rowData);
 
-                    // ✅ NEW: Track updates for compaction
-                    Interlocked.Increment(ref _updatedRowCount);
+                        if (this.PrimaryKeyIndex >= 0)
+                        {
+                            var pkVal = row[this.Columns[this.PrimaryKeyIndex]]?.ToString() ?? string.Empty;
+                            this.Index.Insert(pkVal, newPosition);
+                        }
+
+                        foreach (var kvp in this.hashIndexes)
+                        {
+                            if (oldPosition >= 0 && oldHashKeys != null && oldHashKeys.TryGetValue(kvp.Key, out var oldKey))
+                            {
+                                kvp.Value.Remove(oldKey, oldPosition); // Remove old ref
+                            }
+
+                            kvp.Value.Add(row, newPosition); // Add new ref
+                        }
+
+                        // ✅ Track updates for compaction (only the append path creates stale versions).
+                        Interlocked.Increment(ref _updatedRowCount);
+                    }
                 }
                 else // PageBased
                 {
@@ -1436,27 +1472,62 @@ internal void UpdateMultiple(List<(string where, Dictionary<string, object> upda
                                     oldPosition = searchResult.Value;
                             }
 
-                            long newPosition = engine.Insert(Name, rowData);
-
-                            if (this.PrimaryKeyIndex >= 0)
+                            // Issue #6: in-place UPDATE — overwrite the record in its existing slot
+                            // when the new record fits; the storage reference and PK index stay valid.
+                            if (oldPosition >= 0 && engine.TryUpdateInPlace(Name, oldPosition, rowData))
                             {
-                                var pkVal = row[this.Columns[this.PrimaryKeyIndex]]?.ToString() ?? string.Empty;
-                                this.Index.Delete(pkVal);
-                                this.Index.Insert(pkVal, newPosition);
-                            }
+                                // Position unchanged: move hash entries in place (values may have changed).
+                                foreach (var hashIndex in this.hashIndexes)
+                                {
+                                    if (oldPosition >= 0 &&
+                                        oldHashValues is not null &&
+                                        oldHashValues.TryGetValue(hashIndex.Key, out var oldKey) &&
+                                        oldKey is not null)
+                                    {
+                                        hashIndex.Value.Remove(oldKey, oldPosition);
+                                    }
+
+                                    if (row.TryGetValue(hashIndex.Key, out var newKey) && newKey is not null)
+                                        hashIndex.Value.Add(newKey, oldPosition);
+                                }
 
-                            foreach (var hashIndex in this.hashIndexes)
+                                // Re-point the PK index only when the PK value itself changed.
+                                if (this.PrimaryKeyIndex >= 0)
+                                {
+                                    var newPkVal = row[this.Columns[this.PrimaryKeyIndex]]?.ToString() ?? string.Empty;
+                                    if (!string.Equals(newPkVal, oldPkValue?.ToString(), StringComparison.Ordinal))
+                                    {
+                                        if (!string.IsNullOrEmpty(oldPkValue?.ToString()))
+                                            this.Index.Delete(oldPkValue!.ToString()!);
+                                        if (!string.IsNullOrEmpty(newPkVal))
+                                            this.Index.Insert(newPkVal, oldPosition);
+                                    }
+                                }
+                            }
+                            else
                             {
-                                if (oldPosition >= 0 &&
-                                    oldHashValues is not null &&
-                                    oldHashValues.TryGetValue(hashIndex.Key, out var oldKey) &&
-                                    oldKey is not null)
+                                long newPosition = engine.Insert(Name, rowData);
+
+                                if (this.PrimaryKeyIndex >= 0)
                                 {
-                                    hashIndex.Value.Remove(oldKey, oldPosition);
+                                    var pkVal = row[this.Columns[this.PrimaryKeyIndex]]?.ToString() ?? string.Empty;
+                                    this.Index.Delete(pkVal);
+                                    this.Index.Insert(pkVal, newPosition);
                                 }
 
-                                if (row.TryGetValue(hashIndex.Key, out var newKey) && newKey is not null)
-                                    hashIndex.Value.Add(newKey, newPosition);
+                                foreach (var hashIndex in this.hashIndexes)
+                                {
+                                    if (oldPosition >= 0 &&
+                                        oldHashValues is not null &&
+                                        oldHashValues.TryGetValue(hashIndex.Key, out var oldKey) &&
+                                        oldKey is not null)
+                                    {
+                                        hashIndex.Value.Remove(oldKey, oldPosition);
+                                    }
+
+                                    if (row.TryGetValue(hashIndex.Key, out var newKey) && newKey is not null)
+                                        hashIndex.Value.Add(newKey, newPosition);
+                                }
                             }
 
                             updatedInBatch++;

src/SharpCoreDB/Interfaces/IStorage.cs

@@ -95,6 +95,15 @@ public interface IStorage
     /// <returns>The offset where the data was appended.</returns>
     long AppendBytes(string path, byte[] data);
 
+    /// <summary>
+    /// Overwrites a length-prefixed record in place at <paramref name="offset"/> (in-place UPDATE).
+    /// Returns true only when the new (encrypted) record fits the existing slot — i.e. the stored
+    /// length is unchanged, so every following record stays at a valid offset. When the lengths
+    /// differ the caller must fall back to <see cref="AppendBytes"/>. Not available inside a
+    /// transaction (buffered appends + rollback are append-only by design).
+    /// </summary>
+    bool OverwriteRecordAt(string path, long offset, byte[] data);
+
     /// <summary>
     /// Appends multiple binary data blocks to a file in a single batch operation (used for batch inserts).
     /// </summary>

src/SharpCoreDB/Interfaces/IStorageEngine.cs

@@ -46,6 +46,15 @@ public interface IStorageEngine : IDisposable
     /// </returns>
     long Update(string tableName, long storageReference, byte[] newData);
 
+    /// <summary>
+    /// Attempts to overwrite a record in place at the given storage reference — no relocation,
+    /// no new version, so the reference and every index entry stay valid. Returns true when the
+    /// write succeeded in place; false when the new record does not fit the existing slot (or the
+    /// engine does not support in-place updates), in which case callers must fall back to
+    /// <see cref="Update"/>.
+    /// </summary>
+    bool TryUpdateInPlace(string tableName, long storageReference, byte[] newData);
+
     /// <summary>
     /// Deletes a record at the specified storage reference.
     /// </summary>

src/SharpCoreDB/Services/Storage.Append.cs

@@ -329,6 +329,79 @@ public long AppendBytes(string path, byte[] data)
 
         return position;
     }
+    /// <summary>
+    /// Overwrites a length-prefixed record in place at <paramref name="offset"/> (in-place UPDATE).
+    /// Returns true only when the new (encrypted) record fits the existing slot — i.e. the stored
+    /// length is unchanged, so every following record stays at a valid offset. When the lengths
+    /// differ the caller must fall back to <see cref="AppendBytes"/>. Not available inside a
+    /// transaction (buffered appends + rollback are append-only by design).
+    /// </summary>
+    /// <param name="path">The table data file path.</param>
+    /// <param name="offset">The physical file offset of the record's 4-byte length prefix.</param>
+    /// <param name="data">The plaintext record data to write.</param>
+    /// <returns>True when the record was overwritten in place; false when it did not fit.</returns>
+    [MethodImpl(MethodImplOptions.AggressiveOptimization)]
+    public bool OverwriteRecordAt(string path, long offset, byte[] data)
+    {
+        ArgumentNullException.ThrowIfNull(data);
+
+        // In-place overwrites of already-flushed records cannot be buffered/rolled back with the
+        // append-only transaction machinery — fall back to append semantics in a transaction.
+        if (IsInTransaction)
+        {
+            return false;
+        }
+
+        bool encryptWrites = ShouldEncryptWrites(path);
+        byte[] record = EncryptRecord(data, encryptWrites);
+        int recordLength = record.Length;
+
+        try
+        {
+            using var fs = new FileStream(path, FileMode.Open, FileAccess.ReadWrite, FileShare.Read, 4096, FileOptions.WriteThrough);
+            if (fs.Length < offset + 4)
+            {
+                return false;
+            }
+
+            // Read the existing record's length prefix at the offset (ciphertext length for
+            // encrypted files, plaintext length otherwise — identical to AppendBytes).
+            fs.Position = offset;
+            Span<byte> lengthBuffer = stackalloc byte[4];
+            if (fs.Read(lengthBuffer) != 4)
+            {
+                return false;
+            }
+
+            int existingLength = BinaryPrimitives.ReadInt32LittleEndian(lengthBuffer);
+            if (existingLength != recordLength)
+            {
+                return false;
+            }
+
+            // Overwrite length prefix + payload in place; the file length is unchanged so all
+            // following records keep their offsets.
+            fs.Position = offset;
+            BinaryPrimitives.WriteInt32LittleEndian(lengthBuffer, recordLength);
+            fs.Write(lengthBuffer);
+            fs.Write(record.AsSpan());
+        }
+        catch (IOException)
+        {
+            return false;
+        }
+
+        // Invalidate app-level page cache (mirrors AppendBytes).
+        if (this.pageCache != null)
+        {
+            int pageId = ComputePageId(path, offset);
+            this.pageCache.EvictPage(pageId);
+        }
+
+        return true;
+    }
+
+
 
     /// <inheritdoc />
     [MethodImpl(MethodImplOptions.AggressiveOptimization)]

src/SharpCoreDB/Storage/Engines/AppendOnlyEngine.cs

@@ -112,6 +112,27 @@ public long Update(string tableName, long storageReference, byte[] newData)
 
         return newReference;
     }
+    /// <inheritdoc />
+    [MethodImpl(MethodImplOptions.AggressiveOptimization)]
+    public bool TryUpdateInPlace(string tableName, long storageReference, byte[] newData)
+    {
+        ArgumentNullException.ThrowIfNull(newData);
+
+        // Overwrites [length][data] at the existing offset when the new record fits the slot
+        // (same stored length) — no new version, so the reference and all index entries stay valid.
+        var filePath = GetTableFilePath(tableName);
+        bool overwritten = storage.OverwriteRecordAt(filePath, storageReference, newData);
+
+        if (overwritten)
+        {
+            Interlocked.Increment(ref totalUpdates);
+            Interlocked.Add(ref bytesWritten, newData.Length);
+        }
+
+        return overwritten;
+    }
+
+
 
     /// <inheritdoc />
     [MethodImpl(MethodImplOptions.AggressiveInlining)]

src/SharpCoreDB/Storage/Engines/PageBasedEngine.cs

@@ -174,6 +174,15 @@ public long Update(string tableName, long storageReference, byte[] newData)
             : EncodeStorageReference(newPage.Value, newRecordId.SlotIndex);
     }
 
+    /// <inheritdoc />
+    [MethodImpl(MethodImplOptions.AggressiveInlining)]
+    public bool TryUpdateInPlace(string tableName, long storageReference, byte[] newData)
+    {
+        // PageBasedEngine.Update already keeps the storage reference for in-place and
+        // within-page updates; only a cross-page relocation changes the reference.
+        return Update(tableName, storageReference, newData) == storageReference;
+    }
+
     /// <inheritdoc />
     [MethodImpl(MethodImplOptions.AggressiveInlining)]
     public void Delete(string tableName, long storageReference)

src/SharpCoreDB/Storage/Scdb/PageBasedAdapter.cs

@@ -189,6 +189,15 @@ public long Update(string tableName, long storageReference, byte[] newData)
         }
     }
 
+    /// <inheritdoc/>
+    public bool TryUpdateInPlace(string tableName, long storageReference, byte[] newData)
+    {
+        // The adapter updates records within a page; the slot pointer moves but the
+        // storage reference stays valid (relocation only occurs across pages for a record
+        // that grows past the page size).
+        return Update(tableName, storageReference, newData) == storageReference;
+    }
+
     /// <inheritdoc/>
     public void Delete(string tableName, long storageReference)
     {