Kans/pebble default engine

cmd/baton/sanitize.go

@@ -111,7 +111,9 @@ func runSanitize(cmd *cobra.Command, args []string) error {
 
 	// Default the output engine to the source's so `sanitize` round-trips
 	// the engine unless the operator asks otherwise. An empty source engine
-	// (virtual/unknown store) falls back to the SQLite default.
+	// (virtual/unknown store) is pinned to SQLite explicitly — this
+	// predates the Pebble default flip and is no longer "the default",
+	// but the behavior is deliberately unchanged.
 	dstEngine := c1zstore.Engine(outEngineRaw)
 	if outEngineRaw == "" {
 		dstEngine = c1zstore.Engine(src.Metadata().Engine)

docs/rfcs/0001-pebble-storage-engine.md

@@ -6,6 +6,16 @@
 - **Prior art:** `morgabra/pebble` branch (commit `ae4b9693`, Aug 2025), `.kiro/specs/pebble-storage-engine/`
 - **Related PRs on main:** #591 (parallel sync), #666 (expandable at SQL), #773 (zstd pool), #769/#768 (skip cleanup), #735 (size verification removal)
 
+> **Amendment (Aug 2026):** The default flipped. This RFC was written when
+> SQLite was the default engine and Pebble was opt-in; the SDK default is now
+> Pebble for NEWLY CREATED c1z files (`kans/pebble-default-engine`). Existing
+> files still dispatch on their on-disk magic byte, and only an explicit
+> `WithEngine(EnginePebble)` converts a v1 file in place. Every statement
+> below that describes SQLite as the default engine — including the Summary,
+> §2's closing rationale, §3 Goals 2–3, and §5.3/§5.4 (including §5.4's
+> "writing v1 also stays the default") — reflects the original rollout
+> posture and is retained for historical context.
+
 ## 1. Summary
 
 Introduce a storage-engine abstraction in `pkg/dotc1z/` so that the same sync, compaction, and read/write code paths can run against either the current SQLite-in-a-`.c1z` backend or a new Pebble-backed backend. Connectors continue to default to SQLite (unchanged binary footprint, no cgo regressions, full on-disk compatibility). Backend infra (where we own both read and write sides and want more throughput) can opt into the Pebble engine via explicit configuration.

pkg/dotc1z/c1file.go

@@ -348,7 +348,9 @@ type c1zOptions struct {
 	disableGrantDigestIndex bool
 
 	// engine is the storage engine to use for newly created files.
-	// Reads dispatch on magic byte regardless. Default EngineSQLite.
+	// Reads dispatch on magic byte regardless. NewStore defaults an
+	// unset engine to EnginePebble; NewC1ZFile (SQLite-only) normalizes
+	// unset to EngineSQLite.
 	engine c1zstore.Engine
 
 	// payloadEncoding controls the v3 envelope payload framing. Only
@@ -433,8 +435,10 @@ func WithSyncLimit(limit int) C1ZOption {
 }
 
 // WithEngine selects the storage engine for newly created .c1z files.
-// Default is EngineSQLite (v1 format). EnginePebble enables the v3
-// engine.
+// Under NewStore the default is EnginePebble (v3 format); EngineSQLite
+// selects the legacy v1 engine. NewC1ZFile does not share that default:
+// it is the SQLite-only constructor, treats an unset engine as
+// EngineSQLite, and rejects a writable EnginePebble request.
 //
 // Reading existing files dispatches on the file's magic byte and is
 // independent of this option.

pkg/dotc1z/c1file_test.go

@@ -686,13 +686,13 @@ func TestC1ZCachedViewSyncRunInvalidation(t *testing.T) {
 	require.NoError(t, err)
 }
 
-// TestEngineDefaultsToSQLite proves that omitting WithEngine yields
-// c1zstore.EngineSQLite (the documented default), not the empty zero value.
-func TestEngineDefaultsToSQLite(t *testing.T) {
+// TestEngineDefaultsToPebble proves that omitting WithEngine yields
+// c1zstore.EnginePebble (the documented default), not the empty zero value.
+func TestEngineDefaultsToPebble(t *testing.T) {
 	dir := t.TempDir()
 	f, err := NewStore(context.Background(), filepath.Join(dir, "default.c1z"))
 	require.NoError(t, err, "NewStore")
 	defer f.Close(context.Background())
 	engine := f.Metadata().Engine
-	require.Equal(t, string(c1zstore.EngineSQLite), engine)
+	require.Equal(t, string(c1zstore.EnginePebble), engine)
 }

pkg/dotc1z/c1zstore/engine.go

@@ -13,12 +13,15 @@ import (
 type Engine string
 
 const (
-	// EngineSQLite is the default engine: the v1 .c1z format backed by
-	// a zstd-compressed SQLite database. Connectors use this; backend
-	// infra can opt out.
+	// EngineSQLite is the legacy v1 engine: the v1 .c1z format backed by
+	// a zstd-compressed SQLite database. Callers opt into it via
+	// WithEngine; the NewStore default is EnginePebble. (The SQLite-only
+	// NewC1ZFile constructor is the exception: it treats an unset engine
+	// as EngineSQLite.)
 	EngineSQLite Engine = "sqlite"
 
-	// EnginePebble is the v3 engine: a Pebble LSM wrapped in the v3
+	// EnginePebble is the v3 engine and the NewStore default when
+	// callers do not specify one: a Pebble LSM wrapped in the v3
 	// envelope. This is the in-process identity AND the value callers
 	// select with (the --storage-engine flag and the gRPC sync-task
 	// field both pass "pebble"); it must stay "pebble" for those

pkg/dotc1z/convert_open_test.go

@@ -100,7 +100,12 @@ func TestNewC1ZFileDoesNotConvertNewSQLiteFile(t *testing.T) {
 	require.Equal(t, string(c1zstore.EngineSQLite), f.Metadata().Engine)
 }
 
-func TestNewC1ZFileDoesNotConvertExistingSQLiteWhenEngineSQLite(t *testing.T) {
+// TestEnginelessOpenOfExistingV1DoesNotConvert proves the EnginePebble
+// default never rewrites existing v1 files: engine-less opens dispatch
+// on the magic byte and stay SQLite, whether writable or read-only.
+// Only an explicit WithEngine(EnginePebble) converts (see
+// TestNewStoreConvertsExistingSQLiteToPebble).
+func TestEnginelessOpenOfExistingV1DoesNotConvert(t *testing.T) {
 	ctx := context.Background()
 
 	dir := t.TempDir()
@@ -111,8 +116,34 @@ func TestNewC1ZFileDoesNotConvertExistingSQLiteWhenEngineSQLite(t *testing.T) {
 	require.NoError(t, seedFinishedSQLiteSync(ctx, t, src))
 	require.NoError(t, src.Close(ctx))
 
+	// Writable engine-less open: no conversion.
 	f, err := dotc1z.NewStore(ctx, c1zPath, dotc1z.WithTmpDir(dir))
 	require.NoError(t, err)
+	require.Equal(t, string(c1zstore.EngineSQLite), f.Metadata().Engine)
+	require.NoError(t, f.Close(ctx))
+	require.Equal(t, dotc1z.C1ZFormatV1, mustReadHeaderFormat(t, c1zPath))
+
+	// Read-only engine-less open: no conversion.
+	ro, err := dotc1z.NewStore(ctx, c1zPath, dotc1z.WithTmpDir(dir), dotc1z.WithReadOnly(true))
+	require.NoError(t, err)
+	require.Equal(t, string(c1zstore.EngineSQLite), ro.Metadata().Engine)
+	require.NoError(t, ro.Close(ctx))
+	require.Equal(t, dotc1z.C1ZFormatV1, mustReadHeaderFormat(t, c1zPath))
+}
+
+func TestNewC1ZFileDoesNotConvertExistingSQLiteWhenEngineSQLite(t *testing.T) {
+	ctx := context.Background()
+
+	dir := t.TempDir()
+	c1zPath := filepath.Join(dir, "source.c1z")
+
+	src, err := dotc1z.NewStore(ctx, c1zPath, dotc1z.WithTmpDir(dir), dotc1z.WithEngine(c1zstore.EngineSQLite))
+	require.NoError(t, err)
+	require.NoError(t, seedFinishedSQLiteSync(ctx, t, src))
+	require.NoError(t, src.Close(ctx))
+
+	f, err := dotc1z.NewStore(ctx, c1zPath, dotc1z.WithTmpDir(dir), dotc1z.WithEngine(c1zstore.EngineSQLite))
+	require.NoError(t, err)
 	defer func() { require.NoError(t, f.Close(ctx)) }()
 
 	require.Equal(t, dotc1z.C1ZFormatV1, mustReadHeaderFormat(t, c1zPath))

pkg/dotc1z/engine_registry.go

@@ -203,8 +203,11 @@ func storeOptionsFromC1ZOptions(options *c1zOptions) StoreOptions {
 		MaxDecodedPayloadBytes:  maxDecodedPayloadBytes,
 		MaxDecoderMemoryBytes:   maxDecoderMemoryBytes,
 	}
+	// Defensive only: NewStore, the sole caller, overwrites Engine with
+	// the selected driver's engine on the next line. Real default
+	// selection lives in selectStoreDriver.
 	if out.Engine == "" {
-		out.Engine = c1zstore.EngineSQLite
+		out.Engine = c1zstore.EnginePebble
 	}
 	out.Pragmas = make([]StorePragma, 0, len(options.pragmas))
 	for _, p := range options.pragmas {
@@ -218,14 +221,19 @@ func storeOptionsFromC1ZOptions(options *c1zOptions) StoreOptions {
 // Dispatch policy (in order):
 //
 //  1. If the file doesn't exist or is empty, honor the caller's
-//     `WithEngine(...)` choice (defaulting to EngineSQLite when
+//     `WithEngine(...)` choice (defaulting to EnginePebble when
 //     unset). The about-to-be-written file gets the requested format.
 //  2. If the file exists with content, dispatch by the on-disk magic
 //     byte — v1 → SQLite, v3 → whatever engine name the manifest
 //     records. The caller's `WithEngine` choice is overridden in this
 //     case because we can't re-encode an existing file at open time;
 //     the on-disk format is authoritative. This preserves the
 //     read-any-format semantics that pre-dates the engine option.
+//     Exception: an EXPLICIT WithEngine(EnginePebble) on a writable v1
+//     file converts it to Pebble in place. The EnginePebble default
+//     never triggers that conversion — an engine-less open of an
+//     existing v1 file stays SQLite, so read-intent callers (diff,
+//     stats, provisioning) don't rewrite files as a side effect.
 //
 // When the caller's WithEngine disagrees with the on-disk format we
 // log a warning so the divergence is observable. Callers that want
@@ -235,7 +243,7 @@ func selectStoreDriver(ctx context.Context, outputFilePath string, options *c1zO
 	l := ctxzap.Extract(ctx)
 	requested := options.engine
 	if requested == "" {
-		requested = c1zstore.EngineSQLite
+		requested = c1zstore.EnginePebble
 	}
 
 	stat, err := os.Stat(outputFilePath) // #nosec G703 -- c1z path is caller-controlled by API design.
@@ -267,7 +275,9 @@ func selectStoreDriver(ctx context.Context, outputFilePath string, options *c1zO
 	switch format {
 	case C1ZFormatV1:
 		// Maybe error if the file is read-only?
-		if requested == c1zstore.EnginePebble && !options.readOnly {
+		// Only an explicit pebble request converts; the engine default
+		// (options.engine == "") must not rewrite existing v1 files.
+		if options.engine == c1zstore.EnginePebble && !options.readOnly {
 			// Close our header-read handle before converting: the conversion
 			// renames a temp file over outputFilePath, which fails on Windows
 			// if any handle to the destination is still open. Nil out f so
@@ -277,11 +287,11 @@ func selectStoreDriver(ctx context.Context, outputFilePath string, options *c1zO
 			if closeErr != nil {
 				return nil, closeErr
 			}
-			l.Debug("converting existing v1 c1z to pebble", zap.String("output_file_path", outputFilePath))
+			l.Info("converting existing v1 c1z to pebble", zap.String("output_file_path", outputFilePath))
 			if err := convertExistingV1C1ZFile(ctx, outputFilePath, pebbleOpenOptionsFromC1Z(options)); err != nil {
 				return nil, fmt.Errorf("select-store-driver: convert existing v1 c1z to pebble: %w", err)
 			}
-			l.Debug("converted existing v1 c1z to pebble", zap.String("output_file_path", outputFilePath))
+			l.Info("converted existing v1 c1z to pebble", zap.String("output_file_path", outputFilePath))
 			return requireEngineDriver(c1zstore.EnginePebble)
 		}
 		fileEngine = c1zstore.EngineSQLite

pkg/dotc1z/engine_registry_test.go

@@ -64,14 +64,39 @@ func TestRegisterEngineRejectsDuplicateEngine(t *testing.T) {
 	require.Error(t, err, "RegisterEngine duplicate returned nil error")
 }
 
-func TestNewStoreDefaultsToSQLiteDriver(t *testing.T) {
+func TestNewStoreDefaultsToPebbleDriver(t *testing.T) {
 	ctx := context.Background()
 	path := filepath.Join(t.TempDir(), "default.c1z")
 	store, err := NewStore(ctx, path)
 	require.NoError(t, err)
-	defer func() { _ = store.Close(ctx) }()
+	// The happy path closes explicitly below (the close must succeed for
+	// the on-disk header check); this guard keeps a mid-test require
+	// failure from leaking the store for the rest of the test binary.
+	storeClosed := false
+	defer func() {
+		if !storeClosed {
+			_ = store.Close(ctx)
+		}
+	}()
 	_, ok := store.(*C1File)
-	require.True(t, ok, "NewStore default type = %T, want *C1File", store)
+	require.False(t, ok, "NewStore default type = %T, want the pebble store, not *C1File", store)
+	require.Equal(t, string(c1zstore.EnginePebble), store.Metadata().Engine)
+
+	// Seal a sync and close so the artifact lands on disk, then prove
+	// the on-disk format is v3 by magic byte, not just in-process
+	// metadata.
+	_, err = store.StartNewSync(ctx, connectorstore.SyncTypeFull, "")
+	require.NoError(t, err)
+	require.NoError(t, store.EndSync(ctx))
+	require.NoError(t, store.Close(ctx))
+	storeClosed = true
+
+	f, err := os.Open(path)
+	require.NoError(t, err)
+	defer f.Close()
+	format, err := ReadHeaderFormat(f)
+	require.NoError(t, err)
+	require.Equal(t, C1ZFormatV3, format, "default-engine artifact must be a v3 c1z on disk")
 }
 
 func TestNewStoreRequiresRegisteredEngineForNewFile(t *testing.T) {

pkg/field/defaults.go

@@ -355,10 +355,10 @@ var (
 		}))
 
 	// StorageEngineField selects the dotc1z storage engine for sync tasks.
-	// Empty uses the baton-sdk default (sqlite for new files).
+	// Empty uses the baton-sdk default (pebble for new files).
 	StorageEngineField = StringField("storage-engine",
 		WithDescription("The storage engine to use when opening the sync c1z file: sqlite or pebble. "+
-			"Leave unset to use the baton-sdk default."),
+			"Defaults to pebble when unset."),
 		WithPersistent(true),
 		WithExportTarget(ExportTargetNone),
 		WithString(func(r *StringRuler) {

pkg/sync/full_sync_bench_test.go

@@ -16,8 +16,10 @@ import (
 // BenchmarkFullSync_BatonDemoShape drives pkg/sync.NewSyncer
 // end-to-end against a mockConnector seeded at baton-demo scale —
 // the same data dimensions the public baton-demo connector emits —
-// through both the SQLite engine (WithC1ZPath) and the Pebble
-// engine (WithConnectorStore against a Pebble-backed c1zstore.Store).
+// through both the SQLite engine (WithC1ZPath + WithStorageEngine)
+// and the Pebble engine (WithConnectorStore against a Pebble-backed
+// c1zstore.Store). WithC1ZPath alone follows the NewStore default,
+// which is Pebble, so the sqlite arm must pin EngineSQLite.
 //
 // This is the FULL Sync() pipeline, not just the writer surface:
 // ResourceTypes → Resources → Entitlements → Grants iteration
@@ -138,7 +140,7 @@ func runOneFullSync(b *testing.B, engine string, nUsers, nGroups, membershipsPer
 	var opts []SyncOpt
 	switch engine {
 	case "sqlite":
-		opts = []SyncOpt{WithC1ZPath(c1zPath), WithTmpDir(tmpDir)}
+		opts = []SyncOpt{WithC1ZPath(c1zPath), WithTmpDir(tmpDir), WithStorageEngine(c1zstore.EngineSQLite)}
 	case "pebble":
 		store, err := dotc1z.NewStore(ctx, c1zPath,
 			dotc1z.WithEngine(c1zstore.EnginePebble),

pkg/sync/ingest_filter_test.go

@@ -365,7 +365,7 @@ func TestFreshIngestFilterExternalMatchResolvesUnknownPrincipalType(t *testing.T
 		require.NoError(t, internalSyncer.Sync(ctx))
 		require.NoError(t, internalSyncer.Close(ctx))
 
-		store, err := dotc1z.NewC1ZFile(ctx, internalC1zPath)
+		store, err := dotc1z.NewStore(ctx, internalC1zPath, dotc1z.WithReadOnly(true))
 		require.NoError(t, err)
 		defer func() { require.NoError(t, store.Close(ctx)) }()
 
@@ -436,7 +436,7 @@ func TestFreshIngestFilterInsertResourceGrantsResourceInserts(t *testing.T) {
 	require.NoError(t, syncer.Sync(ctx))
 	require.NoError(t, syncer.Close(ctx))
 
-	store, err := dotc1z.NewC1ZFile(ctx, c1zPath)
+	store, err := dotc1z.NewStore(ctx, c1zPath, dotc1z.WithReadOnly(true))
 	require.NoError(t, err)
 	defer func() { require.NoError(t, store.Close(ctx)) }()
 
@@ -481,7 +481,7 @@ func TestInsertResourceGrantsRejectsReservedBatonIDResource(t *testing.T) {
 	require.ErrorContains(t, syncer.Sync(ctx), "SDK-reserved BatonID ownership annotation")
 	require.NoError(t, syncer.Close(ctx))
 
-	store, err := dotc1z.NewC1ZFile(ctx, c1zPath)
+	store, err := dotc1z.NewStore(ctx, c1zPath, dotc1z.WithReadOnly(true))
 	require.NoError(t, err)
 	defer func() { require.NoError(t, store.Close(ctx)) }()
 	drives, err := store.ListResources(ctx,
@@ -561,7 +561,7 @@ func TestFreshIngestFilterRunsBeforeExclusionGroupValidation(t *testing.T) {
 	require.NoError(t, syncer.Sync(ctx))
 	require.NoError(t, syncer.Close(ctx))
 
-	store, err := dotc1z.NewC1ZFile(ctx, c1zPath)
+	store, err := dotc1z.NewStore(ctx, c1zPath, dotc1z.WithReadOnly(true))
 	require.NoError(t, err)
 	defer func() { require.NoError(t, store.Close(ctx)) }()
 
@@ -797,7 +797,7 @@ func TestFreshIngestFilterHandlesMultiPageResourceTypes(t *testing.T) {
 	require.NoError(t, syncer.Sync(ctx))
 	require.NoError(t, syncer.Close(ctx))
 
-	store, err := dotc1z.NewC1ZFile(ctx, c1zPath)
+	store, err := dotc1z.NewStore(ctx, c1zPath, dotc1z.WithReadOnly(true))
 	require.NoError(t, err)
 	defer func() { require.NoError(t, store.Close(ctx)) }()