diff --git a/cmd/baton/sanitize.go b/cmd/baton/sanitize.go index 7a2725091..79cf41196 100644 --- a/cmd/baton/sanitize.go +++ b/cmd/baton/sanitize.go @@ -111,7 +111,9 @@ func runSanitize(cmd *cobra.Command, args []string) error { // Default the output engine to the source's so `sanitize` round-trips // the engine unless the operator asks otherwise. An empty source engine - // (virtual/unknown store) falls back to the SQLite default. + // (virtual/unknown store) is pinned to SQLite explicitly — this + // predates the Pebble default flip and is no longer "the default", + // but the behavior is deliberately unchanged. dstEngine := c1zstore.Engine(outEngineRaw) if outEngineRaw == "" { dstEngine = c1zstore.Engine(src.Metadata().Engine) diff --git a/docs/rfcs/0001-pebble-storage-engine.md b/docs/rfcs/0001-pebble-storage-engine.md index e7bd619f4..1f8fec51f 100644 --- a/docs/rfcs/0001-pebble-storage-engine.md +++ b/docs/rfcs/0001-pebble-storage-engine.md @@ -6,6 +6,16 @@ - **Prior art:** `morgabra/pebble` branch (commit `ae4b9693`, Aug 2025), `.kiro/specs/pebble-storage-engine/` - **Related PRs on main:** #591 (parallel sync), #666 (expandable at SQL), #773 (zstd pool), #769/#768 (skip cleanup), #735 (size verification removal) +> **Amendment (Aug 2026):** The default flipped. This RFC was written when +> SQLite was the default engine and Pebble was opt-in; the SDK default is now +> Pebble for NEWLY CREATED c1z files (`kans/pebble-default-engine`). Existing +> files still dispatch on their on-disk magic byte, and only an explicit +> `WithEngine(EnginePebble)` converts a v1 file in place. Every statement +> below that describes SQLite as the default engine — including the Summary, +> §2's closing rationale, §3 Goals 2–3, and §5.3/§5.4 (including §5.4's +> "writing v1 also stays the default") — reflects the original rollout +> posture and is retained for historical context. + ## 1. Summary Introduce a storage-engine abstraction in `pkg/dotc1z/` so that the same sync, compaction, and read/write code paths can run against either the current SQLite-in-a-`.c1z` backend or a new Pebble-backed backend. Connectors continue to default to SQLite (unchanged binary footprint, no cgo regressions, full on-disk compatibility). Backend infra (where we own both read and write sides and want more throughput) can opt into the Pebble engine via explicit configuration. diff --git a/pb/c1/connectorapi/baton/v1/baton.pb.go b/pb/c1/connectorapi/baton/v1/baton.pb.go index fe7c7788b..cb17330f3 100644 --- a/pb/c1/connectorapi/baton/v1/baton.pb.go +++ b/pb/c1/connectorapi/baton/v1/baton.pb.go @@ -2935,7 +2935,7 @@ type Task_SyncFullTask struct { SyncResourceTypeIds []string `protobuf:"bytes,5,rep,name=sync_resource_type_ids,json=syncResourceTypeIds,proto3" json:"sync_resource_type_ids,omitempty"` // If true, skip syncing grants. Resources and entitlements will still be synced. SkipGrants bool `protobuf:"varint,6,opt,name=skip_grants,json=skipGrants,proto3" json:"skip_grants,omitempty"` - // Storage engine to use for the sync. If empty, the default engine will be used (currently SQLite). + // Storage engine to use for the sync. If empty, the default engine will be used (currently Pebble for new c1z files; existing files keep their on-disk format). StorageEngine string `protobuf:"bytes,7,opt,name=storage_engine,json=storageEngine,proto3" json:"storage_engine,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache @@ -3057,7 +3057,7 @@ type Task_SyncFullTask_builder struct { SyncResourceTypeIds []string // If true, skip syncing grants. Resources and entitlements will still be synced. SkipGrants bool - // Storage engine to use for the sync. If empty, the default engine will be used (currently SQLite). + // Storage engine to use for the sync. If empty, the default engine will be used (currently Pebble for new c1z files; existing files keep their on-disk format). StorageEngine string } diff --git a/pb/c1/connectorapi/baton/v1/baton_protoopaque.pb.go b/pb/c1/connectorapi/baton/v1/baton_protoopaque.pb.go index 833a47a42..aa86aa5b1 100644 --- a/pb/c1/connectorapi/baton/v1/baton_protoopaque.pb.go +++ b/pb/c1/connectorapi/baton/v1/baton_protoopaque.pb.go @@ -3022,7 +3022,7 @@ type Task_SyncFullTask_builder struct { SyncResourceTypeIds []string // If true, skip syncing grants. Resources and entitlements will still be synced. SkipGrants bool - // Storage engine to use for the sync. If empty, the default engine will be used (currently SQLite). + // Storage engine to use for the sync. If empty, the default engine will be used (currently Pebble for new c1z files; existing files keep their on-disk format). StorageEngine string } diff --git a/pkg/dotc1z/c1file.go b/pkg/dotc1z/c1file.go index 70240e458..9ad1d8d07 100644 --- a/pkg/dotc1z/c1file.go +++ b/pkg/dotc1z/c1file.go @@ -348,7 +348,9 @@ type c1zOptions struct { disableGrantDigestIndex bool // engine is the storage engine to use for newly created files. - // Reads dispatch on magic byte regardless. Default EngineSQLite. + // Reads dispatch on magic byte regardless. NewStore defaults an + // unset engine to EnginePebble; NewC1ZFile (SQLite-only) normalizes + // unset to EngineSQLite. engine c1zstore.Engine // payloadEncoding controls the v3 envelope payload framing. Only @@ -433,8 +435,10 @@ func WithSyncLimit(limit int) C1ZOption { } // WithEngine selects the storage engine for newly created .c1z files. -// Default is EngineSQLite (v1 format). EnginePebble enables the v3 -// engine. +// Under NewStore the default is EnginePebble (v3 format); EngineSQLite +// selects the legacy v1 engine. NewC1ZFile does not share that default: +// it is the SQLite-only constructor, treats an unset engine as +// EngineSQLite, and rejects a writable EnginePebble request. // // Reading existing files dispatches on the file's magic byte and is // independent of this option. diff --git a/pkg/dotc1z/c1file_test.go b/pkg/dotc1z/c1file_test.go index aef2a9254..5f719810d 100644 --- a/pkg/dotc1z/c1file_test.go +++ b/pkg/dotc1z/c1file_test.go @@ -686,13 +686,13 @@ func TestC1ZCachedViewSyncRunInvalidation(t *testing.T) { require.NoError(t, err) } -// TestEngineDefaultsToSQLite proves that omitting WithEngine yields -// c1zstore.EngineSQLite (the documented default), not the empty zero value. -func TestEngineDefaultsToSQLite(t *testing.T) { +// TestEngineDefaultsToPebble proves that omitting WithEngine yields +// c1zstore.EnginePebble (the documented default), not the empty zero value. +func TestEngineDefaultsToPebble(t *testing.T) { dir := t.TempDir() f, err := NewStore(context.Background(), filepath.Join(dir, "default.c1z")) require.NoError(t, err, "NewStore") defer f.Close(context.Background()) engine := f.Metadata().Engine - require.Equal(t, string(c1zstore.EngineSQLite), engine) + require.Equal(t, string(c1zstore.EnginePebble), engine) } diff --git a/pkg/dotc1z/c1zstore/engine.go b/pkg/dotc1z/c1zstore/engine.go index 833fa7cd4..9fb8a684e 100644 --- a/pkg/dotc1z/c1zstore/engine.go +++ b/pkg/dotc1z/c1zstore/engine.go @@ -13,12 +13,15 @@ import ( type Engine string const ( - // EngineSQLite is the default engine: the v1 .c1z format backed by - // a zstd-compressed SQLite database. Connectors use this; backend - // infra can opt out. + // EngineSQLite is the legacy v1 engine: the v1 .c1z format backed by + // a zstd-compressed SQLite database. Callers opt into it via + // WithEngine; the NewStore default is EnginePebble. (The SQLite-only + // NewC1ZFile constructor is the exception: it treats an unset engine + // as EngineSQLite.) EngineSQLite Engine = "sqlite" - // EnginePebble is the v3 engine: a Pebble LSM wrapped in the v3 + // EnginePebble is the v3 engine and the NewStore default when + // callers do not specify one: a Pebble LSM wrapped in the v3 // envelope. This is the in-process identity AND the value callers // select with (the --storage-engine flag and the gRPC sync-task // field both pass "pebble"); it must stay "pebble" for those diff --git a/pkg/dotc1z/convert_open_test.go b/pkg/dotc1z/convert_open_test.go index 311f27981..cc6337f59 100644 --- a/pkg/dotc1z/convert_open_test.go +++ b/pkg/dotc1z/convert_open_test.go @@ -100,7 +100,12 @@ func TestNewC1ZFileDoesNotConvertNewSQLiteFile(t *testing.T) { require.Equal(t, string(c1zstore.EngineSQLite), f.Metadata().Engine) } -func TestNewC1ZFileDoesNotConvertExistingSQLiteWhenEngineSQLite(t *testing.T) { +// TestEnginelessOpenOfExistingV1DoesNotConvert proves the EnginePebble +// default never rewrites existing v1 files: engine-less opens dispatch +// on the magic byte and stay SQLite, whether writable or read-only. +// Only an explicit WithEngine(EnginePebble) converts (see +// TestNewStoreConvertsExistingSQLiteToPebble). +func TestEnginelessOpenOfExistingV1DoesNotConvert(t *testing.T) { ctx := context.Background() dir := t.TempDir() @@ -111,8 +116,34 @@ func TestNewC1ZFileDoesNotConvertExistingSQLiteWhenEngineSQLite(t *testing.T) { require.NoError(t, seedFinishedSQLiteSync(ctx, t, src)) require.NoError(t, src.Close(ctx)) + // Writable engine-less open: no conversion. f, err := dotc1z.NewStore(ctx, c1zPath, dotc1z.WithTmpDir(dir)) require.NoError(t, err) + require.Equal(t, string(c1zstore.EngineSQLite), f.Metadata().Engine) + require.NoError(t, f.Close(ctx)) + require.Equal(t, dotc1z.C1ZFormatV1, mustReadHeaderFormat(t, c1zPath)) + + // Read-only engine-less open: no conversion. + ro, err := dotc1z.NewStore(ctx, c1zPath, dotc1z.WithTmpDir(dir), dotc1z.WithReadOnly(true)) + require.NoError(t, err) + require.Equal(t, string(c1zstore.EngineSQLite), ro.Metadata().Engine) + require.NoError(t, ro.Close(ctx)) + require.Equal(t, dotc1z.C1ZFormatV1, mustReadHeaderFormat(t, c1zPath)) +} + +func TestNewC1ZFileDoesNotConvertExistingSQLiteWhenEngineSQLite(t *testing.T) { + ctx := context.Background() + + dir := t.TempDir() + c1zPath := filepath.Join(dir, "source.c1z") + + src, err := dotc1z.NewStore(ctx, c1zPath, dotc1z.WithTmpDir(dir), dotc1z.WithEngine(c1zstore.EngineSQLite)) + require.NoError(t, err) + require.NoError(t, seedFinishedSQLiteSync(ctx, t, src)) + require.NoError(t, src.Close(ctx)) + + f, err := dotc1z.NewStore(ctx, c1zPath, dotc1z.WithTmpDir(dir), dotc1z.WithEngine(c1zstore.EngineSQLite)) + require.NoError(t, err) defer func() { require.NoError(t, f.Close(ctx)) }() require.Equal(t, dotc1z.C1ZFormatV1, mustReadHeaderFormat(t, c1zPath)) diff --git a/pkg/dotc1z/engine_registry.go b/pkg/dotc1z/engine_registry.go index db95f26ba..180176b92 100644 --- a/pkg/dotc1z/engine_registry.go +++ b/pkg/dotc1z/engine_registry.go @@ -203,8 +203,11 @@ func storeOptionsFromC1ZOptions(options *c1zOptions) StoreOptions { MaxDecodedPayloadBytes: maxDecodedPayloadBytes, MaxDecoderMemoryBytes: maxDecoderMemoryBytes, } + // Defensive only: NewStore, the sole caller, overwrites Engine with + // the selected driver's engine on the next line. Real default + // selection lives in selectStoreDriver. if out.Engine == "" { - out.Engine = c1zstore.EngineSQLite + out.Engine = c1zstore.EnginePebble } out.Pragmas = make([]StorePragma, 0, len(options.pragmas)) for _, p := range options.pragmas { @@ -218,7 +221,7 @@ func storeOptionsFromC1ZOptions(options *c1zOptions) StoreOptions { // Dispatch policy (in order): // // 1. If the file doesn't exist or is empty, honor the caller's -// `WithEngine(...)` choice (defaulting to EngineSQLite when +// `WithEngine(...)` choice (defaulting to EnginePebble when // unset). The about-to-be-written file gets the requested format. // 2. If the file exists with content, dispatch by the on-disk magic // byte — v1 → SQLite, v3 → whatever engine name the manifest @@ -226,6 +229,11 @@ func storeOptionsFromC1ZOptions(options *c1zOptions) StoreOptions { // case because we can't re-encode an existing file at open time; // the on-disk format is authoritative. This preserves the // read-any-format semantics that pre-dates the engine option. +// Exception: an EXPLICIT WithEngine(EnginePebble) on a writable v1 +// file converts it to Pebble in place. The EnginePebble default +// never triggers that conversion — an engine-less open of an +// existing v1 file stays SQLite, so read-intent callers (diff, +// stats, provisioning) don't rewrite files as a side effect. // // When the caller's WithEngine disagrees with the on-disk format we // log a warning so the divergence is observable. Callers that want @@ -235,7 +243,7 @@ func selectStoreDriver(ctx context.Context, outputFilePath string, options *c1zO l := ctxzap.Extract(ctx) requested := options.engine if requested == "" { - requested = c1zstore.EngineSQLite + requested = c1zstore.EnginePebble } stat, err := os.Stat(outputFilePath) // #nosec G703 -- c1z path is caller-controlled by API design. @@ -267,7 +275,9 @@ func selectStoreDriver(ctx context.Context, outputFilePath string, options *c1zO switch format { case C1ZFormatV1: // Maybe error if the file is read-only? - if requested == c1zstore.EnginePebble && !options.readOnly { + // Only an explicit pebble request converts; the engine default + // (options.engine == "") must not rewrite existing v1 files. + if options.engine == c1zstore.EnginePebble && !options.readOnly { // Close our header-read handle before converting: the conversion // renames a temp file over outputFilePath, which fails on Windows // if any handle to the destination is still open. Nil out f so @@ -277,11 +287,11 @@ func selectStoreDriver(ctx context.Context, outputFilePath string, options *c1zO if closeErr != nil { return nil, closeErr } - l.Debug("converting existing v1 c1z to pebble", zap.String("output_file_path", outputFilePath)) + l.Info("converting existing v1 c1z to pebble", zap.String("output_file_path", outputFilePath)) if err := convertExistingV1C1ZFile(ctx, outputFilePath, pebbleOpenOptionsFromC1Z(options)); err != nil { return nil, fmt.Errorf("select-store-driver: convert existing v1 c1z to pebble: %w", err) } - l.Debug("converted existing v1 c1z to pebble", zap.String("output_file_path", outputFilePath)) + l.Info("converted existing v1 c1z to pebble", zap.String("output_file_path", outputFilePath)) return requireEngineDriver(c1zstore.EnginePebble) } fileEngine = c1zstore.EngineSQLite diff --git a/pkg/dotc1z/engine_registry_test.go b/pkg/dotc1z/engine_registry_test.go index 3470d3c2d..2e28611ed 100644 --- a/pkg/dotc1z/engine_registry_test.go +++ b/pkg/dotc1z/engine_registry_test.go @@ -64,14 +64,39 @@ func TestRegisterEngineRejectsDuplicateEngine(t *testing.T) { require.Error(t, err, "RegisterEngine duplicate returned nil error") } -func TestNewStoreDefaultsToSQLiteDriver(t *testing.T) { +func TestNewStoreDefaultsToPebbleDriver(t *testing.T) { ctx := context.Background() path := filepath.Join(t.TempDir(), "default.c1z") store, err := NewStore(ctx, path) require.NoError(t, err) - defer func() { _ = store.Close(ctx) }() + // The happy path closes explicitly below (the close must succeed for + // the on-disk header check); this guard keeps a mid-test require + // failure from leaking the store for the rest of the test binary. + storeClosed := false + defer func() { + if !storeClosed { + _ = store.Close(ctx) + } + }() _, ok := store.(*C1File) - require.True(t, ok, "NewStore default type = %T, want *C1File", store) + require.False(t, ok, "NewStore default type = %T, want the pebble store, not *C1File", store) + require.Equal(t, string(c1zstore.EnginePebble), store.Metadata().Engine) + + // Seal a sync and close so the artifact lands on disk, then prove + // the on-disk format is v3 by magic byte, not just in-process + // metadata. + _, err = store.StartNewSync(ctx, connectorstore.SyncTypeFull, "") + require.NoError(t, err) + require.NoError(t, store.EndSync(ctx)) + require.NoError(t, store.Close(ctx)) + storeClosed = true + + f, err := os.Open(path) + require.NoError(t, err) + defer f.Close() + format, err := ReadHeaderFormat(f) + require.NoError(t, err) + require.Equal(t, C1ZFormatV3, format, "default-engine artifact must be a v3 c1z on disk") } func TestNewStoreRequiresRegisteredEngineForNewFile(t *testing.T) { diff --git a/pkg/field/defaults.go b/pkg/field/defaults.go index 168570423..4dc0bd192 100644 --- a/pkg/field/defaults.go +++ b/pkg/field/defaults.go @@ -355,10 +355,10 @@ var ( })) // StorageEngineField selects the dotc1z storage engine for sync tasks. - // Empty uses the baton-sdk default (sqlite for new files). + // Empty uses the baton-sdk default (pebble for new files). StorageEngineField = StringField("storage-engine", WithDescription("The storage engine to use when opening the sync c1z file: sqlite or pebble. "+ - "Leave unset to use the baton-sdk default."), + "Defaults to pebble when unset."), WithPersistent(true), WithExportTarget(ExportTargetNone), WithString(func(r *StringRuler) { diff --git a/pkg/sync/full_sync_bench_test.go b/pkg/sync/full_sync_bench_test.go index 475a30f64..7581649a7 100644 --- a/pkg/sync/full_sync_bench_test.go +++ b/pkg/sync/full_sync_bench_test.go @@ -16,8 +16,10 @@ import ( // BenchmarkFullSync_BatonDemoShape drives pkg/sync.NewSyncer // end-to-end against a mockConnector seeded at baton-demo scale — // the same data dimensions the public baton-demo connector emits — -// through both the SQLite engine (WithC1ZPath) and the Pebble -// engine (WithConnectorStore against a Pebble-backed c1zstore.Store). +// through both the SQLite engine (WithC1ZPath + WithStorageEngine) +// and the Pebble engine (WithConnectorStore against a Pebble-backed +// c1zstore.Store). WithC1ZPath alone follows the NewStore default, +// which is Pebble, so the sqlite arm must pin EngineSQLite. // // This is the FULL Sync() pipeline, not just the writer surface: // ResourceTypes → Resources → Entitlements → Grants iteration @@ -138,7 +140,7 @@ func runOneFullSync(b *testing.B, engine string, nUsers, nGroups, membershipsPer var opts []SyncOpt switch engine { case "sqlite": - opts = []SyncOpt{WithC1ZPath(c1zPath), WithTmpDir(tmpDir)} + opts = []SyncOpt{WithC1ZPath(c1zPath), WithTmpDir(tmpDir), WithStorageEngine(c1zstore.EngineSQLite)} case "pebble": store, err := dotc1z.NewStore(ctx, c1zPath, dotc1z.WithEngine(c1zstore.EnginePebble), diff --git a/pkg/sync/ingest_filter_test.go b/pkg/sync/ingest_filter_test.go index 692696012..d11b8b99c 100644 --- a/pkg/sync/ingest_filter_test.go +++ b/pkg/sync/ingest_filter_test.go @@ -365,7 +365,7 @@ func TestFreshIngestFilterExternalMatchResolvesUnknownPrincipalType(t *testing.T require.NoError(t, internalSyncer.Sync(ctx)) require.NoError(t, internalSyncer.Close(ctx)) - store, err := dotc1z.NewC1ZFile(ctx, internalC1zPath) + store, err := dotc1z.NewStore(ctx, internalC1zPath, dotc1z.WithReadOnly(true)) require.NoError(t, err) defer func() { require.NoError(t, store.Close(ctx)) }() @@ -436,7 +436,7 @@ func TestFreshIngestFilterInsertResourceGrantsResourceInserts(t *testing.T) { require.NoError(t, syncer.Sync(ctx)) require.NoError(t, syncer.Close(ctx)) - store, err := dotc1z.NewC1ZFile(ctx, c1zPath) + store, err := dotc1z.NewStore(ctx, c1zPath, dotc1z.WithReadOnly(true)) require.NoError(t, err) defer func() { require.NoError(t, store.Close(ctx)) }() @@ -481,7 +481,7 @@ func TestInsertResourceGrantsRejectsReservedBatonIDResource(t *testing.T) { require.ErrorContains(t, syncer.Sync(ctx), "SDK-reserved BatonID ownership annotation") require.NoError(t, syncer.Close(ctx)) - store, err := dotc1z.NewC1ZFile(ctx, c1zPath) + store, err := dotc1z.NewStore(ctx, c1zPath, dotc1z.WithReadOnly(true)) require.NoError(t, err) defer func() { require.NoError(t, store.Close(ctx)) }() drives, err := store.ListResources(ctx, @@ -561,7 +561,7 @@ func TestFreshIngestFilterRunsBeforeExclusionGroupValidation(t *testing.T) { require.NoError(t, syncer.Sync(ctx)) require.NoError(t, syncer.Close(ctx)) - store, err := dotc1z.NewC1ZFile(ctx, c1zPath) + store, err := dotc1z.NewStore(ctx, c1zPath, dotc1z.WithReadOnly(true)) require.NoError(t, err) defer func() { require.NoError(t, store.Close(ctx)) }() @@ -797,7 +797,7 @@ func TestFreshIngestFilterHandlesMultiPageResourceTypes(t *testing.T) { require.NoError(t, syncer.Sync(ctx)) require.NoError(t, syncer.Close(ctx)) - store, err := dotc1z.NewC1ZFile(ctx, c1zPath) + store, err := dotc1z.NewStore(ctx, c1zPath, dotc1z.WithReadOnly(true)) require.NoError(t, err) defer func() { require.NoError(t, store.Close(ctx)) }() diff --git a/pkg/sync/syncer_cleanup_test.go b/pkg/sync/syncer_cleanup_test.go index 7291a6e29..6cb2b58d7 100644 --- a/pkg/sync/syncer_cleanup_test.go +++ b/pkg/sync/syncer_cleanup_test.go @@ -4,12 +4,14 @@ package sync import ( + "os" "path/filepath" "testing" "time" reader_v2 "github.com/conductorone/baton-sdk/pb/c1/reader/v2" "github.com/conductorone/baton-sdk/pkg/dotc1z" + "github.com/conductorone/baton-sdk/pkg/dotc1z/c1zstore" "github.com/conductorone/baton-sdk/pkg/dotc1z/c1ztest" "github.com/conductorone/baton-sdk/pkg/logging" "github.com/stretchr/testify/require" @@ -29,7 +31,11 @@ func TestCleanupContextDeadlineExceeded(t *testing.T) { testFilePath := filepath.Join(tmpDir, "test.c1z") - f, err := dotc1z.NewStore(ctx, testFilePath) + // Pinned to SQLite: the test relies on the 200ms run budget expiring + // during cleanup of ~98 old syncs, which is only reliably slow on the + // row-by-row SQLite delete path. Pebble drops syncs via cheap range + // deletes, the sync completes, and ErrSyncNotComplete never fires. + f, err := dotc1z.NewStore(ctx, testFilePath, dotc1z.WithEngine(c1zstore.EngineSQLite)) require.NoError(t, err) // Create and end a bunch of syncs. We should delete all but 2 of them in Cleanup(). @@ -86,3 +92,84 @@ func TestCleanupContextDeadlineExceeded(t *testing.T) { err = syncer.Close(ctx) require.NoError(t, err) } + +// TestCleanupPebbleSingleSyncLifecycle is the default-engine sibling of +// TestCleanupContextDeadlineExceeded (which is pinned to SQLite because +// its deadline assertion depends on SQLite's slow row-by-row deletes). +// On pebble — now the default — end-of-sync cleanup is a no-op by +// design: each StartNewSync replaces the prior sync in place +// (single-sync contract), so a backlog of old syncs can never +// accumulate and the run budget cannot expire during cleanup. The +// pebble analog of "old syncs dropped within budget" is therefore: +// the syncer run completes well inside a budget, the seeded sync was +// replaced rather than retained, exactly one sealed sync remains, and +// the artifact is v3 on disk. +func TestCleanupPebbleSingleSyncLifecycle(t *testing.T) { + ctx := t.Context() + tmpDir := t.TempDir() + + ctx, err := logging.Init( + ctx, + logging.WithLogFormat(logging.LogFormatConsole), + logging.WithLogLevel("debug"), + ) + require.NoError(t, err) + + testFilePath := filepath.Join(tmpDir, "test.c1z") + + // Engine-less NewStore: the default (pebble) path under test. + f, err := dotc1z.NewStore(ctx, testFilePath) + require.NoError(t, err) + // Fail-safe only: a mid-test require failure must not leak the open + // pebble store for the rest of the test binary. On the happy path + // syncer.Close closes the store and this is a no-op (Close is + // idempotent). + defer func() { _ = f.Close(ctx) }() + + // Seed synced data the same way the SQLite test does. Under the + // single-sync contract each iteration replaces the previous sync, so + // unlike SQLite this can never build up a cleanup backlog. + seededSyncID := "" + for range 3 { + seededSyncID, err = c1ztest.CreateTestSync(ctx, t, f, c1ztest.C1ZCounts{ + ResourceTypeCount: 10, + ResourceCount: 100, + UserCount: 100, + EntitlementCount: 10, + GrantCount: 250, + }) + require.NoError(t, err) + } + + // The budget is intentionally generous: with no-op cleanup the only + // thing that could burn it is the sync itself, and a tight bound + // would test machine speed, not engine behavior. What matters is + // that ErrSyncNotComplete — the SQLite test's expected outcome — + // cannot happen here. + syncer, err := NewSyncer(ctx, newMockConnector(), WithRunDuration(30*time.Second), WithConnectorStore(f)) + require.NoError(t, err) + err = syncer.Sync(ctx) + require.NoError(t, err, "pebble sync must complete within budget; cleanup is a no-op") + + // Exactly one sync remains, it is sealed, and it is the syncer's run, + // not the seeded one (replacement, the pebble analog of cleanup). + syncsResp, err := f.ListSyncs(ctx, reader_v2.SyncsReaderServiceListSyncsRequest_builder{ + PageSize: 100, + }.Build()) + require.NoError(t, err) + syncs := syncsResp.GetSyncs() + require.Len(t, syncs, 1, "single-sync contract: exactly one sync after the run") + require.NotNil(t, syncs[0].GetEndedAt(), "the remaining sync must be sealed") + require.NotEqual(t, seededSyncID, syncs[0].GetId(), "the seeded sync must have been replaced") + + err = syncer.Close(ctx) + require.NoError(t, err) + + // The default-engine artifact must be v3 on disk. + file, err := os.Open(testFilePath) + require.NoError(t, err) + defer file.Close() + format, err := dotc1z.ReadHeaderFormat(file) + require.NoError(t, err) + require.Equal(t, dotc1z.C1ZFormatV3, format, "default-engine artifact must be a v3 c1z on disk") +} diff --git a/pkg/sync/syncer_test.go b/pkg/sync/syncer_test.go index c9e8cc12e..d29ba89ea 100644 --- a/pkg/sync/syncer_test.go +++ b/pkg/sync/syncer_test.go @@ -119,8 +119,9 @@ func TestExpandGrants(t *testing.T) { require.NoError(t, err) // Validate that grants got expanded - store, err := dotc1z.NewC1ZFile(ctx, c1zpath) + store, err := dotc1z.NewStore(ctx, c1zpath, dotc1z.WithReadOnly(true)) require.NoError(t, err) + defer func() { _ = store.Close(ctx) }() // Yes it's wasteful to load all grants into memory, but this connector doesn't make a ton of grants. allGrants := make([]*v2.Grant, 0) @@ -317,8 +318,9 @@ func TestExpandGrantImmutable(t *testing.T) { err = syncer.Close(ctx) require.NoError(t, err) - store, err := dotc1z.NewC1ZFile(ctx, c1zpath) + store, err := dotc1z.NewStore(ctx, c1zpath, dotc1z.WithReadOnly(true)) require.NoError(t, err) + defer func() { _ = store.Close(ctx) }() allGrantsReq := &v2.GrantsServiceListGrantsRequest{} allGrants, err := store.ListGrants(ctx, allGrantsReq) @@ -422,8 +424,9 @@ func TestExpandGrantImmutableCycle(t *testing.T) { err = syncer.Close(ctx) require.NoError(t, err) - store, err := dotc1z.NewC1ZFile(ctx, c1zpath) + store, err := dotc1z.NewStore(ctx, c1zpath, dotc1z.WithReadOnly(true)) require.NoError(t, err) + defer func() { _ = store.Close(ctx) }() allGrantsReq := &v2.GrantsServiceListGrantsRequest{} allGrants, err := store.ListGrants(ctx, allGrantsReq) @@ -629,8 +632,9 @@ func TestExternalResourcePath(t *testing.T) { err = internalSyncer.Close(ctx) require.NoError(t, err) - store, err := dotc1z.NewC1ZFile(ctx, internalC1zpath) + store, err := dotc1z.NewStore(ctx, internalC1zpath, dotc1z.WithReadOnly(true)) require.NoError(t, err) + defer func() { _ = store.Close(ctx) }() resources, err := store.ListResources(ctx, v2.ResourcesServiceListResourcesRequest_builder{ ResourceTypeId: userResourceType.GetId(), @@ -712,8 +716,9 @@ func TestPartialSync(t *testing.T) { err = partialSyncer.Close(ctx) require.NoError(t, err) - store, err := dotc1z.NewC1ZFile(ctx, c1zPath) + store, err := dotc1z.NewStore(ctx, c1zPath, dotc1z.WithReadOnly(true)) require.NoError(t, err) + defer func() { _ = store.Close(ctx) }() resourcesResp, err := store.ListResources(ctx, v2.ResourcesServiceListResourcesRequest_builder{ ResourceTypeId: userResourceType.GetId(), @@ -778,8 +783,9 @@ func TestPartialSyncSkipEntitlementsAndGrants(t *testing.T) { err = syncer.Close(ctx) require.NoError(t, err) - store, err := dotc1z.NewC1ZFile(ctx, c1zPath) + store, err := dotc1z.NewStore(ctx, c1zPath, dotc1z.WithReadOnly(true)) require.NoError(t, err) + defer func() { _ = store.Close(ctx) }() resources, err := store.ListResources(ctx, v2.ResourcesServiceListResourcesRequest_builder{ ResourceTypeId: groupResourceType.GetId(), @@ -843,10 +849,15 @@ func TestPartialSyncUnimplemented(t *testing.T) { err = partialSyncer.Close(ctx) require.NoError(t, err) - store, err := dotc1z.NewC1ZFile(ctx, c1zPath) + store, err := dotc1z.NewStore(ctx, c1zPath, dotc1z.WithReadOnly(true)) require.NoError(t, err) + defer func() { _ = store.Close(ctx) }() - syncs, _, err := store.ListSyncRuns(ctx, "", 100) + lister, ok := store.(interface { + ListSyncRuns(context.Context, string, uint32) ([]*c1zstore.SyncRun, string, error) + }) + require.True(t, ok, "store must support ListSyncRuns") + syncs, _, err := lister.ListSyncRuns(ctx, "", 100) require.NoError(t, err) require.Equal(t, 1, len(syncs)) require.Equal(t, connectorstore.SyncTypePartial, syncs[0].Type) @@ -931,8 +942,9 @@ func TestExternalResourceMatchAll(t *testing.T) { require.NoError(t, err) // Verify grants were created for all external users - store, err := dotc1z.NewC1ZFile(ctx, internalC1zpath) + store, err := dotc1z.NewStore(ctx, internalC1zpath, dotc1z.WithReadOnly(true)) require.NoError(t, err) + defer func() { _ = store.Close(ctx) }() // Get grants for the internal group entitlement grants, err := store.ListGrantsForEntitlement(ctx, reader_v2.GrantsReaderServiceListGrantsForEntitlementRequest_builder{ @@ -1021,8 +1033,9 @@ func TestExternalResourceMatchID(t *testing.T) { require.NoError(t, err) // Verify grant was created for the matching external user - store, err := dotc1z.NewC1ZFile(ctx, internalC1zpath) + store, err := dotc1z.NewStore(ctx, internalC1zpath, dotc1z.WithReadOnly(true)) require.NoError(t, err) + defer func() { _ = store.Close(ctx) }() allGrants, err := store.ListGrants(ctx, &v2.GrantsServiceListGrantsRequest{}) require.NoError(t, err) @@ -1121,8 +1134,9 @@ func TestExternalResourceMatchIDWithExpandableRemapping(t *testing.T) { // The remapping should produce an entitlement ID using the matched external // group's resource, which is the same as the original in this case but was // generated via entitlement.NewEntitlementID(matchedPrincipal, slug). - store, err := dotc1z.NewC1ZFile(ctx, internalC1zpath) + store, err := dotc1z.NewStore(ctx, internalC1zpath, dotc1z.WithReadOnly(true)) require.NoError(t, err) + defer func() { _ = store.Close(ctx) }() allGrants, err := store.ListGrants(ctx, &v2.GrantsServiceListGrantsRequest{}) require.NoError(t, err) @@ -1230,8 +1244,9 @@ func TestExternalResourceEmailMatch(t *testing.T) { require.NoError(t, err) // Verify grant was created for the matching external user - store, err := dotc1z.NewC1ZFile(ctx, internalC1zpath) + store, err := dotc1z.NewStore(ctx, internalC1zpath, dotc1z.WithReadOnly(true)) require.NoError(t, err) + defer func() { _ = store.Close(ctx) }() // Get grants for the internal group entitlement grants, err := store.ListGrantsForEntitlement(ctx, reader_v2.GrantsReaderServiceListGrantsForEntitlementRequest_builder{ @@ -1334,8 +1349,11 @@ func TestExternalResourceUserProfileMatch(t *testing.T) { err = internalSyncer.Close(ctx) require.NoError(t, err) - store, err := dotc1z.NewC1ZFile(ctx, internalC1zpath) + // Engine-neutral open: with the pebble default the synced artifact is + // v3, and the sqlite-only NewC1ZFile constructor would reject it. + store, err := dotc1z.NewStore(ctx, internalC1zpath, dotc1z.WithReadOnly(true)) require.NoError(t, err) + defer func() { _ = store.Close(ctx) }() grants, err := store.ListGrantsForEntitlement(ctx, reader_v2.GrantsReaderServiceListGrantsForEntitlementRequest_builder{ Entitlement: internalGroupEnt, @@ -1424,8 +1442,9 @@ func TestExternalResourceGroupProfileMatch(t *testing.T) { require.NoError(t, err) // Verify grant was created for the matching external group - store, err := dotc1z.NewC1ZFile(ctx, internalC1zpath) + store, err := dotc1z.NewStore(ctx, internalC1zpath, dotc1z.WithReadOnly(true)) require.NoError(t, err) + defer func() { _ = store.Close(ctx) }() // Verify the external group was synced with correct properties groupResources, err := store.ListResources(ctx, v2.ResourcesServiceListResourcesRequest_builder{ @@ -1530,8 +1549,9 @@ func TestExternalResourceMatchAllAppTrait(t *testing.T) { require.NoError(t, internalSyncer.Sync(ctx)) require.NoError(t, internalSyncer.Close(ctx)) - store, err := dotc1z.NewC1ZFile(ctx, internalC1zpath) + store, err := dotc1z.NewStore(ctx, internalC1zpath, dotc1z.WithReadOnly(true)) require.NoError(t, err) + defer func() { _ = store.Close(ctx) }() grants, err := store.ListGrantsForEntitlement(ctx, reader_v2.GrantsReaderServiceListGrantsForEntitlementRequest_builder{ Entitlement: internalGroupEnt, @@ -1602,8 +1622,9 @@ func TestExternalResourceAppTraitNotMatchedByDefault(t *testing.T) { require.NoError(t, internalSyncer.Sync(ctx)) require.NoError(t, internalSyncer.Close(ctx)) - store, err := dotc1z.NewC1ZFile(ctx, internalC1zpath) + store, err := dotc1z.NewStore(ctx, internalC1zpath, dotc1z.WithReadOnly(true)) require.NoError(t, err) + defer func() { _ = store.Close(ctx) }() grants, err := store.ListGrantsForEntitlement(ctx, reader_v2.GrantsReaderServiceListGrantsForEntitlementRequest_builder{ Entitlement: internalGroupEnt, @@ -1670,8 +1691,9 @@ func TestExternalResourceWithGrantToEntitlement(t *testing.T) { require.NoError(t, err) // Verify external resources were synced via SyncExternalResourcesWithGrantToEntitlement - store, err := dotc1z.NewC1ZFile(ctx, internalC1zpath) + store, err := dotc1z.NewStore(ctx, internalC1zpath, dotc1z.WithReadOnly(true)) require.NoError(t, err) + defer func() { _ = store.Close(ctx) }() // SyncExternalResourcesWithGrantToEntitlement syncs resources based on grants to a specific entitlement. // It: @@ -1726,11 +1748,11 @@ func TestExternalResourceWithGrantToEntitlement(t *testing.T) { syncID := resp.GetSyncs()[0].GetId() // Clone the finished sync to a new c1z clonedC1zpath := filepath.Join(tempDir, "cloned.c1z") - err = store.CloneSync(ctx, clonedC1zpath, syncID) + err = store.FileOps().CloneSync(ctx, clonedC1zpath, syncID) require.NoError(t, err) // Load the cloned c1z - clonedStore, err := dotc1z.NewC1ZFile(ctx, clonedC1zpath) + clonedStore, err := dotc1z.NewStore(ctx, clonedC1zpath, dotc1z.WithReadOnly(true)) require.NoError(t, err) clonedSyncs, err := clonedStore.ListSyncs(ctx, reader_v2.SyncsReaderServiceListSyncsRequest_builder{ PageSize: 2, @@ -1824,8 +1846,10 @@ func TestResumeSyncWithChildResources(t *testing.T) { err = syncer1.Close(ctxToCancel) require.NoError(t, err) - // Verify that parent_1 was synced before the failure. - store1, err := dotc1z.NewC1ZFile(ctx, c1zPath) + // Verify that parent_1 was synced before the failure. Read-only so + // inspecting the mid-resume artifact cannot write to it before + // syncer2 resumes from the same checkpoint. + store1, err := dotc1z.NewStore(ctx, c1zPath, dotc1z.WithReadOnly(true)) require.NoError(t, err) parentResources, err := store1.ListResources(ctx, v2.ResourcesServiceListResourcesRequest_builder{ @@ -1858,8 +1882,9 @@ func TestResumeSyncWithChildResources(t *testing.T) { require.NoError(t, err) // Verify that child resources are now synced. - store2, err := dotc1z.NewC1ZFile(ctx, c1zPath) + store2, err := dotc1z.NewStore(ctx, c1zPath, dotc1z.WithReadOnly(true)) require.NoError(t, err) + defer func() { _ = store2.Close(ctx) }() childResourcesAfterResume, err := store2.ListResources(ctx, v2.ResourcesServiceListResourcesRequest_builder{ ResourceTypeId: childResourceType.GetId(), @@ -2219,8 +2244,9 @@ func TestSyncGrants_PropagatesInsertResourceGrantsAnnotation(t *testing.T) { require.NoError(t, syncer.Sync(ctx)) require.NoError(t, syncer.Close(ctx)) - store, err := dotc1z.NewC1ZFile(ctx, c1zPath) + store, err := dotc1z.NewStore(ctx, c1zPath, dotc1z.WithReadOnly(true)) require.NoError(t, err) + defer func() { _ = store.Close(ctx) }() resp, err := store.ListGrants(ctx, v2.GrantsServiceListGrantsRequest_builder{}.Build()) require.NoError(t, err) @@ -2260,8 +2286,9 @@ func TestSyncGrants_DoesNotPropagateAnnotationWhenAbsent(t *testing.T) { require.NoError(t, syncer.Sync(ctx)) require.NoError(t, syncer.Close(ctx)) - store, err := dotc1z.NewC1ZFile(ctx, c1zPath) + store, err := dotc1z.NewStore(ctx, c1zPath, dotc1z.WithReadOnly(true)) require.NoError(t, err) + defer func() { _ = store.Close(ctx) }() resp, err := store.ListGrants(ctx, v2.GrantsServiceListGrantsRequest_builder{}.Build()) require.NoError(t, err) diff --git a/pkg/synccompactor/compactor.go b/pkg/synccompactor/compactor.go index b1cfeee3e..243e77b74 100644 --- a/pkg/synccompactor/compactor.go +++ b/pkg/synccompactor/compactor.go @@ -45,9 +45,11 @@ type Compactor struct { c1zOptions []dotc1z.C1ZOption skipGrantExpansion bool // engine selects the storage engine for the compacted output. - // Empty means EngineSQLite (the default; behavior is unchanged and - // the output is byte-identical to the pre-engine-option compactor). - // EnginePebble produces a v3 Pebble c1z via a native record merge. + // Empty means "follow the inputs": Compact resolves it via + // inferEngineFromInputs (any Pebble input → Pebble, all-SQLite → + // SQLite), so the compactor does NOT follow the dotc1z engine + // default. EnginePebble produces a v3 Pebble c1z via a native + // record merge. engine c1zstore.Engine // pebbleMode optionally forces the Pebble merge strategy; the zero // value (Auto) lets the compactor choose. See WithPebbleCompactorMode. @@ -348,13 +350,12 @@ func (c *Compactor) Compact(ctx context.Context) (*CompactableSync, error) { opts = append(opts, dotc1z.WithDecoderPool(c.decoderPool)) } - if c.resolvedEngine() == c1zstore.EnginePebble { - // Force the resolved engine last so a stray engine passed via - // WithC1ZOptions cannot mislabel the artifact. - c.compactedC1z, err = dotc1z.NewStore(ctx, destFilePath, append(opts, dotc1z.WithEngine(c1zstore.EnginePebble))...) - } else { - c.compactedC1z, err = dotc1z.NewStore(ctx, destFilePath, opts...) - } + // Force the resolved engine last: a stray engine passed via + // WithC1ZOptions cannot mislabel the artifact, and the dotc1z + // engine default (Pebble) cannot leak into a SQLite compaction — + // the destination is a new file, so an engine-less open would + // otherwise create a v3 store under the SQLite merge path. + c.compactedC1z, err = dotc1z.NewStore(ctx, destFilePath, append(opts, dotc1z.WithEngine(c.resolvedEngine()))...) if err != nil { l.Error("doOneCompaction failed: could not create c1z file", zap.Error(err)) return nil, err diff --git a/pkg/synccompactor/compactor_pebble.go b/pkg/synccompactor/compactor_pebble.go index c0ee84083..5dd100a46 100644 --- a/pkg/synccompactor/compactor_pebble.go +++ b/pkg/synccompactor/compactor_pebble.go @@ -27,10 +27,11 @@ import ( ) // WithEngine selects the storage engine for the compacted output. -// The default (unset) is sqlite, which is byte-identical to the -// historical compactor. EnginePebble produces a v3 Pebble c1z via a -// native record merge whose strategy (overlay / fold / kway) is -// resolved per run by resolvePebbleMode. +// The default (unset) follows the inputs — any Pebble input makes the +// output Pebble; all-SQLite inputs keep SQLite output, byte-identical +// to the historical compactor (see inferEngineFromInputs). EnginePebble +// produces a v3 Pebble c1z via a native record merge whose strategy +// (overlay / fold / kway) is resolved per run by resolvePebbleMode. // // This is the only supported way to choose the engine; an engine // passed through WithC1ZOptions does not select the compaction diff --git a/proto/c1/connectorapi/baton/v1/baton.proto b/proto/c1/connectorapi/baton/v1/baton.proto index 64a24b404..8855f9bf9 100644 --- a/proto/c1/connectorapi/baton/v1/baton.proto +++ b/proto/c1/connectorapi/baton/v1/baton.proto @@ -59,7 +59,7 @@ message Task { repeated string sync_resource_type_ids = 5; // If true, skip syncing grants. Resources and entitlements will still be synced. bool skip_grants = 6; - // Storage engine to use for the sync. If empty, the default engine will be used (currently SQLite). + // Storage engine to use for the sync. If empty, the default engine will be used (currently Pebble for new c1z files; existing files keep their on-disk format). string storage_engine = 7 [(validate.rules).string = { in: [ "pebble",