From f930fac2b2910046fc9182cfb6e48a1c89caf644 Mon Sep 17 00:00:00 2001 From: manojacs Date: Thu, 16 Jul 2026 18:58:53 +0000 Subject: [PATCH 01/15] feat(expand): diff-aware incremental grant expansion MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Expand only the subgraph affected by an incremental change instead of rebuilding and walking the whole entitlement graph. Seeds from both new edges and changed-membership entitlements, so a new member on an existing group propagates; new edges that close a cycle fall back to full expansion. Source reads stream and writes flush in chunks to bound memory. Additions only — callers use full expansion for change sets with revocations. - expand: IncrementalExpander.ExpandChanges (streaming, chunked flush) - sync: WithPreserveEntitlementGraph, GraphFromToken, NewExpanderStore - synccompactor: WithIncrementalExpansion (diff-aware, cycle fallback) - tests: expander + Pebble-c1z compactor differentials (incremental == full) Co-authored-by: c1-squire-dev[bot] Co-Authored-By: Claude Opus 4.8 --- pkg/sync/expand/incremental.go | 305 +++++++++++++++++ pkg/sync/expand/incremental_e2e_test.go | 144 ++++++++ pkg/sync/expand/incremental_test.go | 306 +++++++++++++++++ pkg/sync/graph_from_token_test.go | 43 +++ pkg/sync/state.go | 12 + pkg/sync/syncer.go | 51 ++- pkg/synccompactor/compactor.go | 133 ++++++++ .../incremental_expansion_test.go | 313 ++++++++++++++++++ 8 files changed, 1293 insertions(+), 14 deletions(-) create mode 100644 pkg/sync/expand/incremental.go create mode 100644 pkg/sync/expand/incremental_e2e_test.go create mode 100644 pkg/sync/expand/incremental_test.go create mode 100644 pkg/sync/graph_from_token_test.go create mode 100644 pkg/synccompactor/incremental_expansion_test.go diff --git a/pkg/sync/expand/incremental.go b/pkg/sync/expand/incremental.go new file mode 100644 index 000000000..ebece5b37 --- /dev/null +++ b/pkg/sync/expand/incremental.go @@ -0,0 +1,305 @@ +package expand + +import ( + "context" + "errors" + "fmt" + + v2 "github.com/conductorone/baton-sdk/pb/c1/connector/v2" + reader_v2 "github.com/conductorone/baton-sdk/pb/c1/reader/v2" +) + +// ErrIncrementalFallback means a new edge closed a cycle; the caller should +// re-run a full expansion, which handles cycles correctly. +var ErrIncrementalFallback = errors.New("incremental expansion: change introduces a cycle, fall back to full expansion") + +// NewEdge is one edge to fold in: members of Source also get Destination. +type NewEdge struct { + SourceEntitlementID string + DestEntitlementID string + Shallow bool + ResourceTypeIDs []string +} + +// IncrementalResult reports the impacted subgraph and how many grants were written. +type IncrementalResult struct { + EntitlementsWalked []string + GrantsWritten int +} + +// IncrementalExpander folds new edges into an already-expanded graph and +// propagates the change to only the affected subgraph, reading and writing +// through the same ExpanderStore as the full expander. +// +// Preconditions: graph is a prior completed expansion's graph (edges already +// expanded), and store holds that expansion's grants. Additions only; a new +// edge that closes a cycle returns ErrIncrementalFallback. +type IncrementalExpander struct { + store ExpanderStore + graph *EntitlementGraph +} + +func NewIncrementalExpander(store ExpanderStore, graph *EntitlementGraph) *IncrementalExpander { + return &IncrementalExpander{store: store, graph: graph} +} + +// ExpandChanges recomputes grants for only the subgraph affected by a set of +// changes. Both kinds seed the walk: newEdges (new expandable relationships, +// added to the graph here) via their destinations, and changedEntitlementIDs +// (entitlements whose membership changed, e.g. a new group member) via their +// own node. The second kind is essential — a new member adds no edge, so +// seeding only from newEdges would silently drop it. +// +// The walk reads current membership from the store, so changed members +// (already merged in) propagate without being passed in. Returns +// ErrIncrementalFallback if a new edge closes a cycle. Additions only: the +// caller must fall back to full expansion for change sets with revocations. +func (ie *IncrementalExpander) ExpandChanges(ctx context.Context, newEdges []NewEdge, changedEntitlementIDs []string) (*IncrementalResult, error) { + if len(newEdges) == 0 && len(changedEntitlementIDs) == 0 { + return &IncrementalResult{}, nil + } + + seeds := make(map[int]struct{}) + for _, e := range newEdges { + ie.graph.AddEntitlementID(e.SourceEntitlementID) + ie.graph.AddEntitlementID(e.DestEntitlementID) + if err := ie.graph.AddEdge(ctx, e.SourceEntitlementID, e.DestEntitlementID, e.Shallow, e.ResourceTypeIDs); err != nil { + return nil, fmt.Errorf("incremental expansion: add edge %s->%s: %w", e.SourceEntitlementID, e.DestEntitlementID, err) + } + if dst := ie.graph.GetNode(e.DestEntitlementID); dst != nil { + seeds[dst.Id] = struct{}{} + } + } + + // A changed entitlement seeds its own node so descendants are recomputed. + // Entitlements not in the graph have nothing downstream — safely ignored. + for _, entitlementID := range changedEntitlementIDs { + if n := ie.graph.GetNode(entitlementID); n != nil { + seeds[n.Id] = struct{}{} + } + } + + if len(seeds) == 0 { + return &IncrementalResult{}, nil + } + + if cyclic, _ := ie.graph.ComputeCyclicComponents(ctx); len(cyclic) > 0 { + return nil, ErrIncrementalFallback + } + + // Only nodes forward-reachable from a seed are touched. + affected := ie.forwardReachable(seeds) + + // Topological order so each destination reads already-finalized parents. + order, err := topologicalNodeOrder(ie.graph) + if err != nil { + return nil, fmt.Errorf("incremental expansion: topological order: %w", err) + } + + result := &IncrementalResult{} + for _, nodeID := range order { + if _, ok := affected[nodeID]; !ok { + continue + } + node, ok := ie.graph.Nodes[nodeID] + if !ok { + continue + } + for _, destEntitlementID := range node.EntitlementIDs { + written, err := ie.recomputeDestination(ctx, nodeID, destEntitlementID) + if err != nil { + return nil, err + } + result.EntitlementsWalked = append(result.EntitlementsWalked, destEntitlementID) + result.GrantsWritten += written + } + } + return result, nil +} + +func (ie *IncrementalExpander) forwardReachable(seeds map[int]struct{}) map[int]struct{} { + reached := make(map[int]struct{}) + queue := make([]int, 0, len(seeds)) + for id := range seeds { + reached[id] = struct{}{} + queue = append(queue, id) + } + for len(queue) > 0 { + cur := queue[0] + queue = queue[1:] + for child := range ie.graph.SourcesToDestinations[cur] { + if _, ok := reached[child]; !ok { + reached[child] = struct{}{} + queue = append(queue, child) + } + } + } + return reached +} + +// incrementalFlushChunk caps buffered new grants before a flush, so a whale +// destination doesn't materialize its whole output. Mirrors the full +// expander's expansionDirtyFlushChunk. A var only so tests can lower it. +var incrementalFlushChunk = 10000 + +// recomputeDestination writes destEntitlementID's implied grants that aren't +// already present, returning how many. Source grants stream a page at a time +// and writes flush in chunks, so peak memory is one page + one flush buffer + +// the destination's existing-key set — not the whole source or output. +func (ie *IncrementalExpander) recomputeDestination(ctx context.Context, nodeID int, destEntitlementID string) (int, error) { + destEnt, err := ie.getEntitlement(ctx, destEntitlementID) + if err != nil { + return 0, err + } + if destEnt == nil { + return 0, nil + } + + existing, err := ie.principalKeySet(ctx, destEntitlementID, nil) + if err != nil { + return 0, err + } + + // added dedups principals across source edges; keys only, lives for the destination. + added := make(map[string]struct{}) + buf := make([]*v2.Grant, 0, incrementalFlushChunk) + written := 0 + + flush := func() error { + if len(buf) == 0 { + return nil + } + if err := ie.store.StoreExpandedGrants(ctx, buf...); err != nil { + return fmt.Errorf("incremental expansion: store grants on %s: %w", destEntitlementID, err) + } + written += len(buf) + buf = buf[:0] + return nil + } + + for sourceNodeID, edgeID := range ie.graph.DestinationsToSources[nodeID] { + edge, ok := ie.graph.Edges[edgeID] + if !ok { + continue + } + sourceNode, ok := ie.graph.Nodes[sourceNodeID] + if !ok { + continue + } + for _, sourceEntitlementID := range sourceNode.EntitlementIDs { + // Stream the source a page at a time (never materialize a whale). + perGrantErr := ie.forEachGrant(ctx, sourceEntitlementID, edge.ResourceTypeIDs, func(sourceGrant *v2.Grant) error { + isSourceDirect := isDirectGrant(sourceGrant) + if edge.IsShallow && !isSourceDirect { + return nil + } + pid := sourceGrant.GetPrincipal().GetId() + key := pid.GetResourceType() + "\x00" + pid.GetResource() + if _, ok := existing[key]; ok { + return nil + } + if _, ok := added[key]; ok { + return nil + } + grant, err := newExpandedGrant(destEnt, sourceGrant.GetPrincipal(), sourceEntitlementID, isSourceDirect) + if err != nil { + return fmt.Errorf("incremental expansion: build grant on %s: %w", destEntitlementID, err) + } + buf = append(buf, grant) + added[key] = struct{}{} + if len(buf) >= incrementalFlushChunk { + return flush() + } + return nil + }) + if perGrantErr != nil { + return 0, perGrantErr + } + } + } + + if err := flush(); err != nil { + return 0, err + } + return written, nil +} + +func (ie *IncrementalExpander) getEntitlement(ctx context.Context, entitlementID string) (*v2.Entitlement, error) { + resp, err := ie.store.GetEntitlement(ctx, reader_v2.EntitlementsReaderServiceGetEntitlementRequest_builder{ + EntitlementId: entitlementID, + }.Build()) + if err != nil { + return nil, fmt.Errorf("incremental expansion: get entitlement %s: %w", entitlementID, err) + } + if resp == nil { + return nil, nil + } + return resp.GetEntitlement(), nil +} + +// forEachGrant streams an entitlement's grants (filtered by resourceTypeIDs) +// one page at a time, invoking fn per grant — never materializing the whole set. +func (ie *IncrementalExpander) forEachGrant(ctx context.Context, entitlementID string, resourceTypeIDs []string, fn func(*v2.Grant) error) error { + pageToken := "" + for { + resp, err := ie.store.ListGrantsForEntitlement(ctx, reader_v2.GrantsReaderServiceListGrantsForEntitlementRequest_builder{ + Entitlement: v2.Entitlement_builder{Id: entitlementID}.Build(), + PrincipalResourceTypeIds: resourceTypeIDs, + PageToken: pageToken, + }.Build()) + if err != nil { + return fmt.Errorf("incremental expansion: list grants for %s: %w", entitlementID, err) + } + for _, g := range resp.GetList() { + if err := fn(g); err != nil { + return err + } + } + pageToken = resp.GetNextPageToken() + if pageToken == "" { + return nil + } + } +} + +// principalKeySet returns the principal keys already granted on an entitlement +// (the diff baseline), using the keys-only fast path when available and +// streaming either way — never materializing the grants. +func (ie *IncrementalExpander) principalKeySet(ctx context.Context, entitlementID string, resourceTypeIDs []string) (map[string]struct{}, error) { + set := make(map[string]struct{}) + + if lister, ok := ie.store.(entitlementGrantPrincipalKeyLister); ok { + ent := v2.Entitlement_builder{Id: entitlementID}.Build() + pageToken := "" + for { + keys, next, err := lister.ListGrantPrincipalKeysForEntitlement(ctx, ent, pageToken, 0) + if err != nil { + return nil, fmt.Errorf("incremental expansion: list principal keys for %s: %w", entitlementID, err) + } + for _, k := range keys { + set[k] = struct{}{} + } + if next == "" { + return set, nil + } + pageToken = next + } + } + + // Fallback: stream grants and extract keys, holding one page at a time. + err := ie.forEachGrant(ctx, entitlementID, resourceTypeIDs, func(g *v2.Grant) error { + pid := g.GetPrincipal().GetId() + set[pid.GetResourceType()+"\x00"+pid.GetResource()] = struct{}{} + return nil + }) + if err != nil { + return nil, err + } + return set, nil +} + +// isDirectGrant reports whether a grant is a direct membership (no expansion +// sources) rather than one produced by a prior expansion. +func isDirectGrant(g *v2.Grant) bool { + return len(g.GetSources().GetSources()) == 0 +} diff --git a/pkg/sync/expand/incremental_e2e_test.go b/pkg/sync/expand/incremental_e2e_test.go new file mode 100644 index 000000000..85773625e --- /dev/null +++ b/pkg/sync/expand/incremental_e2e_test.go @@ -0,0 +1,144 @@ +package expand + +import ( + "context" + "encoding/json" + "sort" + "testing" + + v2 "github.com/conductorone/baton-sdk/pb/c1/connector/v2" + reader_v2 "github.com/conductorone/baton-sdk/pb/c1/reader/v2" + "github.com/stretchr/testify/require" +) + +// roundTripGraph serializes and reloads a graph, mirroring what c1 does across +// an event: persist the graph in the sync token, then deserialize it later. +func roundTripGraph(t *testing.T, g *EntitlementGraph) *EntitlementGraph { + t.Helper() + data, err := json.Marshal(g) + require.NoError(t, err) + out := &EntitlementGraph{} + require.NoError(t, json.Unmarshal(data, out)) + // json leaves absent maps nil; the token-load path re-inits them. + if out.Nodes == nil { + out.Nodes = map[int]Node{} + } + if out.EntitlementsToNodes == nil { + out.EntitlementsToNodes = map[string]int{} + } + if out.SourcesToDestinations == nil { + out.SourcesToDestinations = map[int]map[int]int{} + } + if out.DestinationsToSources == nil { + out.DestinationsToSources = map[int]map[int]int{} + } + if out.Edges == nil { + out.Edges = map[int]Edge{} + } + return out +} + +// grantSet lists every grant across the given entitlements and returns the set +// of "entitlement|principalType|principalResource" keys — the access outcome, +// independent of provenance details. +func grantSet(t *testing.T, ctx context.Context, store *MockExpanderStore, entIDs ...string) map[string]struct{} { + t.Helper() + set := make(map[string]struct{}) + for _, entID := range entIDs { + resp, err := store.ListGrantsForEntitlement(ctx, reader_v2.GrantsReaderServiceListGrantsForEntitlementRequest_builder{ + Entitlement: v2.Entitlement_builder{Id: entID}.Build(), + }.Build()) + require.NoError(t, err) + for _, g := range resp.GetList() { + pid := g.GetPrincipal().GetId() + set[entID+"|"+pid.GetResourceType()+"|"+pid.GetResource()] = struct{}{} + } + } + return set +} + +func keys(set map[string]struct{}) []string { + out := make([]string, 0, len(set)) + for k := range set { + out = append(out, k) + } + sort.Strings(out) + return out +} + +// seedChainStore registers entitlements A,B,C and their direct members, and +// returns a graph with the given edges (unexpanded). ents[i] gets member +// members[i] if non-empty. +func seedChainStore(t *testing.T, ctx context.Context, store *MockExpanderStore, ents []string, members []string, edges [][2]string) *EntitlementGraph { + t.Helper() + g := NewEntitlementGraph(ctx) + for i, e := range ents { + g.AddEntitlementID(e) + store.AddEntitlement(makeEntitlement(e, makeResource("group", e))) + if members[i] != "" { + store.AddGrant(directGrant(e, makeResource("user", members[i]))) + } + } + for _, e := range edges { + require.NoError(t, g.AddEdge(ctx, e[0], e[1], false, nil)) + } + return g +} + +// TestE2E_IncrementalMatchesFullRebuild is the end-to-end correctness bar: +// after loading a persisted graph and folding in one new edge incrementally, +// the store's grants must exactly equal what a full expansion produces when +// that edge was present from the start. +// +// Scenario: base graph has eng:manager -> eng:member (mandy is a manager). +// Event: eng:senior_manager -> eng:manager arrives (sam is a senior manager). +// Expected everywhere: manager = {mandy, sam}, member = {mandy, sam}. +func TestE2E_IncrementalMatchesFullRebuild(t *testing.T) { + ctx := context.Background() + ents := []string{"eng:senior_manager", "eng:manager", "eng:member"} + + // --- Path 1: base full expansion, persist graph, then incremental edge --- + incStore := NewMockExpanderStore() + // Base only knows manager -> member (senior_manager has no edge yet). + baseGraph := seedChainStore(t, ctx, incStore, + ents, + []string{"sam", "mandy", ""}, + [][2]string{{"eng:manager", "eng:member"}}, + ) + require.NoError(t, NewExpander(incStore, baseGraph).Run(ctx)) + + // Persist + reload the graph across the "event boundary". + loaded := roundTripGraph(t, baseGraph) + + // The event: senior_manager -> manager. + ie := NewIncrementalExpander(incStore, loaded) + res, err := ie.ExpandChanges(ctx, []NewEdge{ + {SourceEntitlementID: "eng:senior_manager", DestEntitlementID: "eng:manager"}, + }, nil) + require.NoError(t, err) + require.ElementsMatch(t, []string{"eng:manager", "eng:member"}, res.EntitlementsWalked) + + // --- Path 2: full expansion from scratch with BOTH edges present --- + fullStore := NewMockExpanderStore() + fullGraph := seedChainStore(t, ctx, fullStore, + ents, + []string{"sam", "mandy", ""}, + [][2]string{{"eng:senior_manager", "eng:manager"}, {"eng:manager", "eng:member"}}, + ) + require.NoError(t, NewExpander(fullStore, fullGraph).Run(ctx)) + + // --- The two must agree on the access outcome --- + incGrants := grantSet(t, ctx, incStore, ents...) + fullGrants := grantSet(t, ctx, fullStore, ents...) + require.Equal(t, keys(fullGrants), keys(incGrants), + "incremental result must equal a full rebuild") + + // And spell out the expected access explicitly. + require.Equal(t, []string{ + "eng:manager|user|mandy", + "eng:manager|user|sam", + "eng:member|user|mandy", + "eng:member|user|sam", + "eng:senior_manager|user|sam", + }, keys(incGrants)) +} diff --git a/pkg/sync/expand/incremental_test.go b/pkg/sync/expand/incremental_test.go new file mode 100644 index 000000000..1aaffe1be --- /dev/null +++ b/pkg/sync/expand/incremental_test.go @@ -0,0 +1,306 @@ +package expand + +import ( + "context" + "testing" + + v2 "github.com/conductorone/baton-sdk/pb/c1/connector/v2" + reader_v2 "github.com/conductorone/baton-sdk/pb/c1/reader/v2" + "github.com/stretchr/testify/require" +) + +// buildExpandedChain constructs an already-expanded linear chain +// e0 -> e1 -> ... -> eN in both the graph and the store: every edge is +// marked expanded, and every downstream entitlement already carries the +// root's member. This mirrors the post-sync state an incremental run starts +// from. +func buildExpandedChain(t *testing.T, ctx context.Context, store *MockExpanderStore, rootMember string, ents ...string) *EntitlementGraph { + t.Helper() + g := NewEntitlementGraph(ctx) + for _, e := range ents { + g.AddEntitlementID(e) + store.AddEntitlement(makeEntitlement(e, makeResource("group", e))) + } + // Root's direct grant. + store.AddGrant(directGrant(ents[0], makeResource("user", rootMember))) + for i := 0; i+1 < len(ents); i++ { + require.NoError(t, g.AddEdge(ctx, ents[i], ents[i+1], false, nil)) + g.MarkEdgeExpanded(ents[i], ents[i+1]) + // Downstream already has the root member (expanded), with a source. + store.AddGrant(expandedGrantWithSource(ents[i+1], makeResource("user", rootMember), ents[i])) + } + return g +} + +func directGrant(entitlementID string, principal *v2.Resource) *v2.Grant { + ent := makeEntitlement(entitlementID, makeResource("group", entitlementID)) + return makeGrant( + entitlementID+":"+principal.GetId().GetResourceType()+":"+principal.GetId().GetResource(), + ent, principal, + ) +} + +func expandedGrantWithSource(entitlementID string, principal *v2.Resource, sourceEntitlementID string) *v2.Grant { + g := directGrant(entitlementID, principal) + g.SetSources(v2.GrantSources_builder{ + Sources: map[string]*v2.GrantSources_GrantSource{sourceEntitlementID: {}}, + }.Build()) + return g +} + +func principalsOn(t *testing.T, ctx context.Context, store *MockExpanderStore, entitlementID string) map[string]struct{} { + t.Helper() + resp, err := store.ListGrantsForEntitlement(ctx, reader_v2.GrantsReaderServiceListGrantsForEntitlementRequest_builder{ + Entitlement: v2.Entitlement_builder{Id: entitlementID}.Build(), + }.Build()) + require.NoError(t, err) + out := make(map[string]struct{}) + for _, g := range resp.GetList() { + out[g.GetPrincipal().GetId().GetResource()] = struct{}{} + } + return out +} + +// TestIncremental_LeafAddition: a brand-new destination hung off an existing, +// already-expanded source. Only the new leaf is walked. +func TestIncremental_LeafAddition(t *testing.T) { + ctx := context.Background() + store := NewMockExpanderStore() + g := buildExpandedChain(t, ctx, store, "alice", "eng:member") + store.AddEntitlement(makeEntitlement("github:access", makeResource("app", "github"))) + + ie := NewIncrementalExpander(store, g) + res, err := ie.ExpandChanges(ctx, []NewEdge{{SourceEntitlementID: "eng:member", DestEntitlementID: "github:access"}}, nil) + require.NoError(t, err) + + require.Equal(t, []string{"github:access"}, res.EntitlementsWalked) + require.Equal(t, 1, res.GrantsWritten) + require.Contains(t, principalsOn(t, ctx, store, "github:access"), "alice") +} + +// TestIncremental_UpstreamCascade: insert a new source ABOVE an existing +// expanded chain (the "director" case). Every downstream entitlement gains +// the new principal; only the affected chain is walked. +func TestIncremental_UpstreamCascade(t *testing.T) { + ctx := context.Background() + store := NewMockExpanderStore() + g := buildExpandedChain(t, ctx, store, "alice", + "eng:senior_manager", "eng:manager", "eng:member", "employee:general") + + // A brand-new director entitlement with its own direct member carol. + store.AddEntitlement(makeEntitlement("eng:director", makeResource("group", "eng:director"))) + store.AddGrant(directGrant("eng:director", makeResource("user", "carol"))) + + ie := NewIncrementalExpander(store, g) + res, err := ie.ExpandChanges(ctx, []NewEdge{{SourceEntitlementID: "eng:director", DestEntitlementID: "eng:senior_manager"}}, nil) + require.NoError(t, err) + + // carol cascades to every node from senior_manager down. + for _, e := range []string{"eng:senior_manager", "eng:manager", "eng:member", "employee:general"} { + require.Contains(t, principalsOn(t, ctx, store, e), "carol", "carol should reach %s", e) + require.Contains(t, principalsOn(t, ctx, store, e), "alice", "alice should still be on %s", e) + } + // eng:director itself is NOT walked (it has no incoming edges / isn't a + // destination); only the impacted chain of 4 is. + require.ElementsMatch(t, []string{"eng:senior_manager", "eng:manager", "eng:member", "employee:general"}, res.EntitlementsWalked) + require.Equal(t, 4, res.GrantsWritten) +} + +// TestIncremental_BoundedWalk: adding a leaf to one node of a wide graph must +// touch only that leaf, not the many unrelated sibling nodes. This is the +// core optimization claim: work scales with the impacted subgraph, not the +// whole graph. +func TestIncremental_BoundedWalk(t *testing.T) { + ctx := context.Background() + store := NewMockExpanderStore() + + g := NewEntitlementGraph(ctx) + // 50 unrelated, already-expanded source->dest pairs. + for i := 0; i < 50; i++ { + src := "src" + itoa(i) + dst := "dst" + itoa(i) + g.AddEntitlementID(src) + g.AddEntitlementID(dst) + store.AddEntitlement(makeEntitlement(src, makeResource("group", src))) + store.AddEntitlement(makeEntitlement(dst, makeResource("group", dst))) + store.AddGrant(directGrant(src, makeResource("user", "u"+itoa(i)))) + require.NoError(t, g.AddEdge(ctx, src, dst, false, nil)) + g.MarkEdgeExpanded(src, dst) + store.AddGrant(expandedGrantWithSource(dst, makeResource("user", "u"+itoa(i)), src)) + } + store.AddEntitlement(makeEntitlement("new:leaf", makeResource("app", "new"))) + + ie := NewIncrementalExpander(store, g) + res, err := ie.ExpandChanges(ctx, []NewEdge{{SourceEntitlementID: "src7", DestEntitlementID: "new:leaf"}}, nil) + require.NoError(t, err) + + // Only new:leaf is walked — 1 of 101 entitlements — proving the walk is + // bounded to the impacted subgraph. + require.Equal(t, []string{"new:leaf"}, res.EntitlementsWalked) + require.Contains(t, principalsOn(t, ctx, store, "new:leaf"), "u7") +} + +// TestIncremental_InsertBetween: a new node spliced between two existing +// nodes (A -> newMid -> C added on top of an already-expanded A -> C). The +// mid node is populated from A, and C gains only what the mid contributes +// that it didn't already have via the pre-existing A -> C edge. +func TestIncremental_InsertBetween(t *testing.T) { + ctx := context.Background() + store := NewMockExpanderStore() + // Existing expanded A -> C: alice flows A to C. + g := buildExpandedChain(t, ctx, store, "alice", "a:member", "c:member") + + // New middle entitlement with its own direct member bob. + store.AddEntitlement(makeEntitlement("mid:member", makeResource("group", "mid:member"))) + store.AddGrant(directGrant("mid:member", makeResource("user", "bob"))) + + ie := NewIncrementalExpander(store, g) + res, err := ie.ExpandChanges(ctx, []NewEdge{ + {SourceEntitlementID: "a:member", DestEntitlementID: "mid:member"}, + {SourceEntitlementID: "mid:member", DestEntitlementID: "c:member"}, + }, nil) + require.NoError(t, err) + + // mid gains alice (from A). C gains bob (via mid); alice stays. + require.Contains(t, principalsOn(t, ctx, store, "mid:member"), "alice") + require.Contains(t, principalsOn(t, ctx, store, "mid:member"), "bob") + require.Contains(t, principalsOn(t, ctx, store, "c:member"), "alice") + require.Contains(t, principalsOn(t, ctx, store, "c:member"), "bob") + require.ElementsMatch(t, []string{"mid:member", "c:member"}, res.EntitlementsWalked) +} + +// TestIncremental_NewRootNoEdges: a new root entitlement that isn't part of +// any expandable relationship produces no edges, so there is nothing to +// expand — an empty edge list is a clean no-op. +func TestIncremental_NewRootNoEdges(t *testing.T) { + ctx := context.Background() + store := NewMockExpanderStore() + g := buildExpandedChain(t, ctx, store, "alice", "eng:member") + + ie := NewIncrementalExpander(store, g) + res, err := ie.ExpandChanges(ctx, nil, nil) + require.NoError(t, err) + require.Equal(t, 0, res.GrantsWritten) + require.Empty(t, res.EntitlementsWalked) +} + +// TestIncremental_NewMemberNoNewEdge is the blocker regression: a new member +// added to an EXISTING expandable entitlement produces NO new edge, only a +// changed membership. Seeding via changedEntitlementIDs must still propagate +// the new member downstream. Before the fix, this silently expanded nothing. +func TestIncremental_NewMemberNoNewEdge(t *testing.T) { + ctx := context.Background() + store := NewMockExpanderStore() + // Existing, already-expanded chain: alice flows manager -> member. + g := buildExpandedChain(t, ctx, store, "alice", "eng:manager", "eng:member") + + // A new member (bob) is added to the existing manager entitlement — no new + // edge, just a new direct grant on an entitlement the graph already knows. + store.AddGrant(directGrant("eng:manager", makeResource("user", "bob"))) + + ie := NewIncrementalExpander(store, g) + // No new edges; only the changed entitlement seeds the walk. + res, err := ie.ExpandChanges(ctx, nil, []string{"eng:manager"}) + require.NoError(t, err) + + // bob must reach eng:member (downstream of the changed entitlement). + require.Contains(t, principalsOn(t, ctx, store, "eng:member"), "bob", + "new member on an existing group must propagate downstream") + require.Contains(t, principalsOn(t, ctx, store, "eng:member"), "alice") + require.Equal(t, 1, res.GrantsWritten) // only bob→member is new +} + +// TestIncremental_ChangedEntitlementNotInGraph: a changed entitlement with no +// expandable edges (not a graph node) has nothing downstream — clean no-op, +// no panic. +func TestIncremental_ChangedEntitlementNotInGraph(t *testing.T) { + ctx := context.Background() + store := NewMockExpanderStore() + g := buildExpandedChain(t, ctx, store, "alice", "eng:member") + + ie := NewIncrementalExpander(store, g) + res, err := ie.ExpandChanges(ctx, nil, []string{"some:unrelated:entitlement"}) + require.NoError(t, err) + require.Equal(t, 0, res.GrantsWritten) + require.Empty(t, res.EntitlementsWalked) +} + +// TestIncremental_ChunkedFlush exercises the multi-flush path: with the flush +// chunk lowered below the number of new grants, the destination's grants must +// be flushed in several batches yet still all land, with none dropped or +// duplicated. +func TestIncremental_ChunkedFlush(t *testing.T) { + ctx := context.Background() + store := NewMockExpanderStore() + + // group -> app, both entitlements exist; group has 25 members, none yet on app. + g := NewEntitlementGraph(ctx) + g.AddEntitlementID("group:member") + g.AddEntitlementID("app:access") + store.AddEntitlement(makeEntitlement("group:member", makeResource("group", "g"))) + store.AddEntitlement(makeEntitlement("app:access", makeResource("app", "a"))) + const n = 25 + for i := 0; i < n; i++ { + store.AddGrant(directGrant("group:member", makeResource("user", "u"+itoa(i)))) + } + + // Force multiple flushes: chunk of 10 over 25 grants → 3 flushes. + orig := incrementalFlushChunk + incrementalFlushChunk = 10 + defer func() { incrementalFlushChunk = orig }() + + ie := NewIncrementalExpander(store, g) + res, err := ie.ExpandChanges(ctx, []NewEdge{{SourceEntitlementID: "group:member", DestEntitlementID: "app:access"}}, nil) + require.NoError(t, err) + + require.Equal(t, n, res.GrantsWritten, "all members written across flushes") + got := principalsOn(t, ctx, store, "app:access") + require.Len(t, got, n, "no member dropped or duplicated across flushes") + for i := 0; i < n; i++ { + require.Contains(t, got, "u"+itoa(i)) + } +} + +// TestIncremental_CycleFallback: a new edge that closes a cycle must return +// ErrIncrementalFallback (and leave the edge in the graph for a full +// expansion) rather than silently produce wrong grants. +func TestIncremental_CycleFallback(t *testing.T) { + ctx := context.Background() + store := NewMockExpanderStore() + g := buildExpandedChain(t, ctx, store, "alice", "a:member", "b:member", "c:member") + + ie := NewIncrementalExpander(store, g) + // c -> a closes the cycle a -> b -> c -> a. + _, err := ie.ExpandChanges(ctx, []NewEdge{{SourceEntitlementID: "c:member", DestEntitlementID: "a:member"}}, nil) + require.ErrorIs(t, err, ErrIncrementalFallback) + + // The edge was still added, so a subsequent full expansion sees it. + require.NotNil(t, g.GetNode("a:member")) + require.True(t, g.HasCycles(ctx)) +} + +// TestIncremental_NoOp: re-adding an edge that grants nothing new writes +// nothing. +func TestIncremental_NoOp(t *testing.T) { + ctx := context.Background() + store := NewMockExpanderStore() + g := buildExpandedChain(t, ctx, store, "alice", "eng:member", "github:access") + + ie := NewIncrementalExpander(store, g) + // github:access already has alice (expanded from eng:member). + res, err := ie.ExpandChanges(ctx, []NewEdge{{SourceEntitlementID: "eng:member", DestEntitlementID: "github:access"}}, nil) + require.NoError(t, err) + require.Equal(t, 0, res.GrantsWritten) +} + +func itoa(i int) string { + if i == 0 { + return "0" + } + var b []byte + for i > 0 { + b = append([]byte{byte('0' + i%10)}, b...) + i /= 10 + } + return string(b) +} diff --git a/pkg/sync/graph_from_token_test.go b/pkg/sync/graph_from_token_test.go new file mode 100644 index 000000000..b4687d8c3 --- /dev/null +++ b/pkg/sync/graph_from_token_test.go @@ -0,0 +1,43 @@ +package sync //nolint:revive,nolintlint // package name kept for compatibility + +import ( + "context" + "testing" + + "github.com/conductorone/baton-sdk/pkg/sync/expand" + "github.com/stretchr/testify/require" +) + +// TestGraphFromToken: a graph put into a state token round-trips out via +// GraphFromToken, so an incremental run can reload a prior sync's graph. +func TestGraphFromToken(t *testing.T) { + ctx := context.Background() + + g := expand.NewEntitlementGraph(ctx) + g.AddEntitlementID("eng:manager") + g.AddEntitlementID("eng:member") + require.NoError(t, g.AddEdge(ctx, "eng:manager", "eng:member", false, nil)) + + st := newState() + st.entitlementGraph = g + token, err := st.Marshal() + require.NoError(t, err) + + loaded, err := GraphFromToken(token) + require.NoError(t, err) + require.NotNil(t, loaded) + require.NotNil(t, loaded.GetNode("eng:manager")) + require.NotNil(t, loaded.GetNode("eng:member")) + require.Len(t, loaded.Edges, 1) +} + +// TestGraphFromToken_Empty: a token with no graph yields nil, not an error. +func TestGraphFromToken_Empty(t *testing.T) { + st := newState() + token, err := st.Marshal() + require.NoError(t, err) + + loaded, err := GraphFromToken(token) + require.NoError(t, err) + require.Nil(t, loaded) +} diff --git a/pkg/sync/state.go b/pkg/sync/state.go index 9fbd487e3..fd89e5bf4 100644 --- a/pkg/sync/state.go +++ b/pkg/sync/state.go @@ -97,6 +97,18 @@ func PrepareExpansionReplayToken(stateStr string) (string, error) { return st.Marshal() } +// GraphFromToken parses a sync token and returns its persisted entitlement +// graph, for running an incremental expansion against a prior sync's graph. +// Returns nil if the token carried no graph (e.g. a sync without +// WithPreserveEntitlementGraph). +func GraphFromToken(stateStr string) (*expand.EntitlementGraph, error) { + st := newState() + if err := st.Unmarshal(stateStr); err != nil { + return nil, err + } + return st.entitlementGraph, nil +} + // ActionOp represents a sync operation. type ActionOp uint8 diff --git a/pkg/sync/syncer.go b/pkg/sync/syncer.go index de3c99604..8a93e3324 100644 --- a/pkg/sync/syncer.go +++ b/pkg/sync/syncer.go @@ -200,19 +200,20 @@ type syncer struct { // event (seed/dequeue/commit/abort/done) for post-hoc verification // of the queue contract. Nil in production: one pointer check per // queue operation. - testQueueAudit *queueAudit - connector types.ConnectorClient - state State - runDuration time.Duration - transitionHandler func(s Action) - progressHandler func(p *Progress) - tmpDir string - storageEngine c1zstore.Engine - skipFullSync bool - lastCheckPointTime time.Time - counts *progresslog.ProgressLog - targetedSyncResources []*v2.Resource - onlyExpandGrants bool + testQueueAudit *queueAudit + connector types.ConnectorClient + state State + runDuration time.Duration + transitionHandler func(s Action) + progressHandler func(p *Progress) + tmpDir string + storageEngine c1zstore.Engine + skipFullSync bool + lastCheckPointTime time.Time + counts *progresslog.ProgressLog + targetedSyncResources []*v2.Resource + onlyExpandGrants bool + preserveEntitlementGraph bool // compactionMergedStore marks the store as a pre-sealed artifact // this process did not collect (WithCompactionMergedStore — the // compactor's keep-newer merge and rollback-expansion's replay): @@ -264,6 +265,14 @@ type expanderStoreAdapter struct { store c1zstore.Store } +// NewExpanderStore adapts a c1zstore.Store into an expand.ExpanderStore, +// bridging engine differences (Pebble exposes StoreExpandedGrants on its +// Grants() sub-store, SQLite at top level). Use this instead of type-asserting +// the store, which is unsafe for Pebble. +func NewExpanderStore(store c1zstore.Store) expand.ExpanderStore { + return expanderStoreAdapter{store: store} +} + func (a expanderStoreAdapter) GetEntitlement(ctx context.Context, req *reader_v2.EntitlementsReaderServiceGetEntitlementRequest) (*reader_v2.EntitlementsReaderServiceGetEntitlementResponse, error) { return a.store.GetEntitlement(ctx, req) } @@ -1052,7 +1061,12 @@ func (s *syncer) Sync(ctx context.Context) error { } // Force a checkpoint to clear completed actions & entitlement graph in sync_token. - s.state.ClearEntitlementGraph(ctx) + // preserveEntitlementGraph keeps the graph in the final token so a later + // incremental expansion can reload it instead of rebuilding from scratch. + if !s.preserveEntitlementGraph { + s.state.ClearEntitlementGraph(ctx) + } + s.state.ClearExclusionGroupTracking(ctx) err = s.Checkpoint(ctx, true) if err != nil { @@ -3992,6 +4006,15 @@ func WithCompactionMergedStore() SyncOpt { } } +// WithPreserveEntitlementGraph keeps the entitlement graph in the final sync +// token instead of clearing it at sync end, so a later incremental expansion +// can reload it rather than rebuilding it from scratch. +func WithPreserveEntitlementGraph() SyncOpt { + return func(s *syncer) { + s.preserveEntitlementGraph = true + } +} + // WithDontExpandGrants sets whether to skip expanding grants. // This is used for speeding up service mode connectors and reducing their c1z upload size. // C1 will process the uploaded c1z and expand grants itself. diff --git a/pkg/synccompactor/compactor.go b/pkg/synccompactor/compactor.go index b1cfeee3e..1d94c7f2d 100644 --- a/pkg/synccompactor/compactor.go +++ b/pkg/synccompactor/compactor.go @@ -16,6 +16,7 @@ import ( "github.com/conductorone/baton-sdk/pkg/dotc1z/c1zstore" "github.com/conductorone/baton-sdk/pkg/sdk" "github.com/conductorone/baton-sdk/pkg/sync" + "github.com/conductorone/baton-sdk/pkg/sync/expand" "github.com/conductorone/baton-sdk/pkg/synccompactor/attached" "github.com/conductorone/baton-sdk/pkg/tempdir" "github.com/grpc-ecosystem/go-grpc-middleware/logging/zap/ctxzap" @@ -44,6 +45,13 @@ type Compactor struct { syncLimit int c1zOptions []dotc1z.C1ZOption skipGrantExpansion bool + // incrementalBaseGraph, when set, enables diff-aware expansion: only changes + // relative to this base-sync graph are expanded. nil (default) = full expansion. + incrementalBaseGraph *expand.EntitlementGraph + // incrementalChangedEntitlementIDs are entitlements whose membership changed + // in the increments; they seed the walk so new members propagate (without + // them a new member on an existing group is missed). + incrementalChangedEntitlementIDs []string // engine selects the storage engine for the compacted output. // Empty means EngineSQLite (the default; behavior is unchanged and // the output is byte-identical to the pre-engine-option compactor). @@ -169,6 +177,21 @@ func WithTmpDir(tempDir string) Option { } } +// WithIncrementalExpansion enables diff-aware grant expansion during compaction. +// baseGraph is the base sync's graph (via sync.GraphFromToken); changedEntitlementIDs +// are the entitlements whose membership changed in the increments (their grants' +// entitlement ids), which seed the walk so new members propagate. A new edge that +// closes a cycle falls back to full expansion; nil baseGraph (default) = full. +// +// Additions-only: do NOT enable for change sets with revocations (pass no base +// graph → full expansion), since removals are not propagated here. +func WithIncrementalExpansion(baseGraph *expand.EntitlementGraph, changedEntitlementIDs []string) Option { + return func(c *Compactor) { + c.incrementalBaseGraph = baseGraph + c.incrementalChangedEntitlementIDs = changedEntitlementIDs + } +} + // Deprecated: There is now only one compactor type, so this option is no longer needed. func WithCompactorType(compactorType CompactorType) Option { return func(c *Compactor) { @@ -565,8 +588,118 @@ func (c *Compactor) doOneCompaction(ctx context.Context, cs *CompactableSync) er return nil } +// expandGrantsIncremental runs a diff-aware expansion over the compacted c1z. +// Returns (true, nil) when handled, (false, nil) when declined (cycle), and +// (false, err) on error. In the false cases the caller runs full expansion; +// writes are idempotent by grant identity, so partial state is tolerated. +func (c *Compactor) expandGrantsIncremental(ctx context.Context, newSyncId string) (bool, error) { + base := c.incrementalBaseGraph + + // The merge left the sync ended; resume it so grants can be written (end + + // close on the way out). Fetch the type first so resume finds the existing sync. + syncResp, err := c.compactedC1z.GetSync(ctx, reader_v2.SyncsReaderServiceGetSyncRequest_builder{SyncId: newSyncId}.Build()) + if err != nil { + return false, fmt.Errorf("incremental expansion: get sync: %w", err) + } + syncType := connectorstore.SyncType(syncResp.GetSync().GetSyncType()) + if _, _, err := c.compactedC1z.StartOrResumeSync(ctx, syncType, newSyncId); err != nil { + return false, fmt.Errorf("incremental expansion: resume sync: %w", err) + } + + // Every rule grant currently in the compacted c1z (base + merged + // increments) yields one or more edges. New edges are those the base graph + // didn't already have expanded. + var newEdges []expand.NewEdge + for pe, err := range c.compactedC1z.Grants().PendingExpansion(ctx) { + if err != nil { + return false, fmt.Errorf("incremental expansion: enumerate pending: %w", err) + } + anno := pe.Annotation + if anno == nil { + continue + } + for _, src := range anno.GetEntitlementIds() { + if baseGraphHasEdge(base, src, pe.TargetEntitlementID) { + continue + } + newEdges = append(newEdges, expand.NewEdge{ + SourceEntitlementID: src, + DestEntitlementID: pe.TargetEntitlementID, + Shallow: anno.GetShallow(), + ResourceTypeIDs: anno.GetResourceTypeIds(), + }) + } + } + + if len(newEdges) == 0 && len(c.incrementalChangedEntitlementIDs) == 0 { + // Nothing changed relative to the base — its grants were already merged in. + return c.finishIncrementalExpansion(ctx) + } + + ie := expand.NewIncrementalExpander(sync.NewExpanderStore(c.compactedC1z), base) + if _, err := ie.ExpandChanges(ctx, newEdges, c.incrementalChangedEntitlementIDs); err != nil { + if errors.Is(err, expand.ErrIncrementalFallback) { + // Restore the ended state and let the full path re-run. + if endErr := c.compactedC1z.EndSync(ctx); endErr != nil { + return false, fmt.Errorf("incremental expansion: end after fallback: %w", endErr) + } + return false, nil + } + return false, err + } + return c.finishIncrementalExpansion(ctx) +} + +// finishIncrementalExpansion ends + closes the store so the file is flushed +// before cpFile copies it (same finalization the full path gets from syncer.Close). +func (c *Compactor) finishIncrementalExpansion(ctx context.Context) (bool, error) { + if err := c.compactedC1z.EndSync(ctx); err != nil { + return false, fmt.Errorf("incremental expansion: end sync: %w", err) + } + if err := c.compactedC1z.Close(ctx); err != nil { + return false, fmt.Errorf("incremental expansion: close: %w", err) + } + return true, nil +} + +// baseGraphHasEdge reports whether the base graph already has an edge src->dst. +// Endpoints collapsed into one node (a fixed cycle) count as already-present. +func baseGraphHasEdge(g *expand.EntitlementGraph, src, dst string) bool { + sn := g.GetNode(src) + dn := g.GetNode(dst) + if sn == nil || dn == nil { + return false + } + if sn.Id == dn.Id { + return true + } + dests, ok := g.SourcesToDestinations[sn.Id] + if !ok { + return false + } + _, ok = dests[dn.Id] + return ok +} + func (c *Compactor) expandGrants(ctx context.Context, newSyncId string, compactionStart time.Time) error { l := ctxzap.Extract(ctx) + + // Diff-aware fast path: with a base graph, expand only what changed relative + // to it. Any doubt (cycle, error) falls through to full expansion below. + if c.incrementalBaseGraph != nil { + done, err := c.expandGrantsIncremental(ctx, newSyncId) + switch { + case err != nil: + l.Warn("incremental expansion failed; falling back to full expansion", zap.Error(err)) + case done: + // Incremental path already ended + closed the store; caller clears + // c.compactedC1z after return, same as the full path. + return nil + default: + l.Info("incremental expansion declined (cycle); falling back to full expansion") + } + } + // Grant expansion doesn't use the connector interface at all, so giving syncer an empty connector is safe... for now. // If that ever changes, we should implement a file connector that is a wrapper around the reader. emptyConnector, err := sdk.NewEmptyConnector() diff --git a/pkg/synccompactor/incremental_expansion_test.go b/pkg/synccompactor/incremental_expansion_test.go new file mode 100644 index 000000000..71b04352e --- /dev/null +++ b/pkg/synccompactor/incremental_expansion_test.go @@ -0,0 +1,313 @@ +package synccompactor + +import ( + "context" + "path/filepath" + "sort" + "testing" + + "github.com/stretchr/testify/require" + + v2 "github.com/conductorone/baton-sdk/pb/c1/connector/v2" + "github.com/conductorone/baton-sdk/pkg/annotations" + "github.com/conductorone/baton-sdk/pkg/connectorstore" + "github.com/conductorone/baton-sdk/pkg/dotc1z" + "github.com/conductorone/baton-sdk/pkg/dotc1z/c1zstore" + "github.com/conductorone/baton-sdk/pkg/sync/expand" + batonGrant "github.com/conductorone/baton-sdk/pkg/types/grant" +) + +// This test proves the compactor's diff-aware incremental expansion produces +// the SAME grants as a full expansion, on real Pebble c1z files. +// +// Scenario: +// base (full, already expanded): ent-b -> ent-c ; mandy is a member of B, +// and (already expanded) of C. +// increment (partial): ent-a -> ent-b ; sam is a member of A. +// After compaction, both sam and mandy should be members of B and C. + +func grp(id string) *v2.Resource { + return v2.Resource_builder{ + Id: v2.ResourceId_builder{ResourceType: "group", Resource: id}.Build(), + DisplayName: id, + }.Build() +} + +func usr(id string) *v2.Resource { + return v2.Resource_builder{ + Id: v2.ResourceId_builder{ResourceType: "user", Resource: id}.Build(), + DisplayName: id, + }.Build() +} + +func ent(id string, resource *v2.Resource) *v2.Entitlement { + return v2.Entitlement_builder{ + Id: id, + Resource: resource, + Purpose: v2.Entitlement_PURPOSE_VALUE_ASSIGNMENT, + }.Build() +} + +// ruleGrant builds an expandable grant: the source group is granted the +// destination entitlement, with a GrantExpandable annotation naming the source +// entitlement — i.e. "members of sourceEntID also get destEnt". +func ruleGrant(destEnt *v2.Entitlement, sourceGroup *v2.Resource, sourceEntID string) *v2.Grant { + g := v2.Grant_builder{ + Id: batonGrant.NewGrantID(sourceGroup, destEnt), + Entitlement: destEnt, + Principal: sourceGroup, + }.Build() + g.SetAnnotations(annotations.New(v2.GrantExpandable_builder{ + EntitlementIds: []string{sourceEntID}, + }.Build())) + return g +} + +func memberGrant(e *v2.Entitlement, principal *v2.Resource) *v2.Grant { + return v2.Grant_builder{ + Id: batonGrant.NewGrantID(principal, e), + Entitlement: e, + Principal: principal, + }.Build() +} + +func expandedGrant(e *v2.Entitlement, principal *v2.Resource, sourceEntID string) *v2.Grant { + g := memberGrant(e, principal) + g.SetSources(v2.GrantSources_builder{ + Sources: map[string]*v2.GrantSources_GrantSource{sourceEntID: {}}, + }.Build()) + return g +} + +// buildIncrementalFixtures writes a base (full, pre-expanded) c1z and an +// increment (partial) c1z into dir, returning the compactable entries. +func buildIncrementalFixtures(t *testing.T, ctx context.Context, dir string) []*CompactableSync { + t.Helper() + + grpA, grpB, grpC := grp("grpA"), grp("grpB"), grp("grpC") + sam, mandy := usr("sam"), usr("mandy") + entA, entB, entC := ent("ent-a", grpA), ent("ent-b", grpB), ent("ent-c", grpC) + + userRT := v2.ResourceType_builder{Id: "user", DisplayName: "User"}.Build() + groupRT := v2.ResourceType_builder{Id: "group", DisplayName: "Group"}.Build() + + // --- base: full, already expanded (ent-b -> ent-c, mandy on both) --- + basePath := filepath.Join(dir, "base.c1z") + base, err := dotc1z.NewStore(ctx, basePath, dotc1z.WithEngine(c1zstore.EnginePebble)) + require.NoError(t, err) + baseSyncID, err := base.StartNewSync(ctx, connectorstore.SyncTypeFull, "") + require.NoError(t, err) + require.NoError(t, base.PutResourceTypes(ctx, userRT, groupRT)) + require.NoError(t, base.PutResources(ctx, grpB, grpC, mandy)) + require.NoError(t, base.PutEntitlements(ctx, entB, entC)) + require.NoError(t, base.PutGrants(ctx, + memberGrant(entB, mandy), // mandy is a direct member of B + expandedGrant(entC, mandy, "ent-b"), // already expanded: mandy on C via B + ruleGrant(entC, grpB, "ent-b"), // rule: members of B get C + )) + require.NoError(t, base.EndSync(ctx)) + require.NoError(t, base.Close(ctx)) + + // --- increment: partial, adds ent-a -> ent-b with sam on A --- + incPath := filepath.Join(dir, "inc.c1z") + inc, err := dotc1z.NewStore(ctx, incPath, dotc1z.WithEngine(c1zstore.EnginePebble)) + require.NoError(t, err) + incSyncID, err := inc.StartNewSync(ctx, connectorstore.SyncTypePartial, "") + require.NoError(t, err) + require.NoError(t, inc.PutResourceTypes(ctx, userRT, groupRT)) + require.NoError(t, inc.PutResources(ctx, grpA, sam)) + require.NoError(t, inc.PutEntitlements(ctx, entA)) + require.NoError(t, inc.PutGrants(ctx, + memberGrant(entA, sam), // sam is a direct member of A + ruleGrant(entB, grpA, "ent-a"), // rule: members of A get B + )) + require.NoError(t, inc.EndSync(ctx)) + require.NoError(t, inc.Close(ctx)) + + return []*CompactableSync{ + {FilePath: basePath, SyncID: baseSyncID}, + {FilePath: incPath, SyncID: incSyncID}, + } +} + +// baseGraphForFixtures returns the in-memory graph the base sync would have +// persisted (ent-b -> ent-c, already expanded) — what sync.GraphFromToken +// would hand back in production. +func baseGraphForFixtures(t *testing.T, ctx context.Context) *expand.EntitlementGraph { + t.Helper() + g := expand.NewEntitlementGraph(ctx) + g.AddEntitlementID("ent-b") + g.AddEntitlementID("ent-c") + require.NoError(t, g.AddEdge(ctx, "ent-b", "ent-c", false, nil)) + g.MarkEdgeExpanded("ent-b", "ent-c") + return g +} + +// grantOutcome reads every grant from a compacted c1z and returns the set of +// "entitlement|principalType|principalResource" keys. +func grantOutcome(t *testing.T, ctx context.Context, path, syncID string) []string { + t.Helper() + store, err := dotc1z.NewStore(ctx, path, dotc1z.WithReadOnly(true)) + require.NoError(t, err) + defer store.Close(ctx) + require.NoError(t, store.SetCurrentSync(ctx, syncID)) + + set := map[string]struct{}{} + pageToken := "" + for { + resp, err := store.ListGrants(ctx, v2.GrantsServiceListGrantsRequest_builder{ + PageSize: 1000, + PageToken: pageToken, + }.Build()) + require.NoError(t, err) + for _, g := range resp.GetList() { + pid := g.GetPrincipal().GetId() + set[g.GetEntitlement().GetId()+"|"+pid.GetResourceType()+"|"+pid.GetResource()] = struct{}{} + } + pageToken = resp.GetNextPageToken() + if pageToken == "" { + break + } + } + out := make([]string, 0, len(set)) + for k := range set { + out = append(out, k) + } + sort.Strings(out) + return out +} + +func TestCompactor_IncrementalExpansionMatchesFull(t *testing.T) { + ctx := context.Background() + + // --- Path A: incremental expansion (base graph supplied) --- + incDir := t.TempDir() + incEntries := buildIncrementalFixtures(t, ctx, incDir) + cInc, cleanupInc, err := NewCompactor(ctx, t.TempDir(), incEntries, + WithTmpDir(t.TempDir()), + WithEngine(c1zstore.EnginePebble), + WithIncrementalExpansion(baseGraphForFixtures(t, ctx), nil), + ) + require.NoError(t, err) + defer func() { _ = cleanupInc() }() + incOut, err := cInc.Compact(ctx) + require.NoError(t, err) + require.NotNil(t, incOut) + + // --- Path B: full expansion (no base graph) --- + fullDir := t.TempDir() + fullEntries := buildIncrementalFixtures(t, ctx, fullDir) + cFull, cleanupFull, err := NewCompactor(ctx, t.TempDir(), fullEntries, + WithTmpDir(t.TempDir()), + WithEngine(c1zstore.EnginePebble), + ) + require.NoError(t, err) + defer func() { _ = cleanupFull() }() + fullOut, err := cFull.Compact(ctx) + require.NoError(t, err) + require.NotNil(t, fullOut) + + // --- The two must agree --- + incGrants := grantOutcome(t, ctx, incOut.FilePath, incOut.SyncID) + fullGrants := grantOutcome(t, ctx, fullOut.FilePath, fullOut.SyncID) + require.Equal(t, fullGrants, incGrants, "incremental expansion must equal full expansion") + + // sam (from A) must have reached B and C via the cascade. + require.Contains(t, incGrants, "ent-b|user|sam") + require.Contains(t, incGrants, "ent-c|user|sam") + require.Contains(t, incGrants, "ent-c|user|mandy") +} + +// buildNewMemberFixtures writes a base (full, pre-expanded ent-b -> ent-c) c1z +// and an increment that adds a NEW MEMBER (bob) to the existing ent-b — with +// NO new rule grant / edge. This is the blocker case: bob must still reach +// ent-c. +func buildNewMemberFixtures(t *testing.T, ctx context.Context, dir string) []*CompactableSync { + t.Helper() + + grpB, grpC := grp("grpB"), grp("grpC") + mandy, bob := usr("mandy"), usr("bob") + entB, entC := ent("ent-b", grpB), ent("ent-c", grpC) + + userRT := v2.ResourceType_builder{Id: "user", DisplayName: "User"}.Build() + groupRT := v2.ResourceType_builder{Id: "group", DisplayName: "Group"}.Build() + + // base: full, already expanded (ent-b -> ent-c, mandy on both). + basePath := filepath.Join(dir, "base.c1z") + base, err := dotc1z.NewStore(ctx, basePath, dotc1z.WithEngine(c1zstore.EnginePebble)) + require.NoError(t, err) + baseSyncID, err := base.StartNewSync(ctx, connectorstore.SyncTypeFull, "") + require.NoError(t, err) + require.NoError(t, base.PutResourceTypes(ctx, userRT, groupRT)) + require.NoError(t, base.PutResources(ctx, grpB, grpC, mandy)) + require.NoError(t, base.PutEntitlements(ctx, entB, entC)) + require.NoError(t, base.PutGrants(ctx, + memberGrant(entB, mandy), + expandedGrant(entC, mandy, "ent-b"), + ruleGrant(entC, grpB, "ent-b"), + )) + require.NoError(t, base.EndSync(ctx)) + require.NoError(t, base.Close(ctx)) + + // increment: partial, adds bob as a direct member of the EXISTING ent-b. + // No rule grant → no new edge. + incPath := filepath.Join(dir, "inc.c1z") + inc, err := dotc1z.NewStore(ctx, incPath, dotc1z.WithEngine(c1zstore.EnginePebble)) + require.NoError(t, err) + incSyncID, err := inc.StartNewSync(ctx, connectorstore.SyncTypePartial, "") + require.NoError(t, err) + require.NoError(t, inc.PutResourceTypes(ctx, userRT, groupRT)) + require.NoError(t, inc.PutResources(ctx, grpB, bob)) + require.NoError(t, inc.PutEntitlements(ctx, entB)) + require.NoError(t, inc.PutGrants(ctx, memberGrant(entB, bob))) + require.NoError(t, inc.EndSync(ctx)) + require.NoError(t, inc.Close(ctx)) + + return []*CompactableSync{ + {FilePath: basePath, SyncID: baseSyncID}, + {FilePath: incPath, SyncID: incSyncID}, + } +} + +// TestCompactor_IncrementalNewMemberMatchesFull is the blocker regression at +// the compactor level: an increment that adds a new member to an existing +// group (no new edge) must still propagate that member downstream, and match +// full expansion. The changed entitlement id ("ent-b") is passed so the walk +// is seeded from it. +func TestCompactor_IncrementalNewMemberMatchesFull(t *testing.T) { + ctx := context.Background() + + // Path A: incremental, seeded with the changed entitlement. + incEntries := buildNewMemberFixtures(t, ctx, t.TempDir()) + cInc, cleanupInc, err := NewCompactor(ctx, t.TempDir(), incEntries, + WithTmpDir(t.TempDir()), + WithEngine(c1zstore.EnginePebble), + WithIncrementalExpansion(baseGraphForFixtures(t, ctx), []string{"ent-b"}), + ) + require.NoError(t, err) + defer func() { _ = cleanupInc() }() + incOut, err := cInc.Compact(ctx) + require.NoError(t, err) + require.NotNil(t, incOut) + + // Path B: full expansion (no base graph). + fullEntries := buildNewMemberFixtures(t, ctx, t.TempDir()) + cFull, cleanupFull, err := NewCompactor(ctx, t.TempDir(), fullEntries, + WithTmpDir(t.TempDir()), + WithEngine(c1zstore.EnginePebble), + ) + require.NoError(t, err) + defer func() { _ = cleanupFull() }() + fullOut, err := cFull.Compact(ctx) + require.NoError(t, err) + require.NotNil(t, fullOut) + + incGrants := grantOutcome(t, ctx, incOut.FilePath, incOut.SyncID) + fullGrants := grantOutcome(t, ctx, fullOut.FilePath, fullOut.SyncID) + require.Equal(t, fullGrants, incGrants, "incremental (new member) must equal full expansion") + + // The blocker: bob (new member of B) must have reached C. + require.Contains(t, incGrants, "ent-b|user|bob") + require.Contains(t, incGrants, "ent-c|user|bob") + require.Contains(t, incGrants, "ent-c|user|mandy") +} From 18ea45a70cbfed569c2008def2e83ff2d8727345 Mon Sep 17 00:00:00 2001 From: manojacs Date: Fri, 17 Jul 2026 04:20:47 +0000 Subject: [PATCH 02/15] address PR #1013 review feedback Fixes from kans's review of the diff-aware incremental grant expansion: - B1: pass full entitlement records (not bare ids) to all grant reads so the incremental path stops erroring into a silent full-expansion fallback; compactor tests assert it actually ran. - B2: scope incremental to Pebble; degrade gracefully to full on SQLite. - C1: use isGrantDirectOnEntitlement for shallow filtering and IsDirect. - C2: merge sources into existing grants and record all contributing edges; parity tests compare full rows including sources maps. - Derive changed entitlements inside the compactor; drop the caller param. Compare edge specs (not just endpoints): widened re-expands, narrowed auto-declines via a named ErrIncrementalRevocationDecline hook. - U1: clone the base graph so a failed/declined run can't poison a retry. - U2: clear a preserved graph in PrepareExpansionReplayToken so replay works. - Converge the finish path: Cleanup/EndSync/Close on a detached ctx, fatal teardown errors vs safe fallback, run-duration bound + ctx polling, dangling-ref skip-with-warn. - Strip transient graph state before the final checkpoint. - Tests: dangling-ref + sealed-artifact lifecycle + revocation parity + shallow-directness differentials. Co-authored-by: c1-squire-dev[bot] Co-Authored-By: Claude Fable 5 --- pkg/sync/expand/incremental.go | 292 +++++++++---- pkg/sync/expand/incremental_e2e_test.go | 109 ++++- pkg/sync/expand/incremental_test.go | 80 +++- pkg/sync/graph_from_token_test.go | 37 ++ pkg/sync/state.go | 17 + pkg/sync/syncer.go | 8 +- pkg/synccompactor/compactor.go | 361 +++++++++++++--- .../incremental_expansion_test.go | 389 +++++++++++++++++- 8 files changed, 1118 insertions(+), 175 deletions(-) diff --git a/pkg/sync/expand/incremental.go b/pkg/sync/expand/incremental.go index ebece5b37..4dd49fb04 100644 --- a/pkg/sync/expand/incremental.go +++ b/pkg/sync/expand/incremental.go @@ -2,17 +2,74 @@ package expand import ( "context" + "encoding/json" "errors" "fmt" + "sort" v2 "github.com/conductorone/baton-sdk/pb/c1/connector/v2" reader_v2 "github.com/conductorone/baton-sdk/pb/c1/reader/v2" + batonGrant "github.com/conductorone/baton-sdk/pkg/types/grant" + "github.com/grpc-ecosystem/go-grpc-middleware/logging/zap/ctxzap" + "go.uber.org/zap" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" ) +// ClearTransientState drops the graph's expansion working state — the action +// queue, the projection plan, and metrics — which a persisted graph doesn't +// need for a later reload and which can bloat the sync token. The structural +// graph (nodes, edges, mappings) is untouched. +func (g *EntitlementGraph) ClearTransientState() { + g.Actions = nil + g.ExpansionPlan = nil + g.ExpansionMetrics = nil +} + +// Clone returns a deep copy of the graph. Incremental expansion mutates the +// graph (adds edges), so callers that keep the base graph across retries must +// pass a clone — otherwise a failed run leaves the never-expanded edges in the +// caller's graph, and a retry would treat them as already present and finish +// with an unexpanded artifact. +func (g *EntitlementGraph) Clone() (*EntitlementGraph, error) { + data, err := json.Marshal(g) + if err != nil { + return nil, fmt.Errorf("clone entitlement graph: %w", err) + } + out := &EntitlementGraph{} + if err := json.Unmarshal(data, out); err != nil { + return nil, fmt.Errorf("clone entitlement graph: %w", err) + } + // json leaves absent maps nil; reinit so the clone is immediately usable. + if out.Nodes == nil { + out.Nodes = map[int]Node{} + } + if out.EntitlementsToNodes == nil { + out.EntitlementsToNodes = map[string]int{} + } + if out.SourcesToDestinations == nil { + out.SourcesToDestinations = map[int]map[int]int{} + } + if out.DestinationsToSources == nil { + out.DestinationsToSources = map[int]map[int]int{} + } + if out.Edges == nil { + out.Edges = map[int]Edge{} + } + return out, nil +} + // ErrIncrementalFallback means a new edge closed a cycle; the caller should // re-run a full expansion, which handles cycles correctly. var ErrIncrementalFallback = errors.New("incremental expansion: change introduces a cycle, fall back to full expansion") +// ErrIncrementalRevocationDecline means the change is revocation-shaped (an +// existing edge's spec narrowed — shallow-ified, filter tightened, or a source +// dropped), which incremental expansion cannot apply without removing grants. +// The caller declines to full expansion. This is the named hook a future +// tombstone/deletion stage flips from "decline" to "apply deletions". +var ErrIncrementalRevocationDecline = errors.New("incremental expansion: revocation-shaped change, fall back to full expansion") + // NewEdge is one edge to fold in: members of Source also get Destination. type NewEdge struct { SourceEntitlementID string @@ -46,14 +103,21 @@ func NewIncrementalExpander(store ExpanderStore, graph *EntitlementGraph) *Incre // ExpandChanges recomputes grants for only the subgraph affected by a set of // changes. Both kinds seed the walk: newEdges (new expandable relationships, // added to the graph here) via their destinations, and changedEntitlementIDs -// (entitlements whose membership changed, e.g. a new group member) via their -// own node. The second kind is essential — a new member adds no edge, so -// seeding only from newEdges would silently drop it. +// (entitlements whose membership changed) via their own node. The second kind +// is essential — a membership change adds no edge, so seeding only from +// newEdges would silently drop it. +// +// changedEntitlementIDs is direction-neutral: it names entitlements whose +// membership changed in EITHER direction (added or removed). Whether a removal +// is actually applied is a WRITE-behavior concern, not a seed concern — today +// this method only adds grants (never removes), so callers decline +// revocation-shaped changes to full expansion. When a future stage learns to +// apply deletions, removed-membership entitlements flow through this same +// parameter with no signature change. // // The walk reads current membership from the store, so changed members // (already merged in) propagate without being passed in. Returns -// ErrIncrementalFallback if a new edge closes a cycle. Additions only: the -// caller must fall back to full expansion for change sets with revocations. +// ErrIncrementalFallback if a new edge closes a cycle. func (ie *IncrementalExpander) ExpandChanges(ctx context.Context, newEdges []NewEdge, changedEntitlementIDs []string) (*IncrementalResult, error) { if len(newEdges) == 0 && len(changedEntitlementIDs) == 0 { return &IncrementalResult{}, nil @@ -98,6 +162,9 @@ func (ie *IncrementalExpander) ExpandChanges(ctx context.Context, newEdges []New result := &IncrementalResult{} for _, nodeID := range order { + if err := ctx.Err(); err != nil { + return nil, err // cancelled / run-duration exceeded + } if _, ok := affected[nodeID]; !ok { continue } @@ -152,31 +219,19 @@ func (ie *IncrementalExpander) recomputeDestination(ctx context.Context, nodeID return 0, err } if destEnt == nil { + // Dangling ref: skip-with-warn, matching the full evaluator (don't + // error into a fallback). + ctxzap.Extract(ctx).Warn("incremental expansion: destination entitlement not in store; skipping", + zap.String("entitlement_id", destEntitlementID)) return 0, nil } - existing, err := ie.principalKeySet(ctx, destEntitlementID, nil) - if err != nil { - return 0, err - } - - // added dedups principals across source edges; keys only, lives for the destination. - added := make(map[string]struct{}) - buf := make([]*v2.Grant, 0, incrementalFlushChunk) - written := 0 - - flush := func() error { - if len(buf) == 0 { - return nil - } - if err := ie.store.StoreExpandedGrants(ctx, buf...); err != nil { - return fmt.Errorf("incremental expansion: store grants on %s: %w", destEntitlementID, err) - } - written += len(buf) - buf = buf[:0] - return nil - } - + // 1. Accumulate, per principal, the union of sources contributed by all + // incoming edges. Streaming reads keep only one source page live, but the + // contribution map holds one entry per distinct principal across ALL + // sources feeding this destination — the same worst-case fan-in footprint + // the full expander's per-destination reduce carries. + contrib := make(map[string]*principalContribution) for sourceNodeID, edgeID := range ie.graph.DestinationsToSources[nodeID] { edge, ok := ie.graph.Edges[edgeID] if !ok { @@ -187,29 +242,35 @@ func (ie *IncrementalExpander) recomputeDestination(ctx context.Context, nodeID continue } for _, sourceEntitlementID := range sourceNode.EntitlementIDs { - // Stream the source a page at a time (never materialize a whale). - perGrantErr := ie.forEachGrant(ctx, sourceEntitlementID, edge.ResourceTypeIDs, func(sourceGrant *v2.Grant) error { - isSourceDirect := isDirectGrant(sourceGrant) + // The store's read path requires a full entitlement record (with + // resource refs), not a bare id — fetch it. A dangling ref (source + // not in the store) is skipped-with-warn, matching the full evaluator. + sourceEnt, err := ie.getEntitlement(ctx, sourceEntitlementID) + if err != nil { + return 0, err + } + if sourceEnt == nil { + ctxzap.Extract(ctx).Warn("incremental expansion: source entitlement not in store; skipping", + zap.String("entitlement_id", sourceEntitlementID)) + continue + } + perGrantErr := ie.forEachGrant(ctx, sourceEnt, edge.ResourceTypeIDs, func(sourceGrant *v2.Grant) error { + // Directness is relative to the source entitlement (matches the + // full expander): a plain direct grant or one whose sources map + // records this entitlement counts as direct. + isSourceDirect := isGrantDirectOnEntitlement(sourceGrant, sourceEntitlementID) if edge.IsShallow && !isSourceDirect { return nil } - pid := sourceGrant.GetPrincipal().GetId() + principal := sourceGrant.GetPrincipal() + pid := principal.GetId() key := pid.GetResourceType() + "\x00" + pid.GetResource() - if _, ok := existing[key]; ok { - return nil - } - if _, ok := added[key]; ok { - return nil - } - grant, err := newExpandedGrant(destEnt, sourceGrant.GetPrincipal(), sourceEntitlementID, isSourceDirect) - if err != nil { - return fmt.Errorf("incremental expansion: build grant on %s: %w", destEntitlementID, err) - } - buf = append(buf, grant) - added[key] = struct{}{} - if len(buf) >= incrementalFlushChunk { - return flush() + pc := contrib[key] + if pc == nil { + pc = &principalContribution{principal: principal} + contrib[key] = pc } + pc.addSource(sourceEntitlementID, isSourceDirect) return nil }) if perGrantErr != nil { @@ -217,6 +278,69 @@ func (ie *IncrementalExpander) recomputeDestination(ctx context.Context, nodeID } } } + if len(contrib) == 0 { + return 0, nil + } + + buf := make([]*v2.Grant, 0, incrementalFlushChunk) + written := 0 + flush := func() error { + if len(buf) == 0 { + return nil + } + if err := ie.store.StoreExpandedGrants(ctx, buf...); err != nil { + return fmt.Errorf("incremental expansion: store grants on %s: %w", destEntitlementID, err) + } + written += len(buf) + buf = buf[:0] + return nil + } + + // 2. Merge contributions into the destination's existing grants (union the + // sources map, upgrade direct-ness), streaming one page at a time. Only a + // grant that actually changed is rewritten; matched principals leave contrib. + mergeErr := ie.forEachGrant(ctx, destEnt, nil, func(g *v2.Grant) error { + pid := g.GetPrincipal().GetId() + key := pid.GetResourceType() + "\x00" + pid.GetResource() + pc := contrib[key] + if pc == nil { + return nil + } + delete(contrib, key) + updated := mergeContributionIntoExistingGrant(g, destEntitlementID, pc.sources) + if updated == nil { + return nil // already had these sources — no write + } + buf = append(buf, updated) + if len(buf) >= incrementalFlushChunk { + return flush() + } + return nil + }) + if mergeErr != nil { + return 0, mergeErr + } + + // 3. Whatever is left in contrib are brand-new principals. Sort for + // deterministic (byte-stable) output. + newKeys := make([]string, 0, len(contrib)) + for key := range contrib { + newKeys = append(newKeys, key) + } + sort.Strings(newKeys) + for _, key := range newKeys { + pc := contrib[key] + grant, err := newExpandedGrantWithSources(destEnt, pc.principal, pc.sources) + if err != nil { + return 0, fmt.Errorf("incremental expansion: build grant on %s: %w", destEntitlementID, err) + } + buf = append(buf, grant) + if len(buf) >= incrementalFlushChunk { + if err := flush(); err != nil { + return 0, err + } + } + } if err := flush(); err != nil { return 0, err @@ -224,11 +348,37 @@ func (ie *IncrementalExpander) recomputeDestination(ctx context.Context, nodeID return written, nil } +// principalContribution accumulates the source entitlements contributing one +// principal to a destination. sources is a small slice (fan-in is tiny), deduped +// by entitlement id with direct-ness upgraded to true if any contribution is direct. +type principalContribution struct { + principal *v2.Resource + sources batonGrant.Sources +} + +func (pc *principalContribution) addSource(entitlementID string, isDirect bool) { + for i := range pc.sources { + if pc.sources[i].EntitlementID == entitlementID { + if isDirect && !pc.sources[i].IsDirect { + pc.sources[i].IsDirect = true + } + return + } + } + pc.sources = append(pc.sources, batonGrant.Source{EntitlementID: entitlementID, IsDirect: isDirect}) +} + +// getEntitlement fetches an entitlement, returning (nil, nil) for a dangling +// ref (NotFound) so callers skip it — matching the full evaluator, which treats +// NotFound as skip rather than a hard error. func (ie *IncrementalExpander) getEntitlement(ctx context.Context, entitlementID string) (*v2.Entitlement, error) { resp, err := ie.store.GetEntitlement(ctx, reader_v2.EntitlementsReaderServiceGetEntitlementRequest_builder{ EntitlementId: entitlementID, }.Build()) if err != nil { + if status.Code(err) == codes.NotFound { + return nil, nil + } return nil, fmt.Errorf("incremental expansion: get entitlement %s: %w", entitlementID, err) } if resp == nil { @@ -238,17 +388,19 @@ func (ie *IncrementalExpander) getEntitlement(ctx context.Context, entitlementID } // forEachGrant streams an entitlement's grants (filtered by resourceTypeIDs) -// one page at a time, invoking fn per grant — never materializing the whole set. -func (ie *IncrementalExpander) forEachGrant(ctx context.Context, entitlementID string, resourceTypeIDs []string, fn func(*v2.Grant) error) error { +// one page at a time, invoking fn per grant — never materializing the whole +// set. entitlement must be a full record (with resource refs); the store's read +// path rejects bare-id entitlements. +func (ie *IncrementalExpander) forEachGrant(ctx context.Context, entitlement *v2.Entitlement, resourceTypeIDs []string, fn func(*v2.Grant) error) error { pageToken := "" for { resp, err := ie.store.ListGrantsForEntitlement(ctx, reader_v2.GrantsReaderServiceListGrantsForEntitlementRequest_builder{ - Entitlement: v2.Entitlement_builder{Id: entitlementID}.Build(), + Entitlement: entitlement, PrincipalResourceTypeIds: resourceTypeIDs, PageToken: pageToken, }.Build()) if err != nil { - return fmt.Errorf("incremental expansion: list grants for %s: %w", entitlementID, err) + return fmt.Errorf("incremental expansion: list grants for %s: %w", entitlement.GetId(), err) } for _, g := range resp.GetList() { if err := fn(g); err != nil { @@ -261,45 +413,3 @@ func (ie *IncrementalExpander) forEachGrant(ctx context.Context, entitlementID s } } } - -// principalKeySet returns the principal keys already granted on an entitlement -// (the diff baseline), using the keys-only fast path when available and -// streaming either way — never materializing the grants. -func (ie *IncrementalExpander) principalKeySet(ctx context.Context, entitlementID string, resourceTypeIDs []string) (map[string]struct{}, error) { - set := make(map[string]struct{}) - - if lister, ok := ie.store.(entitlementGrantPrincipalKeyLister); ok { - ent := v2.Entitlement_builder{Id: entitlementID}.Build() - pageToken := "" - for { - keys, next, err := lister.ListGrantPrincipalKeysForEntitlement(ctx, ent, pageToken, 0) - if err != nil { - return nil, fmt.Errorf("incremental expansion: list principal keys for %s: %w", entitlementID, err) - } - for _, k := range keys { - set[k] = struct{}{} - } - if next == "" { - return set, nil - } - pageToken = next - } - } - - // Fallback: stream grants and extract keys, holding one page at a time. - err := ie.forEachGrant(ctx, entitlementID, resourceTypeIDs, func(g *v2.Grant) error { - pid := g.GetPrincipal().GetId() - set[pid.GetResourceType()+"\x00"+pid.GetResource()] = struct{}{} - return nil - }) - if err != nil { - return nil, err - } - return set, nil -} - -// isDirectGrant reports whether a grant is a direct membership (no expansion -// sources) rather than one produced by a prior expansion. -func isDirectGrant(g *v2.Grant) bool { - return len(g.GetSources().GetSources()) == 0 -} diff --git a/pkg/sync/expand/incremental_e2e_test.go b/pkg/sync/expand/incremental_e2e_test.go index 85773625e..923f6a4b8 100644 --- a/pkg/sync/expand/incremental_e2e_test.go +++ b/pkg/sync/expand/incremental_e2e_test.go @@ -3,7 +3,9 @@ package expand import ( "context" "encoding/json" + "fmt" "sort" + "strings" "testing" v2 "github.com/conductorone/baton-sdk/pb/c1/connector/v2" @@ -39,8 +41,9 @@ func roundTripGraph(t *testing.T, g *EntitlementGraph) *EntitlementGraph { } // grantSet lists every grant across the given entitlements and returns the set -// of "entitlement|principalType|principalResource" keys — the access outcome, -// independent of provenance details. +// of full-row keys "entitlement|principalType|principalResource|sources=..." +// — INCLUDING the sources/provenance map, so incremental is held to producing +// byte-identical provenance to a full expansion, not just the same access. func grantSet(t *testing.T, ctx context.Context, store *MockExpanderStore, entIDs ...string) map[string]struct{} { t.Helper() set := make(map[string]struct{}) @@ -51,12 +54,23 @@ func grantSet(t *testing.T, ctx context.Context, store *MockExpanderStore, entID require.NoError(t, err) for _, g := range resp.GetList() { pid := g.GetPrincipal().GetId() - set[entID+"|"+pid.GetResourceType()+"|"+pid.GetResource()] = struct{}{} + set[entID+"|"+pid.GetResourceType()+"|"+pid.GetResource()+"|"+sourcesString(g)] = struct{}{} } } return set } +// sourcesString renders a grant's sources map as a stable "id:direct,..." string. +func sourcesString(g *v2.Grant) string { + srcs := g.GetSources().GetSources() + parts := make([]string, 0, len(srcs)) + for id, s := range srcs { + parts = append(parts, fmt.Sprintf("%s:%t", id, s.GetIsDirect())) + } + sort.Strings(parts) + return "sources=" + strings.Join(parts, ",") +} + func keys(set map[string]struct{}) []string { out := make([]string, 0, len(set)) for k := range set { @@ -127,18 +141,85 @@ func TestE2E_IncrementalMatchesFullRebuild(t *testing.T) { ) require.NoError(t, NewExpander(fullStore, fullGraph).Run(ctx)) - // --- The two must agree on the access outcome --- + // --- The two must agree on full rows, INCLUDING sources/provenance --- incGrants := grantSet(t, ctx, incStore, ents...) fullGrants := grantSet(t, ctx, fullStore, ents...) require.Equal(t, keys(fullGrants), keys(incGrants), - "incremental result must equal a full rebuild") - - // And spell out the expected access explicitly. - require.Equal(t, []string{ - "eng:manager|user|mandy", - "eng:manager|user|sam", - "eng:member|user|mandy", - "eng:member|user|sam", - "eng:senior_manager|user|sam", - }, keys(incGrants)) + "incremental result must equal a full rebuild, sources included") + + // Sanity: the expected access is present (sources vary, so match by prefix). + require.NotEmpty(t, incGrants) + requireHasGrant(t, incGrants, "eng:manager|user|sam") + requireHasGrant(t, incGrants, "eng:member|user|sam") + requireHasGrant(t, incGrants, "eng:member|user|mandy") +} + +// TestE2E_ShallowEdgeDirectnessMatchesFull (C1 differential): a principal who +// is BOTH a direct member of the source and expanded into it (sources map +// carries the source entitlement) must propagate over a new shallow edge, and +// a principal who is only expanded into the source must not — exactly matching +// a full rebuild, sources included. +// +// alice: direct on X and direct on B. bob: direct on X only (reaches B via +// X -> B deep). New shallow edge B -> Y arrives: alice qualifies (direct on +// B), bob does not. +func TestE2E_ShallowEdgeDirectnessMatchesFull(t *testing.T) { + ctx := context.Background() + ents := []string{"x:member", "b:member", "y:access"} + + seed := func(store *MockExpanderStore) *EntitlementGraph { + g := NewEntitlementGraph(ctx) + for _, e := range ents { + g.AddEntitlementID(e) + store.AddEntitlement(makeEntitlement(e, makeResource("group", e))) + } + store.AddGrant(directGrant("x:member", makeResource("user", "alice"))) + store.AddGrant(directGrant("x:member", makeResource("user", "bob"))) + store.AddGrant(directGrant("b:member", makeResource("user", "alice"))) + require.NoError(t, g.AddEdge(ctx, "x:member", "b:member", false, nil)) // deep + return g + } + + // --- Path 1: expand the base (X->B), persist+reload, then the shallow edge --- + incStore := NewMockExpanderStore() + baseGraph := seed(incStore) + require.NoError(t, NewExpander(incStore, baseGraph).Run(ctx)) + loaded := roundTripGraph(t, baseGraph) + + ie := NewIncrementalExpander(incStore, loaded) + _, err := ie.ExpandChanges(ctx, []NewEdge{ + {SourceEntitlementID: "b:member", DestEntitlementID: "y:access", Shallow: true}, + }, nil) + require.NoError(t, err) + + // --- Path 2: full expansion from scratch with BOTH edges present --- + fullStore := NewMockExpanderStore() + fullGraph := seed(fullStore) + require.NoError(t, fullGraph.AddEdge(ctx, "b:member", "y:access", true, nil)) // shallow + require.NoError(t, NewExpander(fullStore, fullGraph).Run(ctx)) + + // --- Full-row parity, sources included --- + incGrants := grantSet(t, ctx, incStore, ents...) + fullGrants := grantSet(t, ctx, fullStore, ents...) + require.Equal(t, keys(fullGrants), keys(incGrants), + "shallow-edge incremental must equal a full rebuild, sources included") + + // alice (direct on B) crossed the shallow edge; bob (expanded-only) did not. + requireHasGrant(t, incGrants, "y:access|user|alice") + for k := range incGrants { + require.False(t, strings.HasPrefix(k, "y:access|user|bob|"), + "bob is not direct on b:member and must not cross a shallow edge") + } +} + +// requireHasGrant asserts some key in the set starts with the given +// entitlement|type|resource prefix (ignoring the sources suffix). +func requireHasGrant(t *testing.T, set map[string]struct{}, prefix string) { + t.Helper() + for k := range set { + if strings.HasPrefix(k, prefix+"|") { + return + } + } + t.Fatalf("expected a grant with prefix %q in %v", prefix, keys(set)) } diff --git a/pkg/sync/expand/incremental_test.go b/pkg/sync/expand/incremental_test.go index 1aaffe1be..bc6c5e0d9 100644 --- a/pkg/sync/expand/incremental_test.go +++ b/pkg/sync/expand/incremental_test.go @@ -2,6 +2,7 @@ package expand import ( "context" + "encoding/json" "testing" v2 "github.com/conductorone/baton-sdk/pb/c1/connector/v2" @@ -25,10 +26,10 @@ func buildExpandedChain(t *testing.T, ctx context.Context, store *MockExpanderSt store.AddGrant(directGrant(ents[0], makeResource("user", rootMember))) for i := 0; i+1 < len(ents); i++ { require.NoError(t, g.AddEdge(ctx, ents[i], ents[i+1], false, nil)) - g.MarkEdgeExpanded(ents[i], ents[i+1]) - // Downstream already has the root member (expanded), with a source. - store.AddGrant(expandedGrantWithSource(ents[i+1], makeResource("user", rootMember), ents[i])) } + // Run the real expander so the base carries realistic expanded grants and + // provenance (sources maps), not hand-seeded approximations. + require.NoError(t, NewExpander(store, g).Run(ctx)) return g } @@ -261,6 +262,79 @@ func TestIncremental_ChunkedFlush(t *testing.T) { } } +// TestIncremental_ShallowEdgeDirectness (C1): a principal who is direct on the +// source but whose grant also carries a sources map must still count as direct +// on a shallow edge. The naive "no sources == direct" check would drop them; +// the correct predicate (sources empty OR the source entitlement is recorded) +// keeps them. +func TestIncremental_ShallowEdgeDirectness(t *testing.T) { + ctx := context.Background() + store := NewMockExpanderStore() + + g := NewEntitlementGraph(ctx) + g.AddEntitlementID("group-b:member") + g.AddEntitlementID("y:access") + store.AddEntitlement(makeEntitlement("group-b:member", makeResource("group", "b"))) + store.AddEntitlement(makeEntitlement("y:access", makeResource("app", "y"))) + // alice is a direct member of B, but her grant records B in its sources map + // (IsDirect) — so len(sources) != 0 even though she IS direct on B. + store.AddGrant(expandedGrantWithSource("group-b:member", makeResource("user", "alice"), "group-b:member")) + + ie := NewIncrementalExpander(store, g) + // New SHALLOW edge B -> Y: only direct members of B should propagate. + res, err := ie.ExpandChanges(ctx, []NewEdge{ + {SourceEntitlementID: "group-b:member", DestEntitlementID: "y:access", Shallow: true}, + }, nil) + require.NoError(t, err) + + // alice is direct on B, so she must reach Y over the shallow edge. + require.Contains(t, principalsOn(t, ctx, store, "y:access"), "alice", + "a direct member with a non-empty sources map must propagate over a shallow edge") + require.Equal(t, 1, res.GrantsWritten) +} + +// TestIncremental_PreservedGraphTokenSize (#12) measures the serialized size of +// a large nested-groups graph and confirms ClearTransientState shrinks it. It's +// a measurement (logged), plus a guard that stripping never grows the token. +func TestIncremental_PreservedGraphTokenSize(t *testing.T) { + ctx := context.Background() + g := NewEntitlementGraph(ctx) + + // A wide, shallow nested-groups graph: 5000 groups each feeding a common + // downstream entitlement (fan-in), plus a chain spine (depth). + const groups = 5000 + g.AddEntitlementID("downstream:access") + for i := 0; i < groups; i++ { + src := "group:" + itoa(i) + ":member" + g.AddEntitlementID(src) + require.NoError(t, g.AddEdge(ctx, src, "downstream:access", false, nil)) + g.MarkEdgeExpanded(src, "downstream:access") + } + // Simulate leftover transient state a real run would carry. + g.Actions = []*EntitlementGraphAction{{SourceEntitlementID: "group:0:member"}} + g.ExpansionMetrics = &EntitlementGraphMetrics{Algorithm: "topological_projection"} + + full, err := json.Marshal(g) + require.NoError(t, err) + + g.ClearTransientState() + stripped, err := json.Marshal(g) + require.NoError(t, err) + + t.Logf("preserved-graph token: %d nodes, full=%d bytes, stripped=%d bytes", + groups+1, len(full), len(stripped)) + require.LessOrEqual(t, len(stripped), len(full), "stripping transient state must not grow the token") + require.Nil(t, g.Actions) + require.Nil(t, g.ExpansionPlan) + require.Nil(t, g.ExpansionMetrics) + + // The stripped graph still round-trips and is usable. + loaded, err := g.Clone() + require.NoError(t, err) + require.NotNil(t, loaded.GetNode("downstream:access")) + require.Len(t, loaded.Edges, groups) +} + // TestIncremental_CycleFallback: a new edge that closes a cycle must return // ErrIncrementalFallback (and leave the edge in the graph for a full // expansion) rather than silently produce wrong grants. diff --git a/pkg/sync/graph_from_token_test.go b/pkg/sync/graph_from_token_test.go index b4687d8c3..f8846aee3 100644 --- a/pkg/sync/graph_from_token_test.go +++ b/pkg/sync/graph_from_token_test.go @@ -41,3 +41,40 @@ func TestGraphFromToken_Empty(t *testing.T) { require.NoError(t, err) require.Nil(t, loaded) } + +// TestPrepareExpansionReplayToken_ClearsPreservedGraph (U2): a token that +// preserved its entitlement graph (WithPreserveEntitlementGraph) must have that +// graph cleared when rewritten for replay — otherwise the replay skips graph +// loading (graph already "expanded") and silently no-ops. +func TestPrepareExpansionReplayToken_ClearsPreservedGraph(t *testing.T) { + ctx := context.Background() + + // A finished sync that preserved its (fully-expanded) graph. + g := expand.NewEntitlementGraph(ctx) + g.AddEntitlementID("eng:manager") + g.AddEntitlementID("eng:member") + require.NoError(t, g.AddEdge(ctx, "eng:manager", "eng:member", false, nil)) + g.MarkEdgeExpanded("eng:manager", "eng:member") + g.Loaded = true + + st := newState() + st.entitlementGraph = g + token, err := st.Marshal() + require.NoError(t, err) + // Sanity: the token really does carry a graph. + pre, err := GraphFromToken(token) + require.NoError(t, err) + require.NotNil(t, pre) + + // Rewriting for replay must strip the graph so the replay rebuilds it. + replay, err := PrepareExpansionReplayToken(token) + require.NoError(t, err) + + got, err := GraphFromToken(replay) + require.NoError(t, err) + require.Nil(t, got, "replay token must not carry the preserved graph") + + needs, err := NeedsExpansion(replay) + require.NoError(t, err) + require.True(t, needs, "replay token must be marked needs-expansion") +} diff --git a/pkg/sync/state.go b/pkg/sync/state.go index fd89e5bf4..edcd1eec0 100644 --- a/pkg/sync/state.go +++ b/pkg/sync/state.go @@ -35,6 +35,7 @@ type State interface { NextPage(ctx context.Context, actionID string, pageToken string) error EntitlementGraph(ctx context.Context) *expand.EntitlementGraph ClearEntitlementGraph(ctx context.Context) + ClearEntitlementGraphTransientState(ctx context.Context) Current() *Action GetAction(id string) *Action PeekMatchingActions(ctx context.Context, op ActionOp) []*Action @@ -86,6 +87,12 @@ func PrepareExpansionReplayToken(stateStr string) (string, error) { return "", err } st.SetNeedsExpansion() + // Clear any preserved entitlement graph. A graph preserved by + // WithPreserveEntitlementGraph has Loaded=true with every edge already + // marked expanded, so a replayed sync would skip graph loading and the + // expander would report done immediately — the replay would silently + // no-op. Clearing it makes the replay rebuild the graph from scratch. + st.ClearEntitlementGraph(context.Background()) if st.Current() == nil { // A finished sync deserializes with no action map, so seed one before // queuing the InitOp that drives the resumed run. @@ -1110,6 +1117,16 @@ func (st *state) ClearEntitlementGraph(ctx context.Context) { st.entitlementGraph = nil } +// ClearEntitlementGraphTransientState strips a preserved graph's expansion +// working state before the final checkpoint. A no-op when no graph was built — +// deliberately NOT EntitlementGraph(ctx), which would allocate an empty graph +// into the final token where prior behavior serialized none. +func (st *state) ClearEntitlementGraphTransientState(_ context.Context) { + if st.entitlementGraph != nil { + st.entitlementGraph.ClearTransientState() + } +} + func (st *state) GetCompletedActionsCount() uint64 { st.mtx.RLock() defer st.mtx.RUnlock() diff --git a/pkg/sync/syncer.go b/pkg/sync/syncer.go index 8a93e3324..99d3c8e76 100644 --- a/pkg/sync/syncer.go +++ b/pkg/sync/syncer.go @@ -1062,8 +1062,12 @@ func (s *syncer) Sync(ctx context.Context) error { // Force a checkpoint to clear completed actions & entitlement graph in sync_token. // preserveEntitlementGraph keeps the graph in the final token so a later - // incremental expansion can reload it instead of rebuilding from scratch. - if !s.preserveEntitlementGraph { + // incremental expansion can reload it instead of rebuilding from scratch — + // but strip its transient working state (action queue, expansion plan, + // metrics) first, which a reload doesn't need and which bloats the token. + if s.preserveEntitlementGraph { + s.state.ClearEntitlementGraphTransientState(ctx) + } else { s.state.ClearEntitlementGraph(ctx) } s.state.ClearExclusionGroupTracking(ctx) diff --git a/pkg/synccompactor/compactor.go b/pkg/synccompactor/compactor.go index 1d94c7f2d..4c710c097 100644 --- a/pkg/synccompactor/compactor.go +++ b/pkg/synccompactor/compactor.go @@ -8,8 +8,10 @@ import ( "os" "path" "path/filepath" + "sort" "time" + v2 "github.com/conductorone/baton-sdk/pb/c1/connector/v2" reader_v2 "github.com/conductorone/baton-sdk/pb/c1/reader/v2" "github.com/conductorone/baton-sdk/pkg/connectorstore" "github.com/conductorone/baton-sdk/pkg/dotc1z" @@ -47,11 +49,13 @@ type Compactor struct { skipGrantExpansion bool // incrementalBaseGraph, when set, enables diff-aware expansion: only changes // relative to this base-sync graph are expanded. nil (default) = full expansion. + // The set of changed entitlements is derived from the applied increments + // during expansion, not supplied by the caller. incrementalBaseGraph *expand.EntitlementGraph - // incrementalChangedEntitlementIDs are entitlements whose membership changed - // in the increments; they seed the walk so new members propagate (without - // them a new member on an existing group is missed). - incrementalChangedEntitlementIDs []string + // incrementalExpansionRan records whether the diff-aware path actually + // handled expansion (vs falling back to full). Read by tests to prove the + // fast path ran rather than silently falling back. + incrementalExpansionRan bool // engine selects the storage engine for the compacted output. // Empty means EngineSQLite (the default; behavior is unchanged and // the output is byte-identical to the pre-engine-option compactor). @@ -178,17 +182,17 @@ func WithTmpDir(tempDir string) Option { } // WithIncrementalExpansion enables diff-aware grant expansion during compaction. -// baseGraph is the base sync's graph (via sync.GraphFromToken); changedEntitlementIDs -// are the entitlements whose membership changed in the increments (their grants' -// entitlement ids), which seed the walk so new members propagate. A new edge that -// closes a cycle falls back to full expansion; nil baseGraph (default) = full. +// baseGraph is the base sync's graph (via sync.GraphFromToken). The set of +// entitlements whose membership changed is derived from the applied increments +// during expansion (not supplied by the caller), so new members propagate. A +// new edge that closes a cycle falls back to full expansion; nil baseGraph +// (default) = full. // -// Additions-only: do NOT enable for change sets with revocations (pass no base -// graph → full expansion), since removals are not propagated here. -func WithIncrementalExpansion(baseGraph *expand.EntitlementGraph, changedEntitlementIDs []string) Option { +// Additions-only: a revocation-shaped change (a narrowed edge spec) auto-declines +// to full expansion; removals are not propagated incrementally. +func WithIncrementalExpansion(baseGraph *expand.EntitlementGraph) Option { return func(c *Compactor) { c.incrementalBaseGraph = baseGraph - c.incrementalChangedEntitlementIDs = changedEntitlementIDs } } @@ -589,20 +593,51 @@ func (c *Compactor) doOneCompaction(ctx context.Context, cs *CompactableSync) er } // expandGrantsIncremental runs a diff-aware expansion over the compacted c1z. -// Returns (true, nil) when handled, (false, nil) when declined (cycle), and -// (false, err) on error. In the false cases the caller runs full expansion; -// writes are idempotent by grant identity, so partial state is tolerated. -func (c *Compactor) expandGrantsIncremental(ctx context.Context, newSyncId string) (bool, error) { - base := c.incrementalBaseGraph +// errIncrementalFatal marks incremental-expansion errors that must FAIL the +// compaction rather than fall back to full expansion: the store is mid-teardown +// (or could not be restored to its ended state), so running the full path +// against it is unsafe. Every other error is safe to fall back on — the store +// was untouched or restored, and expanded-grant writes are idempotent. +var errIncrementalFatal = errors.New("incremental expansion: fatal") + +// Returns (true, nil) when it handled expansion. Errors come in three shapes: +// decline sentinels (ErrIncrementalFallback for a cycle, +// ErrIncrementalRevocationDecline for a narrowed edge) and plain errors both +// mean "fall back to full expansion" — the store is in the ended state the +// full path expects; errors wrapped in errIncrementalFatal mean the store's +// finalization failed and the compaction must fail. Finalization always runs +// on a detached context so a run-duration timeout can't abort it. +func (c *Compactor) expandGrantsIncremental(ctx context.Context, newSyncId string, compactionStart time.Time) (bool, error) { + // Clone so a failed/declined run never mutates the caller-held base graph + // (a retry with the original must not see never-expanded edges as present). + base, err := c.incrementalBaseGraph.Clone() + if err != nil { + return false, fmt.Errorf("incremental expansion: clone base graph: %w", err) + } + + // Bound the walk by the remaining run duration; the walk polls ctx.Err(). + // Finalization uses detached contexts, so an expired walk deadline never + // aborts the end/cleanup/close. + walkCtx := ctx + if c.runDuration > 0 { + remaining := c.runDuration - time.Since(compactionStart) + if remaining <= 0 { + // Let the full path surface its canonical run-duration error. + return false, fmt.Errorf("incremental expansion: run duration expired before expansion") + } + var cancel context.CancelFunc + walkCtx, cancel = context.WithTimeout(ctx, remaining) + defer cancel() + } // The merge left the sync ended; resume it so grants can be written (end + // close on the way out). Fetch the type first so resume finds the existing sync. - syncResp, err := c.compactedC1z.GetSync(ctx, reader_v2.SyncsReaderServiceGetSyncRequest_builder{SyncId: newSyncId}.Build()) + syncResp, err := c.compactedC1z.GetSync(walkCtx, reader_v2.SyncsReaderServiceGetSyncRequest_builder{SyncId: newSyncId}.Build()) if err != nil { return false, fmt.Errorf("incremental expansion: get sync: %w", err) } syncType := connectorstore.SyncType(syncResp.GetSync().GetSyncType()) - if _, _, err := c.compactedC1z.StartOrResumeSync(ctx, syncType, newSyncId); err != nil { + if _, _, err := c.compactedC1z.StartOrResumeSync(walkCtx, syncType, newSyncId); err != nil { return false, fmt.Errorf("incremental expansion: resume sync: %w", err) } @@ -610,8 +645,11 @@ func (c *Compactor) expandGrantsIncremental(ctx context.Context, newSyncId strin // increments) yields one or more edges. New edges are those the base graph // didn't already have expanded. var newEdges []expand.NewEdge - for pe, err := range c.compactedC1z.Grants().PendingExpansion(ctx) { + for pe, err := range c.compactedC1z.Grants().PendingExpansion(walkCtx) { if err != nil { + if endErr := c.restoreEndedSync(ctx); endErr != nil { + return false, endErr + } return false, fmt.Errorf("incremental expansion: enumerate pending: %w", err) } anno := pe.Annotation @@ -619,66 +657,264 @@ func (c *Compactor) expandGrantsIncremental(ctx context.Context, newSyncId strin continue } for _, src := range anno.GetEntitlementIds() { - if baseGraphHasEdge(base, src, pe.TargetEntitlementID) { - continue - } - newEdges = append(newEdges, expand.NewEdge{ + baseEdge, inBase := baseGraphEdge(base, src, pe.TargetEntitlementID) + curEdge := expand.NewEdge{ SourceEntitlementID: src, DestEntitlementID: pe.TargetEntitlementID, Shallow: anno.GetShallow(), ResourceTypeIDs: anno.GetResourceTypeIds(), - }) + } + if !inBase { + newEdges = append(newEdges, curEdge) // brand-new edge + continue + } + // Existing edge: compare specs, not just endpoints (C3). + switch classifyEdgeSpecChange(baseEdge, curEdge) { + case edgeSpecNarrowed: + // Revocation-shaped (shallow-ified / filter tightened): can't + // remove grants incrementally — decline via the named hook (#6). + if endErr := c.restoreEndedSync(ctx); endErr != nil { + return false, endErr + } + return false, expand.ErrIncrementalRevocationDecline + case edgeSpecWidened: + // More members now qualify: re-expand (AddEdge folds the wider + // spec into the graph, deep-wins/unfiltered-wins). + newEdges = append(newEdges, curEdge) + case edgeSpecUnchanged: + // nothing to do + } + } + } + + // Changed entitlements are derived from the applied increments (their + // grants' entitlement ids), not supplied by the caller — trust the data. + changedEntitlementIDs, err := c.deriveChangedEntitlementIDs(walkCtx) + if err != nil { + if endErr := c.restoreEndedSync(ctx); endErr != nil { + return false, endErr } + return false, err } - if len(newEdges) == 0 && len(c.incrementalChangedEntitlementIDs) == 0 { + if len(newEdges) == 0 && len(changedEntitlementIDs) == 0 { // Nothing changed relative to the base — its grants were already merged in. return c.finishIncrementalExpansion(ctx) } ie := expand.NewIncrementalExpander(sync.NewExpanderStore(c.compactedC1z), base) - if _, err := ie.ExpandChanges(ctx, newEdges, c.incrementalChangedEntitlementIDs); err != nil { - if errors.Is(err, expand.ErrIncrementalFallback) { - // Restore the ended state and let the full path re-run. - if endErr := c.compactedC1z.EndSync(ctx); endErr != nil { - return false, fmt.Errorf("incremental expansion: end after fallback: %w", endErr) - } - return false, nil + res, err := ie.ExpandChanges(walkCtx, newEdges, changedEntitlementIDs) + if err != nil { + // Restore the ended state so the full path re-runs against a consistent + // store — for the cycle decline and any real error alike. Writes are + // idempotent by grant identity, so partial progress is safe to re-cover. + if endErr := c.restoreEndedSync(ctx); endErr != nil { + return false, endErr } - return false, err + return false, err // sentinel or plain → caller falls back to full } + + ctxzap.Extract(ctx).Info("incremental grant expansion complete", + zap.Int("entitlements_walked", len(res.EntitlementsWalked)), + zap.Int("grants_written", res.GrantsWritten)) return c.finishIncrementalExpansion(ctx) } -// finishIncrementalExpansion ends + closes the store so the file is flushed -// before cpFile copies it (same finalization the full path gets from syncer.Close). +// restoreEndedSync returns the compacted sync to the ended state the full path +// expects, after an incremental attempt that resumed it. Runs on a detached, +// timeout-bounded context so a cancelled parent can't strand the store +// mid-resume. Its failure is FATAL (errIncrementalFatal): the store is in an +// unknown state and the full path must not run against it. +func (c *Compactor) restoreEndedSync(ctx context.Context) error { + endCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), dotc1z.FinalizeTimeout()) + defer cancel() + if err := c.compactedC1z.EndSync(endCtx); err != nil { + return fmt.Errorf("%w: restore ended sync: %w", errIncrementalFatal, err) + } + return nil +} + +// finishIncrementalExpansion ends, cleans up, and closes the store so the file +// is flushed before cpFile copies it — converging with the other compaction +// paths (Cleanup is a Pebble no-op today, kept for parity). Runs on a detached, +// timeout-bounded context so a cancelled or run-duration-expired parent can't +// abort finalization. All errors here are FATAL (errIncrementalFatal): the +// store is being torn down, so falling back to full expansion against it is +// not safe. func (c *Compactor) finishIncrementalExpansion(ctx context.Context) (bool, error) { - if err := c.compactedC1z.EndSync(ctx); err != nil { - return false, fmt.Errorf("incremental expansion: end sync: %w", err) + finalizeCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), dotc1z.FinalizeTimeout()) + defer cancel() + if err := c.compactedC1z.EndSync(finalizeCtx); err != nil { + return false, fmt.Errorf("%w: end sync: %w", errIncrementalFatal, err) + } + if err := c.compactedC1z.Cleanup(finalizeCtx); err != nil { + return false, fmt.Errorf("%w: cleanup: %w", errIncrementalFatal, err) } - if err := c.compactedC1z.Close(ctx); err != nil { - return false, fmt.Errorf("incremental expansion: close: %w", err) + if err := c.compactedC1z.Close(finalizeCtx); err != nil { + return false, fmt.Errorf("%w: close: %w", errIncrementalFatal, err) } return true, nil } -// baseGraphHasEdge reports whether the base graph already has an edge src->dst. -// Endpoints collapsed into one node (a fixed cycle) count as already-present. -func baseGraphHasEdge(g *expand.EntitlementGraph, src, dst string) bool { +// deriveChangedEntitlementIDs returns the distinct entitlement ids touched by +// the applied increments (entries[1:]; entries[0] is the base). These seed the +// incremental walk so a new member on an already-expanded entitlement — which +// adds no edge — still propagates. Derived from the data, not the caller. +func (c *Compactor) deriveChangedEntitlementIDs(ctx context.Context) ([]string, error) { + if len(c.entries) < 2 { + return nil, nil + } + seen := make(map[string]struct{}) + for _, e := range c.entries[1:] { + // Same open options as doOneCompaction: honor the caller's tmp dir + // (extraction must not silently land in os.TempDir()) and parallel decode. + store, err := dotc1z.NewStore(ctx, e.FilePath, + dotc1z.WithTmpDir(c.tmpDir), + dotc1z.WithDecoderOptions(dotc1z.WithDecoderConcurrency(-1)), + dotc1z.WithReadOnly(true), + ) + if err != nil { + return nil, fmt.Errorf("incremental expansion: open increment %s: %w", e.SyncID, err) + } + err = collectGrantEntitlementIDs(ctx, store, e.SyncID, seen) + if closeErr := store.Close(ctx); closeErr != nil && err == nil { + err = fmt.Errorf("incremental expansion: close increment %s: %w", e.SyncID, closeErr) + } + if err != nil { + return nil, err + } + } + out := make([]string, 0, len(seen)) + for id := range seen { + out = append(out, id) + } + sort.Strings(out) + return out, nil +} + +// collectGrantEntitlementIDs adds every entitlement id that has a grant in the +// given sync to seen. +func collectGrantEntitlementIDs(ctx context.Context, store c1zstore.Store, syncID string, seen map[string]struct{}) error { + if err := store.SetCurrentSync(ctx, syncID); err != nil { + return fmt.Errorf("incremental expansion: set increment sync %s: %w", syncID, err) + } + pageToken := "" + for { + resp, err := store.ListGrants(ctx, v2.GrantsServiceListGrantsRequest_builder{ + PageSize: 1000, + PageToken: pageToken, + }.Build()) + if err != nil { + return fmt.Errorf("incremental expansion: list increment grants for %s: %w", syncID, err) + } + for _, g := range resp.GetList() { + if id := g.GetEntitlement().GetId(); id != "" { + seen[id] = struct{}{} + } + } + pageToken = resp.GetNextPageToken() + if pageToken == "" { + return nil + } + } +} + +// baseGraphEdge returns the base graph's edge src->dst and whether one exists. +// Endpoints collapsed into one node (a fixed cycle) count as present with no +// distinct edge (nil), which classifyEdgeSpecChange treats as unchanged. +func baseGraphEdge(g *expand.EntitlementGraph, src, dst string) (*expand.Edge, bool) { sn := g.GetNode(src) dn := g.GetNode(dst) if sn == nil || dn == nil { - return false + return nil, false } if sn.Id == dn.Id { - return true + return nil, true } dests, ok := g.SourcesToDestinations[sn.Id] if !ok { - return false + return nil, false + } + edgeID, ok := dests[dn.Id] + if !ok { + return nil, false + } + e, ok := g.Edges[edgeID] + if !ok { + return nil, false } - _, ok = dests[dn.Id] - return ok + return &e, true +} + +type edgeSpecChange int + +const ( + edgeSpecUnchanged edgeSpecChange = iota + edgeSpecWidened + edgeSpecNarrowed +) + +// classifyEdgeSpecChange compares an existing base edge's spec to the current +// (increment) spec. Narrowing (deep->shallow, filter tightened) is +// revocation-shaped; widening (shallow->deep, filter broadened) needs +// re-expansion. Any narrowing wins (safest: decline to full). +func classifyEdgeSpecChange(base *expand.Edge, cur expand.NewEdge) edgeSpecChange { + if base == nil { + return edgeSpecUnchanged // collapsed cycle: no distinct edge + } + widened, narrowed := false, false + if base.IsShallow && !cur.Shallow { + widened = true // shallow -> deep + } + if !base.IsShallow && cur.Shallow { + narrowed = true // deep -> shallow + } + rw, rn := compareResourceTypeFilter(base.ResourceTypeIDs, cur.ResourceTypeIDs) + widened = widened || rw + narrowed = narrowed || rn + switch { + case narrowed: + return edgeSpecNarrowed + case widened: + return edgeSpecWidened + default: + return edgeSpecUnchanged + } +} + +// compareResourceTypeFilter compares two principal-type filters where an empty +// filter means "all types" (the widest). Returns whether the current filter is +// wider and/or narrower than the base. +func compareResourceTypeFilter(base, cur []string) (widened, narrowed bool) { + baseAll := len(base) == 0 + curAll := len(cur) == 0 + switch { + case baseAll && curAll: + return false, false + case baseAll && !curAll: + return false, true // all -> some + case !baseAll && curAll: + return true, false // some -> all + } + baseSet := make(map[string]struct{}, len(base)) + for _, t := range base { + baseSet[t] = struct{}{} + } + curSet := make(map[string]struct{}, len(cur)) + for _, t := range cur { + curSet[t] = struct{}{} + } + for t := range curSet { + if _, ok := baseSet[t]; !ok { + widened = true + } + } + for t := range baseSet { + if _, ok := curSet[t]; !ok { + narrowed = true + } + } + return widened, narrowed } func (c *Compactor) expandGrants(ctx context.Context, newSyncId string, compactionStart time.Time) error { @@ -686,17 +922,38 @@ func (c *Compactor) expandGrants(ctx context.Context, newSyncId string, compacti // Diff-aware fast path: with a base graph, expand only what changed relative // to it. Any doubt (cycle, error) falls through to full expansion below. - if c.incrementalBaseGraph != nil { - done, err := c.expandGrantsIncremental(ctx, newSyncId) + // Pebble-only: it reopens the ended compacted sync to write grants, which + // only Pebble supports; on other engines we degrade gracefully to full. + switch { + case c.incrementalBaseGraph == nil: + // not requested + case c.resolvedEngine() != c1zstore.EnginePebble: + l.Info("incremental expansion is Pebble-only; using full expansion", + zap.String("engine", string(c.resolvedEngine()))) + default: + done, err := c.expandGrantsIncremental(ctx, newSyncId, compactionStart) switch { + case errors.Is(err, errIncrementalFatal): + // The store's finalization (or restore-to-ended) failed: it is in an + // unknown/torn-down state, so running full expansion against it is + // unsafe. Fail the compaction. + return fmt.Errorf("incremental grant expansion: %w", err) + case errors.Is(err, expand.ErrIncrementalRevocationDecline): + // Named revocation hook (#6): today declines to full; a future + // tombstone stage flips this one site to apply deletions. + l.Info("incremental expansion declined (revocation-shaped change); falling back to full expansion") + case errors.Is(err, expand.ErrIncrementalFallback): + // New edge closed a cycle: full expansion handles cycles correctly. + l.Info("incremental expansion declined (cycle); falling back to full expansion") case err != nil: + // Pre-write or restored-state failure: the store is back in the + // ended state the full path expects, so falling back is safe. l.Warn("incremental expansion failed; falling back to full expansion", zap.Error(err)) case done: // Incremental path already ended + closed the store; caller clears // c.compactedC1z after return, same as the full path. + c.incrementalExpansionRan = true return nil - default: - l.Info("incremental expansion declined (cycle); falling back to full expansion") } } diff --git a/pkg/synccompactor/incremental_expansion_test.go b/pkg/synccompactor/incremental_expansion_test.go index 71b04352e..253ea67d6 100644 --- a/pkg/synccompactor/incremental_expansion_test.go +++ b/pkg/synccompactor/incremental_expansion_test.go @@ -1,22 +1,41 @@ package synccompactor import ( + "bytes" "context" "path/filepath" "sort" + "strconv" + "strings" "testing" + "github.com/cockroachdb/pebble/v2" "github.com/stretchr/testify/require" + reader_v2 "github.com/conductorone/baton-sdk/pb/c1/reader/v2" + v2 "github.com/conductorone/baton-sdk/pb/c1/connector/v2" "github.com/conductorone/baton-sdk/pkg/annotations" "github.com/conductorone/baton-sdk/pkg/connectorstore" "github.com/conductorone/baton-sdk/pkg/dotc1z" "github.com/conductorone/baton-sdk/pkg/dotc1z/c1zstore" + enginepkg "github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble" "github.com/conductorone/baton-sdk/pkg/sync/expand" batonGrant "github.com/conductorone/baton-sdk/pkg/types/grant" ) +// hasGrant asserts some row in outcome starts with the given +// entitlement|type|resource prefix (grantOutcome rows carry a sources suffix). +func hasGrant(t *testing.T, outcome []string, prefix string) { + t.Helper() + for _, k := range outcome { + if strings.HasPrefix(k, prefix+"|") { + return + } + } + t.Fatalf("expected a grant with prefix %q in %v", prefix, outcome) +} + // This test proves the compactor's diff-aware incremental expansion produces // the SAME grants as a full expansion, on real Pebble c1z files. // @@ -63,6 +82,21 @@ func ruleGrant(destEnt *v2.Entitlement, sourceGroup *v2.Resource, sourceEntID st return g } +// ruleGrantSpec is ruleGrant with an explicit shallow flag, for edge-spec +// change tests. +func ruleGrantSpec(destEnt *v2.Entitlement, sourceGroup *v2.Resource, sourceEntID string, shallow bool) *v2.Grant { + g := v2.Grant_builder{ + Id: batonGrant.NewGrantID(sourceGroup, destEnt), + Entitlement: destEnt, + Principal: sourceGroup, + }.Build() + g.SetAnnotations(annotations.New(v2.GrantExpandable_builder{ + EntitlementIds: []string{sourceEntID}, + Shallow: shallow, + }.Build())) + return g +} + func memberGrant(e *v2.Entitlement, principal *v2.Resource) *v2.Grant { return v2.Grant_builder{ Id: batonGrant.NewGrantID(principal, e), @@ -71,10 +105,13 @@ func memberGrant(e *v2.Entitlement, principal *v2.Resource) *v2.Grant { }.Build() } +// expandedGrant builds a grant expanded from a direct membership on sourceEntID +// (IsDirect: true) — matching what a real expansion records for a principal +// that is a direct member of the source. func expandedGrant(e *v2.Entitlement, principal *v2.Resource, sourceEntID string) *v2.Grant { g := memberGrant(e, principal) g.SetSources(v2.GrantSources_builder{ - Sources: map[string]*v2.GrantSources_GrantSource{sourceEntID: {}}, + Sources: map[string]*v2.GrantSources_GrantSource{sourceEntID: {IsDirect: true}}, }.Build()) return g } @@ -82,6 +119,12 @@ func expandedGrant(e *v2.Entitlement, principal *v2.Resource, sourceEntID string // buildIncrementalFixtures writes a base (full, pre-expanded) c1z and an // increment (partial) c1z into dir, returning the compactable entries. func buildIncrementalFixtures(t *testing.T, ctx context.Context, dir string) []*CompactableSync { + return buildIncrementalFixturesEngine(t, ctx, dir, c1zstore.EnginePebble) +} + +// buildIncrementalFixturesEngine is buildIncrementalFixtures with a chosen +// storage engine, so the SQLite degrade path can be exercised too. +func buildIncrementalFixturesEngine(t *testing.T, ctx context.Context, dir string, engine c1zstore.Engine) []*CompactableSync { t.Helper() grpA, grpB, grpC := grp("grpA"), grp("grpB"), grp("grpC") @@ -93,7 +136,7 @@ func buildIncrementalFixtures(t *testing.T, ctx context.Context, dir string) []* // --- base: full, already expanded (ent-b -> ent-c, mandy on both) --- basePath := filepath.Join(dir, "base.c1z") - base, err := dotc1z.NewStore(ctx, basePath, dotc1z.WithEngine(c1zstore.EnginePebble)) + base, err := dotc1z.NewStore(ctx, basePath, dotc1z.WithEngine(engine)) require.NoError(t, err) baseSyncID, err := base.StartNewSync(ctx, connectorstore.SyncTypeFull, "") require.NoError(t, err) @@ -110,7 +153,7 @@ func buildIncrementalFixtures(t *testing.T, ctx context.Context, dir string) []* // --- increment: partial, adds ent-a -> ent-b with sam on A --- incPath := filepath.Join(dir, "inc.c1z") - inc, err := dotc1z.NewStore(ctx, incPath, dotc1z.WithEngine(c1zstore.EnginePebble)) + inc, err := dotc1z.NewStore(ctx, incPath, dotc1z.WithEngine(engine)) require.NoError(t, err) incSyncID, err := inc.StartNewSync(ctx, connectorstore.SyncTypePartial, "") require.NoError(t, err) @@ -144,7 +187,9 @@ func baseGraphForFixtures(t *testing.T, ctx context.Context) *expand.Entitlement } // grantOutcome reads every grant from a compacted c1z and returns the set of -// "entitlement|principalType|principalResource" keys. +// full-row keys "entitlement|principalType|principalResource|sources=..." — +// INCLUDING the sources/provenance map, so the differential also pins that +// incremental produces the same provenance as full expansion. func grantOutcome(t *testing.T, ctx context.Context, path, syncID string) []string { t.Helper() store, err := dotc1z.NewStore(ctx, path, dotc1z.WithReadOnly(true)) @@ -162,7 +207,13 @@ func grantOutcome(t *testing.T, ctx context.Context, path, syncID string) []stri require.NoError(t, err) for _, g := range resp.GetList() { pid := g.GetPrincipal().GetId() - set[g.GetEntitlement().GetId()+"|"+pid.GetResourceType()+"|"+pid.GetResource()] = struct{}{} + srcs := g.GetSources().GetSources() + parts := make([]string, 0, len(srcs)) + for id, s := range srcs { + parts = append(parts, id+":"+strconv.FormatBool(s.GetIsDirect())) + } + sort.Strings(parts) + set[g.GetEntitlement().GetId()+"|"+pid.GetResourceType()+"|"+pid.GetResource()+"|sources="+strings.Join(parts, ",")] = struct{}{} } pageToken = resp.GetNextPageToken() if pageToken == "" { @@ -186,13 +237,14 @@ func TestCompactor_IncrementalExpansionMatchesFull(t *testing.T) { cInc, cleanupInc, err := NewCompactor(ctx, t.TempDir(), incEntries, WithTmpDir(t.TempDir()), WithEngine(c1zstore.EnginePebble), - WithIncrementalExpansion(baseGraphForFixtures(t, ctx), nil), + WithIncrementalExpansion(baseGraphForFixtures(t, ctx)), ) require.NoError(t, err) defer func() { _ = cleanupInc() }() incOut, err := cInc.Compact(ctx) require.NoError(t, err) require.NotNil(t, incOut) + require.True(t, cInc.incrementalExpansionRan, "incremental path must have run, not fallen back to full") // --- Path B: full expansion (no base graph) --- fullDir := t.TempDir() @@ -213,9 +265,9 @@ func TestCompactor_IncrementalExpansionMatchesFull(t *testing.T) { require.Equal(t, fullGrants, incGrants, "incremental expansion must equal full expansion") // sam (from A) must have reached B and C via the cascade. - require.Contains(t, incGrants, "ent-b|user|sam") - require.Contains(t, incGrants, "ent-c|user|sam") - require.Contains(t, incGrants, "ent-c|user|mandy") + hasGrant(t, incGrants, "ent-b|user|sam") + hasGrant(t, incGrants, "ent-c|user|sam") + hasGrant(t, incGrants, "ent-c|user|mandy") } // buildNewMemberFixtures writes a base (full, pre-expanded ent-b -> ent-c) c1z @@ -282,13 +334,14 @@ func TestCompactor_IncrementalNewMemberMatchesFull(t *testing.T) { cInc, cleanupInc, err := NewCompactor(ctx, t.TempDir(), incEntries, WithTmpDir(t.TempDir()), WithEngine(c1zstore.EnginePebble), - WithIncrementalExpansion(baseGraphForFixtures(t, ctx), []string{"ent-b"}), + WithIncrementalExpansion(baseGraphForFixtures(t, ctx)), ) require.NoError(t, err) defer func() { _ = cleanupInc() }() incOut, err := cInc.Compact(ctx) require.NoError(t, err) require.NotNil(t, incOut) + require.True(t, cInc.incrementalExpansionRan, "incremental path must have run, not fallen back to full") // Path B: full expansion (no base graph). fullEntries := buildNewMemberFixtures(t, ctx, t.TempDir()) @@ -307,7 +360,317 @@ func TestCompactor_IncrementalNewMemberMatchesFull(t *testing.T) { require.Equal(t, fullGrants, incGrants, "incremental (new member) must equal full expansion") // The blocker: bob (new member of B) must have reached C. - require.Contains(t, incGrants, "ent-b|user|bob") - require.Contains(t, incGrants, "ent-c|user|bob") - require.Contains(t, incGrants, "ent-c|user|mandy") + hasGrant(t, incGrants, "ent-b|user|bob") + hasGrant(t, incGrants, "ent-c|user|bob") + hasGrant(t, incGrants, "ent-c|user|mandy") +} + +// buildSpecChangeFixtures builds a base with a B->C rule at baseShallow, plus +// mandy (direct on B) and bob (indirect on B), pre-expanded per the base spec; +// and an increment that overwrites the B->C rule to incShallow (same grant id). +func buildSpecChangeFixtures(t *testing.T, ctx context.Context, dir string, baseShallow, incShallow bool) []*CompactableSync { + t.Helper() + grpB, grpC := grp("grpB"), grp("grpC") + mandy, bob := usr("mandy"), usr("bob") + entB, entC := ent("ent-b", grpB), ent("ent-c", grpC) + userRT := v2.ResourceType_builder{Id: "user", DisplayName: "User"}.Build() + groupRT := v2.ResourceType_builder{Id: "group", DisplayName: "Group"}.Build() + + basePath := filepath.Join(dir, "base.c1z") + base, err := dotc1z.NewStore(ctx, basePath, dotc1z.WithEngine(c1zstore.EnginePebble)) + require.NoError(t, err) + baseSyncID, err := base.StartNewSync(ctx, connectorstore.SyncTypeFull, "") + require.NoError(t, err) + require.NoError(t, base.PutResourceTypes(ctx, userRT, groupRT)) + require.NoError(t, base.PutResources(ctx, grpB, grpC, mandy, bob)) + require.NoError(t, base.PutEntitlements(ctx, entB, entC)) + baseGrants := []*v2.Grant{ + memberGrant(entB, mandy), // mandy: direct member of B + expandedGrant(entB, bob, "ent-x"), // bob: indirect on B (source is not B) + ruleGrantSpec(entC, grpB, "ent-b", baseShallow), + expandedGrant(entC, mandy, "ent-b"), // mandy on C (direct qualifies either way) + } + if !baseShallow { + baseGrants = append(baseGrants, expandedGrant(entC, bob, "ent-b")) // bob on C only when deep + } + require.NoError(t, base.PutGrants(ctx, baseGrants...)) + require.NoError(t, base.EndSync(ctx)) + require.NoError(t, base.Close(ctx)) + + incPath := filepath.Join(dir, "inc.c1z") + inc, err := dotc1z.NewStore(ctx, incPath, dotc1z.WithEngine(c1zstore.EnginePebble)) + require.NoError(t, err) + incSyncID, err := inc.StartNewSync(ctx, connectorstore.SyncTypePartial, "") + require.NoError(t, err) + require.NoError(t, inc.PutResourceTypes(ctx, userRT, groupRT)) + require.NoError(t, inc.PutResources(ctx, grpB, grpC)) + require.NoError(t, inc.PutEntitlements(ctx, entB, entC)) + require.NoError(t, inc.PutGrants(ctx, ruleGrantSpec(entC, grpB, "ent-b", incShallow))) + require.NoError(t, inc.EndSync(ctx)) + require.NoError(t, inc.Close(ctx)) + + return []*CompactableSync{ + {FilePath: basePath, SyncID: baseSyncID}, + {FilePath: incPath, SyncID: incSyncID}, + } +} + +func specChangeBaseGraph(t *testing.T, ctx context.Context, shallow bool) *expand.EntitlementGraph { + t.Helper() + g := expand.NewEntitlementGraph(ctx) + g.AddEntitlementID("ent-b") + g.AddEntitlementID("ent-c") + require.NoError(t, g.AddEdge(ctx, "ent-b", "ent-c", shallow, nil)) + g.MarkEdgeExpanded("ent-b", "ent-c") + return g +} + +// TestCompactor_IncrementalWidenedEdgeReExpands (C3): an increment that widens +// an existing edge (shallow -> deep) must re-expand it — the previously-excluded +// indirect member now propagates — and match full expansion. +func TestCompactor_IncrementalWidenedEdgeReExpands(t *testing.T) { + ctx := context.Background() + + incEntries := buildSpecChangeFixtures(t, ctx, t.TempDir(), true, false) // shallow -> deep + cInc, cleanupInc, err := NewCompactor(ctx, t.TempDir(), incEntries, + WithTmpDir(t.TempDir()), + WithEngine(c1zstore.EnginePebble), + WithIncrementalExpansion(specChangeBaseGraph(t, ctx, true)), // base edge is shallow + ) + require.NoError(t, err) + defer func() { _ = cleanupInc() }() + incOut, err := cInc.Compact(ctx) + require.NoError(t, err) + require.True(t, cInc.incrementalExpansionRan, "widened edge must re-expand, not fall back") + + fullEntries := buildSpecChangeFixtures(t, ctx, t.TempDir(), true, false) + cFull, cleanupFull, err := NewCompactor(ctx, t.TempDir(), fullEntries, + WithTmpDir(t.TempDir()), WithEngine(c1zstore.EnginePebble)) + require.NoError(t, err) + defer func() { _ = cleanupFull() }() + fullOut, err := cFull.Compact(ctx) + require.NoError(t, err) + + incGrants := grantOutcome(t, ctx, incOut.FilePath, incOut.SyncID) + fullGrants := grantOutcome(t, ctx, fullOut.FilePath, fullOut.SyncID) + require.Equal(t, fullGrants, incGrants, "widened re-expansion must equal full") + hasGrant(t, incGrants, "ent-c|user|bob") // bob now qualifies (deep) +} + +// TestCompactor_IncrementalNarrowedEdgeDeclines (C3/#6/#11c): an increment that +// narrows an existing edge (deep -> shallow) is revocation-shaped; incremental +// must decline (via the named branch) and fall back to full. Pins today's +// behavior: incremental-with-fallback == full. The future deletion stage turns +// this red then green. +func TestCompactor_IncrementalNarrowedEdgeDeclines(t *testing.T) { + ctx := context.Background() + + incEntries := buildSpecChangeFixtures(t, ctx, t.TempDir(), false, true) // deep -> shallow + cInc, cleanupInc, err := NewCompactor(ctx, t.TempDir(), incEntries, + WithTmpDir(t.TempDir()), + WithEngine(c1zstore.EnginePebble), + WithIncrementalExpansion(specChangeBaseGraph(t, ctx, false)), // base edge is deep + ) + require.NoError(t, err) + defer func() { _ = cleanupInc() }() + incOut, err := cInc.Compact(ctx) + require.NoError(t, err) + require.False(t, cInc.incrementalExpansionRan, "narrowed edge must decline to full expansion") + + fullEntries := buildSpecChangeFixtures(t, ctx, t.TempDir(), false, true) + cFull, cleanupFull, err := NewCompactor(ctx, t.TempDir(), fullEntries, + WithTmpDir(t.TempDir()), WithEngine(c1zstore.EnginePebble)) + require.NoError(t, err) + defer func() { _ = cleanupFull() }() + fullOut, err := cFull.Compact(ctx) + require.NoError(t, err) + + incGrants := grantOutcome(t, ctx, incOut.FilePath, incOut.SyncID) + fullGrants := grantOutcome(t, ctx, fullOut.FilePath, fullOut.SyncID) + require.Equal(t, fullGrants, incGrants, "declined incremental must equal full expansion") +} + +// TestCompactor_IncrementalDoesNotMutateBaseGraph (U1): running the incremental +// expansion must not mutate the caller-held base graph, so a retry with the same +// graph can't treat never-expanded edges as already present. +func TestCompactor_IncrementalDoesNotMutateBaseGraph(t *testing.T) { + ctx := context.Background() + + entries := buildIncrementalFixtures(t, ctx, t.TempDir()) // increment adds ent-a -> ent-b + base := baseGraphForFixtures(t, ctx) // holds only ent-b -> ent-c + edgesBefore := len(base.Edges) + nodesBefore := len(base.Nodes) + + c, cleanup, err := NewCompactor(ctx, t.TempDir(), entries, + WithTmpDir(t.TempDir()), + WithEngine(c1zstore.EnginePebble), + WithIncrementalExpansion(base), + ) + require.NoError(t, err) + defer func() { _ = cleanup() }() + + _, err = c.Compact(ctx) + require.NoError(t, err) + require.True(t, c.incrementalExpansionRan) + + // The caller's graph is untouched — the new ent-a -> ent-b edge went into a + // clone, not this graph. + require.Equal(t, edgesBefore, len(base.Edges), "base graph edges must be unchanged") + require.Equal(t, nodesBefore, len(base.Nodes), "base graph nodes must be unchanged") + require.Nil(t, base.GetNode("ent-a"), "new edge's node must not leak into the caller's graph") +} + +// buildDanglingRefFixtures builds a base (ent-b -> ent-c, mandy) and a single +// increment whose only change is a rule grant whose SOURCE entitlement +// ("ent-ghost") is absent from the merged set — a dangling ref, which both +// paths must skip. +func buildDanglingRefFixtures(t *testing.T, ctx context.Context, dir string) []*CompactableSync { + t.Helper() + grpB, grpC := grp("grpB"), grp("grpC") + mandy := usr("mandy") + entB, entC := ent("ent-b", grpB), ent("ent-c", grpC) + userRT := v2.ResourceType_builder{Id: "user", DisplayName: "User"}.Build() + groupRT := v2.ResourceType_builder{Id: "group", DisplayName: "Group"}.Build() + + basePath := filepath.Join(dir, "base.c1z") + base, err := dotc1z.NewStore(ctx, basePath, dotc1z.WithEngine(c1zstore.EnginePebble)) + require.NoError(t, err) + baseSyncID, err := base.StartNewSync(ctx, connectorstore.SyncTypeFull, "") + require.NoError(t, err) + require.NoError(t, base.PutResourceTypes(ctx, userRT, groupRT)) + require.NoError(t, base.PutResources(ctx, grpB, grpC, mandy)) + require.NoError(t, base.PutEntitlements(ctx, entB, entC)) + require.NoError(t, base.PutGrants(ctx, + memberGrant(entB, mandy), + expandedGrant(entC, mandy, "ent-b"), + ruleGrant(entC, grpB, "ent-b"), + )) + require.NoError(t, base.EndSync(ctx)) + require.NoError(t, base.Close(ctx)) + + incPath := filepath.Join(dir, "inc.c1z") + inc, err := dotc1z.NewStore(ctx, incPath, dotc1z.WithEngine(c1zstore.EnginePebble)) + require.NoError(t, err) + incSyncID, err := inc.StartNewSync(ctx, connectorstore.SyncTypePartial, "") + require.NoError(t, err) + require.NoError(t, inc.PutResourceTypes(ctx, userRT, groupRT)) + require.NoError(t, inc.PutResources(ctx, grpB, grpC)) + require.NoError(t, inc.PutEntitlements(ctx, entC)) + // rule grant: members of ent-ghost (which does not exist) get ent-c. + require.NoError(t, inc.PutGrants(ctx, ruleGrant(entC, grpB, "ent-ghost"))) + require.NoError(t, inc.EndSync(ctx)) + require.NoError(t, inc.Close(ctx)) + + return []*CompactableSync{ + {FilePath: basePath, SyncID: baseSyncID}, + {FilePath: incPath, SyncID: incSyncID}, + } +} + +// TestCompactor_IncrementalDanglingRefMatchesFull (#11a): an increment with a +// grant referencing an entitlement absent from the merged set is skipped by +// both paths; incremental (skip-with-warn) must equal full (NotFound skip). +func TestCompactor_IncrementalDanglingRefMatchesFull(t *testing.T) { + ctx := context.Background() + + incEntries := buildDanglingRefFixtures(t, ctx, t.TempDir()) + cInc, cleanupInc, err := NewCompactor(ctx, t.TempDir(), incEntries, + WithTmpDir(t.TempDir()), + WithEngine(c1zstore.EnginePebble), + WithIncrementalExpansion(baseGraphForFixtures(t, ctx)), + ) + require.NoError(t, err) + defer func() { _ = cleanupInc() }() + incOut, err := cInc.Compact(ctx) + require.NoError(t, err) + require.True(t, cInc.incrementalExpansionRan, "dangling ref must skip, not fall back") + + fullEntries := buildDanglingRefFixtures(t, ctx, t.TempDir()) + cFull, cleanupFull, err := NewCompactor(ctx, t.TempDir(), fullEntries, + WithTmpDir(t.TempDir()), WithEngine(c1zstore.EnginePebble)) + require.NoError(t, err) + defer func() { _ = cleanupFull() }() + fullOut, err := cFull.Compact(ctx) + require.NoError(t, err) + + incGrants := grantOutcome(t, ctx, incOut.FilePath, incOut.SyncID) + fullGrants := grantOutcome(t, ctx, fullOut.FilePath, fullOut.SyncID) + require.Equal(t, fullGrants, incGrants, "dangling-ref incremental must equal full") +} + +// TestCompactor_IncrementalSealedArtifactLifecycle (#11b): after the +// incremental end→resume→write→end sequence, the reopened artifact's sync must +// be sealed (finished) and the Pebble by_principal index must cover the +// incrementally-written grants (rebuilt at the final EndSync). +func TestCompactor_IncrementalSealedArtifactLifecycle(t *testing.T) { + ctx := context.Background() + + entries := buildIncrementalFixtures(t, ctx, t.TempDir()) // increment brings sam + c, cleanup, err := NewCompactor(ctx, t.TempDir(), entries, + WithTmpDir(t.TempDir()), + WithEngine(c1zstore.EnginePebble), + WithIncrementalExpansion(baseGraphForFixtures(t, ctx)), + ) + require.NoError(t, err) + defer func() { _ = cleanup() }() + out, err := c.Compact(ctx) + require.NoError(t, err) + require.True(t, c.incrementalExpansionRan) + + store, err := dotc1z.NewStore(ctx, out.FilePath, dotc1z.WithReadOnly(true)) + require.NoError(t, err) + defer store.Close(ctx) + + // (1) Sealed: the compacted sync is finished. + fin, err := store.GetLatestFinishedSync(ctx, reader_v2.SyncsReaderServiceGetLatestFinishedSyncRequest_builder{}.Build()) + require.NoError(t, err) + require.Equal(t, out.SyncID, fin.GetSync().GetId(), "compacted sync must be sealed/finished") + + // (2) by_principal index is populated and covers sam (written incrementally). + eng, ok := enginepkg.AsEngine(store) + require.True(t, ok, "expected a pebble engine") + it, err := eng.DB().NewIter(&pebble.IterOptions{ + LowerBound: enginepkg.GrantByPrincipalLowerBound(), + UpperBound: enginepkg.GrantByPrincipalUpperBound(), + }) + require.NoError(t, err) + defer it.Close() + total, sawSam := 0, false + for it.First(); it.Valid(); it.Next() { + total++ + if bytes.Contains(it.Key(), []byte("sam")) { + sawSam = true + } + } + require.NoError(t, it.Error()) + require.Positive(t, total, "by_principal index must have entries") + require.True(t, sawSam, "by_principal index must cover sam's incrementally-written grants") +} + +// TestCompactor_IncrementalDegradesGracefullyOnSQLite: the fast path is +// Pebble-only (it reopens an ended sync, which SQLite refuses). On a SQLite +// output, requesting incremental must degrade to full expansion — no error, +// correct grants, and incrementalExpansionRan must be false. +func TestCompactor_IncrementalDegradesGracefullyOnSQLite(t *testing.T) { + ctx := context.Background() + + entries := buildIncrementalFixturesEngine(t, ctx, t.TempDir(), c1zstore.EngineSQLite) + // No WithEngine → engine inferred from the SQLite inputs. + c, cleanup, err := NewCompactor(ctx, t.TempDir(), entries, + WithTmpDir(t.TempDir()), + WithIncrementalExpansion(baseGraphForFixtures(t, ctx)), + ) + require.NoError(t, err) + defer func() { _ = cleanup() }() + + out, err := c.Compact(ctx) + require.NoError(t, err, "SQLite must degrade gracefully, not error") + require.NotNil(t, out) + require.False(t, c.incrementalExpansionRan, "SQLite must fall back to full expansion") + + // Grants are still correct (produced by full expansion). + grants := grantOutcome(t, ctx, out.FilePath, out.SyncID) + hasGrant(t, grants, "ent-b|user|sam") + hasGrant(t, grants, "ent-c|user|sam") + hasGrant(t, grants, "ent-c|user|mandy") } From 282951c2bd67f621bc13f6c47d29cd0506ec9500 Mon Sep 17 00:00:00 2001 From: manojacs Date: Sat, 18 Jul 2026 01:36:04 +0000 Subject: [PATCH 03/15] feat(compactor): derive changed entitlements during the Pebble fold The fold already reads every applied increment record; collect the distinct grant entitlement ids there (FoldStats.GrantEntitlementIDs) and seed incremental expansion from that set instead of re-opening and re-scanning each increment c1z. No-op resubmissions (byte-identical or older-than-incumbent records) change nothing and are excluded, so the walk gets fewer wasted seeds. Rebuild-mode compactions (no fold) keep the re-read fallback. Co-authored-by: c1-squire-dev[bot] Co-Authored-By: Claude Opus 4.8 --- pkg/synccompactor/compactor.go | 30 +++++-- pkg/synccompactor/compactor_pebble.go | 7 ++ .../incremental_expansion_test.go | 78 +++++++++++++++++++ pkg/synccompactor/pebble/merge.go | 29 +++++++ pkg/synccompactor/pebble/merge_test.go | 61 +++++++++++++++ 5 files changed, 199 insertions(+), 6 deletions(-) diff --git a/pkg/synccompactor/compactor.go b/pkg/synccompactor/compactor.go index 4c710c097..7a6bcf7b5 100644 --- a/pkg/synccompactor/compactor.go +++ b/pkg/synccompactor/compactor.go @@ -56,6 +56,9 @@ type Compactor struct { // handled expansion (vs falling back to full). Read by tests to prove the // fast path ran rather than silently falling back. incrementalExpansionRan bool + // foldChangedEntitlementIDs: changed-entitlement set collected by the + // Pebble fold; nil when no fold ran (derive fallback). + foldChangedEntitlementIDs map[string]struct{} // engine selects the storage engine for the compacted output. // Empty means EngineSQLite (the default; behavior is unchanged and // the output is byte-identical to the pre-engine-option compactor). @@ -689,7 +692,7 @@ func (c *Compactor) expandGrantsIncremental(ctx context.Context, newSyncId strin // Changed entitlements are derived from the applied increments (their // grants' entitlement ids), not supplied by the caller — trust the data. - changedEntitlementIDs, err := c.deriveChangedEntitlementIDs(walkCtx) + changedEntitlementIDs, err := c.changedEntitlementIDs(walkCtx) if err != nil { if endErr := c.restoreEndedSync(ctx); endErr != nil { return false, endErr @@ -756,10 +759,24 @@ func (c *Compactor) finishIncrementalExpansion(ctx context.Context) (bool, error return true, nil } -// deriveChangedEntitlementIDs returns the distinct entitlement ids touched by -// the applied increments (entries[1:]; entries[0] is the base). These seed the -// incremental walk so a new member on an already-expanded entitlement — which -// adds no edge — still propagates. Derived from the data, not the caller. +// changedEntitlementIDs returns the entitlement ids whose grants changed in +// the applied increments, seeding the incremental walk. The fold collects +// them during its merge (no re-read, no-ops excluded); rebuild-mode +// compactions fall back to deriveChangedEntitlementIDs. +func (c *Compactor) changedEntitlementIDs(ctx context.Context) ([]string, error) { + if c.foldChangedEntitlementIDs != nil { + out := make([]string, 0, len(c.foldChangedEntitlementIDs)) + for id := range c.foldChangedEntitlementIDs { + out = append(out, id) + } + sort.Strings(out) + return out, nil + } + return c.deriveChangedEntitlementIDs(ctx) +} + +// deriveChangedEntitlementIDs is the no-fold fallback: re-open each increment +// (entries[1:]) and collect its grants' entitlement ids. func (c *Compactor) deriveChangedEntitlementIDs(ctx context.Context) ([]string, error) { if len(c.entries) < 2 { return nil, nil @@ -885,7 +902,8 @@ func classifyEdgeSpecChange(base *expand.Edge, cur expand.NewEdge) edgeSpecChang // compareResourceTypeFilter compares two principal-type filters where an empty // filter means "all types" (the widest). Returns whether the current filter is // wider and/or narrower than the base. -func compareResourceTypeFilter(base, cur []string) (widened, narrowed bool) { +func compareResourceTypeFilter(base, cur []string) (bool, bool) { + var widened, narrowed bool baseAll := len(base) == 0 curAll := len(cur) == 0 switch { diff --git a/pkg/synccompactor/compactor_pebble.go b/pkg/synccompactor/compactor_pebble.go index c0ee84083..0481f37a1 100644 --- a/pkg/synccompactor/compactor_pebble.go +++ b/pkg/synccompactor/compactor_pebble.go @@ -559,6 +559,13 @@ func (c *Compactor) compactPebbleFold(ctx context.Context) (string, error) { } } + // Hand the fold's changed-entitlement set to incremental expansion. + // Non-nil even when empty: nil means "no fold ran" (derive fallback). + c.foldChangedEntitlementIDs = foldStats.GrantEntitlementIDs + if c.foldChangedEntitlementIDs == nil { + c.foldChangedEntitlementIDs = map[string]struct{}{} + } + // Record the bytes this fold shadowed in the base keyspace. The // store inherited the base manifest's running fold_dead_bytes at // open (the dest is a byte copy of the base), so adding the delta diff --git a/pkg/synccompactor/incremental_expansion_test.go b/pkg/synccompactor/incremental_expansion_test.go index 253ea67d6..f94664f44 100644 --- a/pkg/synccompactor/incremental_expansion_test.go +++ b/pkg/synccompactor/incremental_expansion_test.go @@ -674,3 +674,81 @@ func TestCompactor_IncrementalDegradesGracefullyOnSQLite(t *testing.T) { hasGrant(t, grants, "ent-c|user|sam") hasGrant(t, grants, "ent-c|user|mandy") } + +// TestCompactor_IncrementalNewMemberFoldCollectsChangedEnts: fold mode +// collects the changed-entitlement set during the merge (no re-read) and +// still matches full expansion. +func TestCompactor_IncrementalNewMemberFoldCollectsChangedEnts(t *testing.T) { + ctx := context.Background() + + incEntries := buildNewMemberFixtures(t, ctx, t.TempDir()) + cInc, cleanupInc, err := NewCompactor(ctx, t.TempDir(), incEntries, + WithTmpDir(t.TempDir()), + WithEngine(c1zstore.EnginePebble), + WithPebbleCompactorMode(PebbleCompactorModeFold), + WithIncrementalExpansion(baseGraphForFixtures(t, ctx)), + ) + require.NoError(t, err) + defer func() { _ = cleanupInc() }() + incOut, err := cInc.Compact(ctx) + require.NoError(t, err) + require.NotNil(t, incOut) + require.True(t, cInc.incrementalExpansionRan, "incremental path must have run, not fallen back to full") + + require.NotNil(t, cInc.foldChangedEntitlementIDs, "fold mode must hand its collected set to expansion") + require.Contains(t, cInc.foldChangedEntitlementIDs, "ent-b", + "bob's new membership grant on ent-b was applied by the fold") + + fullEntries := buildNewMemberFixtures(t, ctx, t.TempDir()) + cFull, cleanupFull, err := NewCompactor(ctx, t.TempDir(), fullEntries, + WithTmpDir(t.TempDir()), + WithEngine(c1zstore.EnginePebble), + WithPebbleCompactorMode(PebbleCompactorModeFold), + ) + require.NoError(t, err) + defer func() { _ = cleanupFull() }() + fullOut, err := cFull.Compact(ctx) + require.NoError(t, err) + + incGrants := grantOutcome(t, ctx, incOut.FilePath, incOut.SyncID) + fullGrants := grantOutcome(t, ctx, fullOut.FilePath, fullOut.SyncID) + require.Equal(t, fullGrants, incGrants, "fold-collected incremental must equal full expansion") + hasGrant(t, incGrants, "ent-c|user|bob") +} + +// TestCompactor_IncrementalNewMemberRebuildFallsBackToDerive: no fold -> +// derive fallback; the fast path still runs and matches full. +func TestCompactor_IncrementalNewMemberRebuildFallsBackToDerive(t *testing.T) { + ctx := context.Background() + + incEntries := buildNewMemberFixtures(t, ctx, t.TempDir()) + cInc, cleanupInc, err := NewCompactor(ctx, t.TempDir(), incEntries, + WithTmpDir(t.TempDir()), + WithEngine(c1zstore.EnginePebble), + WithPebbleCompactorMode(PebbleCompactorModeOverlay), + WithIncrementalExpansion(baseGraphForFixtures(t, ctx)), + ) + require.NoError(t, err) + defer func() { _ = cleanupInc() }() + incOut, err := cInc.Compact(ctx) + require.NoError(t, err) + require.NotNil(t, incOut) + require.True(t, cInc.incrementalExpansionRan, "incremental path must have run, not fallen back to full") + require.Nil(t, cInc.foldChangedEntitlementIDs, "no fold ran, so the derive fallback must have been used") + + fullEntries := buildNewMemberFixtures(t, ctx, t.TempDir()) + cFull, cleanupFull, err := NewCompactor(ctx, t.TempDir(), fullEntries, + WithTmpDir(t.TempDir()), + WithEngine(c1zstore.EnginePebble), + WithPebbleCompactorMode(PebbleCompactorModeOverlay), + ) + require.NoError(t, err) + defer func() { _ = cleanupFull() }() + fullOut, err := cFull.Compact(ctx) + require.NoError(t, err) + + incGrants := grantOutcome(t, ctx, incOut.FilePath, incOut.SyncID) + fullGrants := grantOutcome(t, ctx, fullOut.FilePath, fullOut.SyncID) + require.Equal(t, fullGrants, incGrants, "derive-fallback incremental must equal full expansion") + hasGrant(t, incGrants, "ent-c|user|bob") +} diff --git a/pkg/synccompactor/pebble/merge.go b/pkg/synccompactor/pebble/merge.go index 627ee9054..8ef3ae444 100644 --- a/pkg/synccompactor/pebble/merge.go +++ b/pkg/synccompactor/pebble/merge.go @@ -56,6 +56,9 @@ type FoldStats struct { // (Engine.InvalidateGrantDigestPartitions + // Engine.RepairMissingGrantDigests), instead of the whole file. TouchedGrantPartitions map[string]struct{} + // GrantEntitlementIDs: distinct entitlement ids of applied grant records + // (no-ops excluded). Seeds incremental expansion without re-reading inputs. + GrantEntitlementIDs map[string]struct{} } func (s *FoldStats) Add(o FoldStats) { @@ -74,6 +77,24 @@ func (s *FoldStats) Add(o FoldStats) { } s.TouchedGrantPartitions[p] = struct{}{} } + for id := range o.GrantEntitlementIDs { + s.noteGrantEntitlementID([]byte(id)) + } +} + +// noteGrantEntitlementID records one applied grant's entitlement id; +// read-before-insert keeps repeats allocation-free. +func (s *FoldStats) noteGrantEntitlementID(id []byte) { + if len(id) == 0 { + return + } + if _, ok := s.GrantEntitlementIDs[string(id)]; ok { + return + } + if s.GrantEntitlementIDs == nil { + s.GrantEntitlementIDs = make(map[string]struct{}) + } + s.GrantEntitlementIDs[string(id)] = struct{}{} } func (s *FoldStats) bumpAdded(bucket string, n int64) { @@ -287,6 +308,9 @@ func mergeBucketRawIfNewer(ctx context.Context, dest *enginepkg.Engine, src *eng if err := batch.Set(key, value); err != nil { return stats, err } + // Applied grant (skips continued above): count it toward the + // digest-repair signal and collect its entitlement id for + // incremental expansion — both during the read the fold already does. if bucket.id == runBucketGrants { stats.GrantWrites++ if partition, ok := enginepkg.GrantPartitionFromPrimaryKey(key); ok { @@ -295,6 +319,11 @@ func mergeBucketRawIfNewer(ctx context.Context, dest *enginepkg.Engine, src *eng } stats.TouchedGrantPartitions[partition] = struct{}{} } + _, _, entID, _, _, _, scanErr := scanGrantIndexFieldsBytes(value) + if scanErr != nil { + return stats, scanErr + } + stats.noteGrantEntitlementID(entID) } if err := forEachIndexKeyFromRaw(bucket, key, lower, value, &scratch, nil, setIndexKey); err != nil { return stats, err diff --git a/pkg/synccompactor/pebble/merge_test.go b/pkg/synccompactor/pebble/merge_test.go index 8e4acaf99..13fb28b48 100644 --- a/pkg/synccompactor/pebble/merge_test.go +++ b/pkg/synccompactor/pebble/merge_test.go @@ -347,3 +347,64 @@ func TestMergeIntoDeadBytesExactCount(t *testing.T) { "dead bytes must equal the incumbent's primary key+value plus its index keys") assertIndexesMatchDerived(t, ctx, dest) } + +// grantEnt builds a grant on a specific entitlement id. +func grantEnt(externalID, entID, principal string, at time.Time) *v3.GrantRecord { + return v3.GrantRecord_builder{ + ExternalId: externalID, + Entitlement: v3.EntitlementRef_builder{ + ResourceTypeId: "app", ResourceId: "github", EntitlementId: entID, + }.Build(), + Principal: v3.PrincipalRef_builder{ResourceTypeId: "user", ResourceId: principal}.Build(), + DiscoveredAt: timestamppb.New(at), + }.Build() +} + +// TestMergeIntoCollectsGrantEntitlementIDs: applied records (added/replacing) +// contribute their entitlement id; no-ops (identical/older) don't. +func TestMergeIntoCollectsGrantEntitlementIDs(t *testing.T) { + ctx := context.Background() + older := time.Unix(1000, 0).UTC() + newer := time.Unix(2000, 0).UTC() + + // Base (= dest, fold semantics) holds three incumbents. + identical := grantEnt("g-same", "ent-same", "alice", older) + dest, destSync := seedBase(t, ctx, "entids-dest", recordSet{ + gs: []*v3.GrantRecord{ + identical, // resubmitted byte-identical + grantEnt("g-stale", "ent-stale", "bob", newer), // src's copy is older + grantEnt("g-repl", "ent-replaced", "carol", older), // src's copy is newer + }, + }) + + src := buildEngineSource(t, ctx, "entids-src", recordSet{ + gs: []*v3.GrantRecord{ + identical, // byte-identical no-op -> NOT collected + grantEnt("g-stale", "ent-stale", "bob", older), // loses to incumbent -> NOT collected + grantEnt("g-repl", "ent-replaced", "carol", newer), // wins -> collected + grantEnt("g-new", "ent-added", "dave", newer), // no incumbent -> collected + }, + }) + + stats, err := MergeInto(ctx, dest, []SourceSync{src}, destSync) + require.NoError(t, err) + + got := make([]string, 0, len(stats.GrantEntitlementIDs)) + for id := range stats.GrantEntitlementIDs { + got = append(got, id) + } + require.ElementsMatch(t, []string{"ent-replaced", "ent-added"}, got, + "only applied records contribute; identical/older no-ops must not") +} + +// TestFoldStatsAddUnionsGrantEntitlementIDs: Add unions the sets. +func TestFoldStatsAddUnionsGrantEntitlementIDs(t *testing.T) { + var total FoldStats + total.Add(FoldStats{GrantEntitlementIDs: map[string]struct{}{"a": {}, "b": {}}}) + total.Add(FoldStats{GrantEntitlementIDs: map[string]struct{}{"b": {}, "c": {}}}) + total.Add(FoldStats{}) // nil map: no-op + require.Len(t, total.GrantEntitlementIDs, 3) + for _, id := range []string{"a", "b", "c"} { + require.Contains(t, total.GrantEntitlementIDs, id) + } +} From 7367e2a5eaac7d90d2a6048a9c1074388c4022b9 Mon Sep 17 00:00:00 2001 From: manojacs Date: Sat, 18 Jul 2026 01:36:55 +0000 Subject: [PATCH 04/15] feat(sync): persist the preserved entitlement graph in the c1z, not the token A preserved graph costs ~170-190 bytes/node in the sync token (~10MB at 50k entitlements, measured in TestGraphBlobSizeAtScale), and tokens travel through workflow state. Store it as a Pebble engine-meta sidecar (same single-key shape as the stats sidecar) instead: - WithPreserveEntitlementGraph writes the sidecar when the store supports it and keeps the token skinny; SQLite (or a failed sidecar write) keeps the graph in the token as before. - sync.GraphFromStore(ctx, store, syncID) loads it; a sync-id guard in the blob rejects a stale fold-inherited sidecar. GraphFromToken remains for legacy artifacts. - The compactor writes the post-expansion graph into the compacted artifact (updated clone on incremental success; fresh graph via preserve on the decline->full path when opted in) so the artifact self-carries its base graph for the next round. Without the opt-in, any inherited sidecar is deleted. - StartNewSync wipes the sidecar with the rest of the keyspace, so a replacement sync never inherits a prior sync's graph. Co-authored-by: c1-squire-dev[bot] Co-Authored-By: Claude Opus 4.8 --- pkg/dotc1z/engine/pebble/cleanup.go | 3 + .../pebble/entitlement_graph_sidecar.go | 66 +++++++++++++++ pkg/dotc1z/pebble_store.go | 15 ++++ pkg/sync/expand/graph_blob.go | 47 +++++++++++ pkg/sync/expand/graph_blob_test.go | 82 +++++++++++++++++++ pkg/sync/expand/incremental.go | 29 ++++--- pkg/sync/graph_from_store_test.go | 73 +++++++++++++++++ pkg/sync/state.go | 39 ++++++++- pkg/sync/syncer.go | 33 +++++++- pkg/synccompactor/compactor.go | 31 +++++++ .../incremental_expansion_test.go | 70 ++++++++++++++++ 11 files changed, 471 insertions(+), 17 deletions(-) create mode 100644 pkg/dotc1z/engine/pebble/entitlement_graph_sidecar.go create mode 100644 pkg/sync/expand/graph_blob.go create mode 100644 pkg/sync/expand/graph_blob_test.go create mode 100644 pkg/sync/graph_from_store_test.go diff --git a/pkg/dotc1z/engine/pebble/cleanup.go b/pkg/dotc1z/engine/pebble/cleanup.go index 416c29b50..057c08f6f 100644 --- a/pkg/dotc1z/engine/pebble/cleanup.go +++ b/pkg/dotc1z/engine/pebble/cleanup.go @@ -41,6 +41,8 @@ func scopedRanges() [][2][]byte { // Stats sidecar — single key; the half-open range shape // contains exactly that one key. {encodeSyncStatsKey(), upperBoundOf(encodeSyncStatsKey())}, + // Entitlement-graph sidecar — same single-key shape. + {EntitlementGraphSidecarLowerBound(), EntitlementGraphSidecarUpperBound()}, } } @@ -86,6 +88,7 @@ func (e *Engine) ResetForNewSync(ctx context.Context) error { spans := []pebble.KeyRange{ {Start: []byte{versionV3, typeResourceType}, End: []byte{versionV3, typeEngineMeta}}, {Start: SyncStatsSidecarLowerBound(), End: SyncStatsSidecarUpperBound()}, + {Start: EntitlementGraphSidecarLowerBound(), End: EntitlementGraphSidecarUpperBound()}, } // AllowSealed: StartNewSync legitimately replaces a finished (sealed) // sync; the wipe is the first step of leaving the sealed state. The diff --git a/pkg/dotc1z/engine/pebble/entitlement_graph_sidecar.go b/pkg/dotc1z/engine/pebble/entitlement_graph_sidecar.go new file mode 100644 index 000000000..a5c8f0f1d --- /dev/null +++ b/pkg/dotc1z/engine/pebble/entitlement_graph_sidecar.go @@ -0,0 +1,66 @@ +package pebble + +import ( + "context" + "errors" + + "github.com/cockroachdb/pebble/v2" + + "github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/codec" +) + +// Entitlement-graph sidecar: an opaque blob (owned by pkg/sync/expand) +// holding the sync's expansion graph, so it rides the c1z instead of +// bloating the sync token. Same single-fixed-key shape as the stats +// sidecar; absent on files written by a pre-sidecar SDK. + +// encodeEntitlementGraphKey returns the engine-meta key for the single +// sync's graph blob. One sync per file, so no sync_id in the key. +func encodeEntitlementGraphKey() []byte { + buf := make([]byte, 0, 6+len("entitlement-graph")) + buf = append(buf, versionV3, typeEngineMeta) + buf = codec.AppendTupleString(buf, "entitlement-graph") + buf = codec.AppendTupleSeparator(buf) + return buf +} + +// EntitlementGraphSidecarLowerBound / UpperBound expose the sidecar's +// single-key range for cleanup and compaction. +func EntitlementGraphSidecarLowerBound() []byte { + return encodeEntitlementGraphKey() +} + +func EntitlementGraphSidecarUpperBound() []byte { + return upperBoundOf(EntitlementGraphSidecarLowerBound()) +} + +// PutEntitlementGraphSidecar stores the opaque graph blob. Same write +// barrier as the stats sidecar: callers span EndSync's sealed window. +func (e *Engine) PutEntitlementGraphSidecar(ctx context.Context, data []byte) error { + return e.withWriteAllowSealed(func() error { + return e.db.MetaSet(encodeEntitlementGraphKey(), data, pebble.Sync) + }) +} + +// GetEntitlementGraphSidecar returns the stored blob, or (nil, nil) if +// none exists. +func (e *Engine) GetEntitlementGraphSidecar(ctx context.Context) ([]byte, error) { + val, closer, err := e.db.Get(encodeEntitlementGraphKey()) + if err != nil { + if errors.Is(err, pebble.ErrNotFound) { + return nil, nil + } + return nil, err + } + defer closer.Close() + out := make([]byte, len(val)) + copy(out, val) + return out, nil +} + +// DeleteEntitlementGraphSidecar removes the blob (no-op when absent). +func (e *Engine) DeleteEntitlementGraphSidecar(ctx context.Context) error { + return e.withWriteAllowSealed(func() error { + return e.db.MetaDelete(encodeEntitlementGraphKey(), pebble.Sync) + }) +} diff --git a/pkg/dotc1z/pebble_store.go b/pkg/dotc1z/pebble_store.go index 05652cddf..92b7f1ca3 100644 --- a/pkg/dotc1z/pebble_store.go +++ b/pkg/dotc1z/pebble_store.go @@ -498,6 +498,21 @@ func (s *pebbleStore) PutAsset(ctx context.Context, assetRef *v2.AssetRef, conte return s.markDirty(s.Engine.PutAsset(ctx, assetRef, contentType, data)) } +// PutEntitlementGraphBlob / GetEntitlementGraphBlob / DeleteEntitlementGraphBlob +// expose the entitlement-graph sidecar (see pkg/sync's EntitlementGraphStore). +// The blob format is owned by pkg/sync/expand; the store treats it as opaque. +func (s *pebbleStore) PutEntitlementGraphBlob(ctx context.Context, data []byte) error { + return s.markDirty(s.engine.PutEntitlementGraphSidecar(ctx, data)) +} + +func (s *pebbleStore) GetEntitlementGraphBlob(ctx context.Context) ([]byte, error) { + return s.engine.GetEntitlementGraphSidecar(ctx) +} + +func (s *pebbleStore) DeleteEntitlementGraphBlob(ctx context.Context) error { + return s.markDirty(s.engine.DeleteEntitlementGraphSidecar(ctx)) +} + // SetSupportsDiff marks the given sync as diff-capable, matching the // SQLite engine's sync_runs.supports_diff column. The c1z sanitizer // carries this marker from a source sync to its sanitized copy so the diff --git a/pkg/sync/expand/graph_blob.go b/pkg/sync/expand/graph_blob.go new file mode 100644 index 000000000..f188c7445 --- /dev/null +++ b/pkg/sync/expand/graph_blob.go @@ -0,0 +1,47 @@ +package expand + +import ( + "encoding/json" + "fmt" +) + +// graphBlobEnvelope is the serialized form of the entitlement-graph sidecar +// stored in a c1z (instead of bloating the sync token). SyncID guards +// against reading a graph inherited from a different sync (e.g. a fold-copied +// compaction base). +type graphBlobEnvelope struct { + SyncID string `json:"sync_id"` + Graph *EntitlementGraph `json:"graph"` +} + +// MarshalGraphBlob serializes a graph for the c1z sidecar, stamped with the +// sync it belongs to. Transient state is stripped first (a reload rebuilds it). +func MarshalGraphBlob(syncID string, g *EntitlementGraph) ([]byte, error) { + if g == nil { + return nil, fmt.Errorf("marshal graph blob: nil graph") + } + g.ClearTransientState() + data, err := json.Marshal(graphBlobEnvelope{SyncID: syncID, Graph: g}) + if err != nil { + return nil, fmt.Errorf("marshal graph blob: %w", err) + } + return data, nil +} + +// UnmarshalGraphBlob parses a sidecar blob. Returns (nil, nil) when the blob +// belongs to a different sync than wantSyncID (stale inherited sidecar); +// pass "" to skip the guard. +func UnmarshalGraphBlob(data []byte, wantSyncID string) (*EntitlementGraph, error) { + var env graphBlobEnvelope + if err := json.Unmarshal(data, &env); err != nil { + return nil, fmt.Errorf("unmarshal graph blob: %w", err) + } + if wantSyncID != "" && env.SyncID != wantSyncID { + return nil, nil + } + if env.Graph == nil { + return nil, nil + } + env.Graph.reinitMaps() + return env.Graph, nil +} diff --git a/pkg/sync/expand/graph_blob_test.go b/pkg/sync/expand/graph_blob_test.go new file mode 100644 index 000000000..ecc1766bc --- /dev/null +++ b/pkg/sync/expand/graph_blob_test.go @@ -0,0 +1,82 @@ +package expand + +import ( + "context" + "fmt" + "testing" + + "github.com/stretchr/testify/require" +) + +// TestGraphBlobRoundTrip: marshal/unmarshal preserves the graph; the sync-id +// guard rejects a blob from a different sync. +func TestGraphBlobRoundTrip(t *testing.T) { + ctx := context.Background() + g := NewEntitlementGraph(ctx) + g.AddEntitlementID("ent-a") + g.AddEntitlementID("ent-b") + require.NoError(t, g.AddEdge(ctx, "ent-a", "ent-b", false, nil)) + + data, err := MarshalGraphBlob("sync-1", g) + require.NoError(t, err) + + got, err := UnmarshalGraphBlob(data, "sync-1") + require.NoError(t, err) + require.NotNil(t, got) + require.NotNil(t, got.GetNode("ent-a")) + require.Len(t, got.Edges, 1) + // reinitMaps ran: absent maps are usable, not nil. + require.NotNil(t, got.EntitlementsToNodes) + + // Wrong sync id -> nil (stale inherited sidecar). + stale, err := UnmarshalGraphBlob(data, "sync-2") + require.NoError(t, err) + require.Nil(t, stale) + + // Empty want skips the guard. + unguarded, err := UnmarshalGraphBlob(data, "") + require.NoError(t, err) + require.NotNil(t, unguarded) +} + +// TestMarshalGraphBlob_StripsTransientState: the blob never carries the +// expansion scaffolding. +func TestMarshalGraphBlob_StripsTransientState(t *testing.T) { + ctx := context.Background() + g := NewEntitlementGraph(ctx) + g.AddEntitlementID("ent-a") + g.Actions = []*EntitlementGraphAction{{}} + + data, err := MarshalGraphBlob("s", g) + require.NoError(t, err) + got, err := UnmarshalGraphBlob(data, "s") + require.NoError(t, err) + require.Nil(t, got.Actions, "transient state must be stripped from the blob") +} + +// TestGraphBlobSizeAtScale measures the sidecar blob for a nested-groups graph +// at increasing node counts — the measurement behind moving the graph out of +// the sync token (tokens travel through workflow state; the c1z does not). +func TestGraphBlobSizeAtScale(t *testing.T) { + ctx := context.Background() + for _, n := range []int{1_000, 10_000, 50_000} { + g := NewEntitlementGraph(ctx) + for i := 0; i < n; i++ { + g.AddEntitlementID(entName(i)) + } + // Nested chains: every node points at the next, 10-deep trees. + for i := 0; i+1 < n; i++ { + if i%10 != 9 { + require.NoError(t, g.AddEdge(ctx, entName(i), entName(i+1), false, nil)) + } + } + data, err := MarshalGraphBlob("sync", g) + require.NoError(t, err) + require.NotEmpty(t, data) + t.Logf("nodes=%d edges=%d blob=%d bytes (%.0f B/node)", n, len(g.Edges), len(data), float64(len(data))/float64(n)) + } +} + +func entName(i int) string { + return fmt.Sprintf("group:g%06d:member", i) +} diff --git a/pkg/sync/expand/incremental.go b/pkg/sync/expand/incremental.go index 4dd49fb04..1268975b7 100644 --- a/pkg/sync/expand/incremental.go +++ b/pkg/sync/expand/incremental.go @@ -40,23 +40,28 @@ func (g *EntitlementGraph) Clone() (*EntitlementGraph, error) { if err := json.Unmarshal(data, out); err != nil { return nil, fmt.Errorf("clone entitlement graph: %w", err) } - // json leaves absent maps nil; reinit so the clone is immediately usable. - if out.Nodes == nil { - out.Nodes = map[int]Node{} + out.reinitMaps() + return out, nil +} + +// reinitMaps replaces nil maps (json leaves absent maps nil) so a +// deserialized graph is immediately usable. +func (g *EntitlementGraph) reinitMaps() { + if g.Nodes == nil { + g.Nodes = map[int]Node{} } - if out.EntitlementsToNodes == nil { - out.EntitlementsToNodes = map[string]int{} + if g.EntitlementsToNodes == nil { + g.EntitlementsToNodes = map[string]int{} } - if out.SourcesToDestinations == nil { - out.SourcesToDestinations = map[int]map[int]int{} + if g.SourcesToDestinations == nil { + g.SourcesToDestinations = map[int]map[int]int{} } - if out.DestinationsToSources == nil { - out.DestinationsToSources = map[int]map[int]int{} + if g.DestinationsToSources == nil { + g.DestinationsToSources = map[int]map[int]int{} } - if out.Edges == nil { - out.Edges = map[int]Edge{} + if g.Edges == nil { + g.Edges = map[int]Edge{} } - return out, nil } // ErrIncrementalFallback means a new edge closed a cycle; the caller should diff --git a/pkg/sync/graph_from_store_test.go b/pkg/sync/graph_from_store_test.go new file mode 100644 index 000000000..9f92e346a --- /dev/null +++ b/pkg/sync/graph_from_store_test.go @@ -0,0 +1,73 @@ +package sync //nolint:revive,nolintlint // package name kept for compatibility + +import ( + "context" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/conductorone/baton-sdk/pkg/connectorstore" + "github.com/conductorone/baton-sdk/pkg/dotc1z" + "github.com/conductorone/baton-sdk/pkg/dotc1z/c1zstore" + "github.com/conductorone/baton-sdk/pkg/sync/expand" +) + +// TestGraphFromStore: a graph blob written to a Pebble c1z round-trips out via +// GraphFromStore; a mismatched sync id or missing blob yields nil. +func TestGraphFromStore(t *testing.T) { + ctx := context.Background() + + store, err := dotc1z.NewStore(ctx, filepath.Join(t.TempDir(), "g.c1z"), + dotc1z.WithEngine(c1zstore.EnginePebble), dotc1z.WithTmpDir(t.TempDir())) + require.NoError(t, err) + defer store.Close(ctx) + syncID, err := store.StartNewSync(ctx, connectorstore.SyncTypeFull, "") + require.NoError(t, err) + require.NoError(t, store.EndSync(ctx)) + + // No blob yet -> nil, no error. + got, err := GraphFromStore(ctx, store, syncID) + require.NoError(t, err) + require.Nil(t, got) + + g := expand.NewEntitlementGraph(ctx) + g.AddEntitlementID("ent-a") + g.AddEntitlementID("ent-b") + require.NoError(t, g.AddEdge(ctx, "ent-a", "ent-b", false, nil)) + data, err := expand.MarshalGraphBlob(syncID, g) + require.NoError(t, err) + gs, ok := store.(EntitlementGraphStore) + require.True(t, ok, "pebble store must implement EntitlementGraphStore") + require.NoError(t, gs.PutEntitlementGraphBlob(ctx, data)) + + got, err = GraphFromStore(ctx, store, syncID) + require.NoError(t, err) + require.NotNil(t, got) + require.NotNil(t, got.GetNode("ent-a")) + require.Len(t, got.Edges, 1) + + // Wrong sync id -> nil (stale sidecar guard). + got, err = GraphFromStore(ctx, store, "some-other-sync") + require.NoError(t, err) + require.Nil(t, got) + + // StartNewSync resets the keyspace, wiping the sidecar. + _, err = store.StartNewSync(ctx, connectorstore.SyncTypeFull, "") + require.NoError(t, err) + got, err = GraphFromStore(ctx, store, "") + require.NoError(t, err) + require.Nil(t, got, "new sync must not inherit the prior sync's graph sidecar") +} + +// TestGraphFromStore_SQLiteUnsupported: SQLite has no sidecar; nil, no error. +func TestGraphFromStore_SQLiteUnsupported(t *testing.T) { + ctx := context.Background() + store, err := dotc1z.NewStore(ctx, filepath.Join(t.TempDir(), "g.c1z"), dotc1z.WithTmpDir(t.TempDir())) + require.NoError(t, err) + defer store.Close(ctx) + + got, err := GraphFromStore(ctx, store, "any") + require.NoError(t, err) + require.Nil(t, got) +} diff --git a/pkg/sync/state.go b/pkg/sync/state.go index edcd1eec0..cea10f3ac 100644 --- a/pkg/sync/state.go +++ b/pkg/sync/state.go @@ -10,6 +10,7 @@ import ( "sync" "time" + "github.com/conductorone/baton-sdk/pkg/dotc1z/c1zstore" "github.com/conductorone/baton-sdk/pkg/sync/expand" "github.com/grpc-ecosystem/go-grpc-middleware/logging/zap/ctxzap" @@ -34,6 +35,7 @@ type State interface { FinishAction(ctx context.Context, action *Action) NextPage(ctx context.Context, actionID string, pageToken string) error EntitlementGraph(ctx context.Context) *expand.EntitlementGraph + PeekEntitlementGraph() *expand.EntitlementGraph ClearEntitlementGraph(ctx context.Context) ClearEntitlementGraphTransientState(ctx context.Context) Current() *Action @@ -107,7 +109,8 @@ func PrepareExpansionReplayToken(stateStr string) (string, error) { // GraphFromToken parses a sync token and returns its persisted entitlement // graph, for running an incremental expansion against a prior sync's graph. // Returns nil if the token carried no graph (e.g. a sync without -// WithPreserveEntitlementGraph). +// WithPreserveEntitlementGraph). Legacy: preserve now writes the graph into +// the c1z sidecar when the store supports it — prefer GraphFromStore. func GraphFromToken(stateStr string) (*expand.EntitlementGraph, error) { st := newState() if err := st.Unmarshal(stateStr); err != nil { @@ -116,6 +119,33 @@ func GraphFromToken(stateStr string) (*expand.EntitlementGraph, error) { return st.entitlementGraph, nil } +// EntitlementGraphStore is the optional store capability backing graph +// persistence in the c1z (Pebble implements it; SQLite does not). The blob +// format is owned by pkg/sync/expand. +type EntitlementGraphStore interface { + PutEntitlementGraphBlob(ctx context.Context, data []byte) error + GetEntitlementGraphBlob(ctx context.Context) ([]byte, error) + DeleteEntitlementGraphBlob(ctx context.Context) error +} + +// GraphFromStore loads the entitlement graph persisted in the c1z sidecar for +// syncID. Returns nil (no error) when the store lacks the capability, no graph +// was preserved, or the stored graph belongs to a different sync. +func GraphFromStore(ctx context.Context, store c1zstore.Store, syncID string) (*expand.EntitlementGraph, error) { + gs, ok := store.(EntitlementGraphStore) + if !ok { + return nil, nil + } + data, err := gs.GetEntitlementGraphBlob(ctx) + if err != nil { + return nil, err + } + if data == nil { + return nil, nil + } + return expand.UnmarshalGraphBlob(data, syncID) +} + // ActionOp represents a sync operation. type ActionOp uint8 @@ -1112,6 +1142,13 @@ func (st *state) EntitlementGraph(ctx context.Context) *expand.EntitlementGraph return st.entitlementGraph } +// PeekEntitlementGraph returns the graph without allocating one when absent +// (unlike EntitlementGraph). Used by the preserve path to decide whether +// there is a graph worth persisting. +func (st *state) PeekEntitlementGraph() *expand.EntitlementGraph { + return st.entitlementGraph +} + // ClearEntitlementGraph clears the entitlement graph. This is meant to make the final sync token less confusing. func (st *state) ClearEntitlementGraph(ctx context.Context) { st.entitlementGraph = nil diff --git a/pkg/sync/syncer.go b/pkg/sync/syncer.go index 99d3c8e76..f63feef4c 100644 --- a/pkg/sync/syncer.go +++ b/pkg/sync/syncer.go @@ -273,6 +273,30 @@ func NewExpanderStore(store c1zstore.Store) expand.ExpanderStore { return expanderStoreAdapter{store: store} } +// persistEntitlementGraphToStore moves the preserved graph from the sync token +// into the c1z sidecar when the store supports it, keeping the token skinny. +// On any failure the graph stays in the token (GraphFromToken still works). +func (s *syncer) persistEntitlementGraphToStore(ctx context.Context, syncID string) { + g := s.state.PeekEntitlementGraph() + if g == nil { + return + } + gs, ok := s.store.(EntitlementGraphStore) + if !ok { + return // no sidecar support (SQLite): graph stays in the token + } + data, err := expand.MarshalGraphBlob(syncID, g) + if err != nil { + ctxzap.Extract(ctx).Warn("preserve entitlement graph: marshal failed; keeping graph in token", zap.Error(err)) + return + } + if err := gs.PutEntitlementGraphBlob(ctx, data); err != nil { + ctxzap.Extract(ctx).Warn("preserve entitlement graph: sidecar write failed; keeping graph in token", zap.Error(err)) + return + } + s.state.ClearEntitlementGraph(ctx) +} + func (a expanderStoreAdapter) GetEntitlement(ctx context.Context, req *reader_v2.EntitlementsReaderServiceGetEntitlementRequest) (*reader_v2.EntitlementsReaderServiceGetEntitlementResponse, error) { return a.store.GetEntitlement(ctx, req) } @@ -1061,12 +1085,13 @@ func (s *syncer) Sync(ctx context.Context) error { } // Force a checkpoint to clear completed actions & entitlement graph in sync_token. - // preserveEntitlementGraph keeps the graph in the final token so a later - // incremental expansion can reload it instead of rebuilding from scratch — - // but strip its transient working state (action queue, expansion plan, - // metrics) first, which a reload doesn't need and which bloats the token. + // preserveEntitlementGraph keeps the graph for a later incremental + // expansion: written to the c1z sidecar when the store supports it (token + // stays skinny — a whale graph is megabytes), else kept in the final token. + // Transient working state is stripped either way; a reload rebuilds it. if s.preserveEntitlementGraph { s.state.ClearEntitlementGraphTransientState(ctx) + s.persistEntitlementGraphToStore(ctx, syncID) } else { s.state.ClearEntitlementGraph(ctx) } diff --git a/pkg/synccompactor/compactor.go b/pkg/synccompactor/compactor.go index 7a6bcf7b5..c9c410a44 100644 --- a/pkg/synccompactor/compactor.go +++ b/pkg/synccompactor/compactor.go @@ -702,6 +702,7 @@ func (c *Compactor) expandGrantsIncremental(ctx context.Context, newSyncId strin if len(newEdges) == 0 && len(changedEntitlementIDs) == 0 { // Nothing changed relative to the base — its grants were already merged in. + c.persistGraphSidecar(ctx, base, newSyncId) return c.finishIncrementalExpansion(ctx) } @@ -720,9 +721,27 @@ func (c *Compactor) expandGrantsIncremental(ctx context.Context, newSyncId strin ctxzap.Extract(ctx).Info("incremental grant expansion complete", zap.Int("entitlements_walked", len(res.EntitlementsWalked)), zap.Int("grants_written", res.GrantsWritten)) + c.persistGraphSidecar(ctx, base, newSyncId) return c.finishIncrementalExpansion(ctx) } +// persistGraphSidecar writes the post-expansion graph into the compacted c1z +// so the artifact carries its own base graph for the next incremental run. +// Best-effort: on failure the next run just falls back to full expansion. +func (c *Compactor) persistGraphSidecar(ctx context.Context, g *expand.EntitlementGraph, syncID string) { + gs, ok := c.compactedC1z.(sync.EntitlementGraphStore) + if !ok { + return + } + data, err := expand.MarshalGraphBlob(syncID, g) + if err == nil { + err = gs.PutEntitlementGraphBlob(ctx, data) + } + if err != nil { + ctxzap.Extract(ctx).Warn("incremental expansion: persist graph sidecar failed", zap.Error(err)) + } +} + // restoreEndedSync returns the compacted sync to the ended state the full path // expects, after an incremental attempt that resumed it. Runs on a detached, // timeout-bounded context so a cancelled parent can't strand the store @@ -997,6 +1016,18 @@ func (c *Compactor) expandGrants(ctx context.Context, newSyncId string, compacti sync.WithCompactionMergedStore(), } + // Keep the artifact's graph sidecar coherent with this full expansion: + // opted-in compactions preserve a fresh graph (so the incremental chain + // heals after a fallback); otherwise drop any sidecar inherited from a + // fold-copied base. + if c.incrementalBaseGraph != nil { + syncOpts = append(syncOpts, sync.WithPreserveEntitlementGraph()) + } else if gs, ok := c.compactedC1z.(sync.EntitlementGraphStore); ok { + if err := gs.DeleteEntitlementGraphBlob(ctx); err != nil { + l.Warn("expandGrants: delete inherited graph sidecar failed", zap.Error(err)) + } + } + compactionDuration := time.Since(compactionStart) runDuration := c.runDuration - compactionDuration l.Debug("finished compaction", zap.Duration("compaction_duration", compactionDuration)) diff --git a/pkg/synccompactor/incremental_expansion_test.go b/pkg/synccompactor/incremental_expansion_test.go index f94664f44..12266e5c4 100644 --- a/pkg/synccompactor/incremental_expansion_test.go +++ b/pkg/synccompactor/incremental_expansion_test.go @@ -20,6 +20,7 @@ import ( "github.com/conductorone/baton-sdk/pkg/dotc1z" "github.com/conductorone/baton-sdk/pkg/dotc1z/c1zstore" enginepkg "github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble" + sdksync "github.com/conductorone/baton-sdk/pkg/sync" "github.com/conductorone/baton-sdk/pkg/sync/expand" batonGrant "github.com/conductorone/baton-sdk/pkg/types/grant" ) @@ -752,3 +753,72 @@ func TestCompactor_IncrementalNewMemberRebuildFallsBackToDerive(t *testing.T) { require.Equal(t, fullGrants, incGrants, "derive-fallback incremental must equal full expansion") hasGrant(t, incGrants, "ent-c|user|bob") } + +// artifactGraph loads the graph sidecar from a compacted artifact. +func artifactGraph(t *testing.T, ctx context.Context, path, syncID string) *expand.EntitlementGraph { + t.Helper() + store, err := dotc1z.NewStore(ctx, path, dotc1z.WithReadOnly(true), dotc1z.WithTmpDir(t.TempDir())) + require.NoError(t, err) + defer store.Close(ctx) + g, err := sdksync.GraphFromStore(ctx, store, syncID) + require.NoError(t, err) + return g +} + +// TestCompactor_ArtifactCarriesGraphSidecar: the compacted artifact self-carries +// its post-expansion graph for the next incremental run — on the incremental +// path (updated clone), on the decline->full path (fresh graph via preserve), +// and not at all when incremental wasn't requested. +func TestCompactor_ArtifactCarriesGraphSidecar(t *testing.T) { + ctx := context.Background() + + // (a) Incremental success: sidecar = base graph + the increment's new edge. + entries := buildIncrementalFixtures(t, ctx, t.TempDir()) // increment adds ent-a -> ent-b + cInc, cleanupInc, err := NewCompactor(ctx, t.TempDir(), entries, + WithTmpDir(t.TempDir()), + WithEngine(c1zstore.EnginePebble), + WithIncrementalExpansion(baseGraphForFixtures(t, ctx)), // ent-b -> ent-c + ) + require.NoError(t, err) + defer func() { _ = cleanupInc() }() + incOut, err := cInc.Compact(ctx) + require.NoError(t, err) + require.True(t, cInc.incrementalExpansionRan) + + g := artifactGraph(t, ctx, incOut.FilePath, incOut.SyncID) + require.NotNil(t, g, "incremental artifact must carry its graph sidecar") + require.NotNil(t, g.GetNode("ent-a"), "sidecar graph must include the increment's new edge source") + require.Len(t, g.Edges, 2, "base edge + folded-in new edge") + + // (b) Decline -> full (narrowed edge) with incremental requested: the full + // path preserves a fresh graph so the chain heals after the fallback. + declEntries := buildSpecChangeFixtures(t, ctx, t.TempDir(), false, true) // deep -> shallow + cDecl, cleanupDecl, err := NewCompactor(ctx, t.TempDir(), declEntries, + WithTmpDir(t.TempDir()), + WithEngine(c1zstore.EnginePebble), + WithIncrementalExpansion(specChangeBaseGraph(t, ctx, false)), + ) + require.NoError(t, err) + defer func() { _ = cleanupDecl() }() + declOut, err := cDecl.Compact(ctx) + require.NoError(t, err) + require.False(t, cDecl.incrementalExpansionRan, "narrowed edge must decline to full") + + g = artifactGraph(t, ctx, declOut.FilePath, declOut.SyncID) + require.NotNil(t, g, "declined-to-full artifact must still carry a fresh graph sidecar") + require.NotNil(t, g.GetNode("ent-b")) + + // (c) Incremental not requested: no sidecar. + plainEntries := buildIncrementalFixtures(t, ctx, t.TempDir()) + cPlain, cleanupPlain, err := NewCompactor(ctx, t.TempDir(), plainEntries, + WithTmpDir(t.TempDir()), + WithEngine(c1zstore.EnginePebble), + ) + require.NoError(t, err) + defer func() { _ = cleanupPlain() }() + plainOut, err := cPlain.Compact(ctx) + require.NoError(t, err) + + g = artifactGraph(t, ctx, plainOut.FilePath, plainOut.SyncID) + require.Nil(t, g, "artifact without incremental opt-in must carry no graph sidecar") +} From 4a4c42daf3b6c4c2c0dcdf156b28469603f49eef Mon Sep 17 00:00:00 2001 From: manojacs Date: Wed, 29 Jul 2026 21:02:56 +0000 Subject: [PATCH 05/15] fix rebase compatibility with main Co-authored-by: c1-squire-dev[bot] --- pkg/dotc1z/pebble_store.go | 6 +++--- pkg/sync/syncer.go | 2 -- pkg/synccompactor/incremental_expansion_test.go | 2 +- 3 files changed, 4 insertions(+), 6 deletions(-) diff --git a/pkg/dotc1z/pebble_store.go b/pkg/dotc1z/pebble_store.go index 92b7f1ca3..93fa38972 100644 --- a/pkg/dotc1z/pebble_store.go +++ b/pkg/dotc1z/pebble_store.go @@ -502,15 +502,15 @@ func (s *pebbleStore) PutAsset(ctx context.Context, assetRef *v2.AssetRef, conte // expose the entitlement-graph sidecar (see pkg/sync's EntitlementGraphStore). // The blob format is owned by pkg/sync/expand; the store treats it as opaque. func (s *pebbleStore) PutEntitlementGraphBlob(ctx context.Context, data []byte) error { - return s.markDirty(s.engine.PutEntitlementGraphSidecar(ctx, data)) + return s.markDirty(s.Engine.PutEntitlementGraphSidecar(ctx, data)) } func (s *pebbleStore) GetEntitlementGraphBlob(ctx context.Context) ([]byte, error) { - return s.engine.GetEntitlementGraphSidecar(ctx) + return s.Engine.GetEntitlementGraphSidecar(ctx) } func (s *pebbleStore) DeleteEntitlementGraphBlob(ctx context.Context) error { - return s.markDirty(s.engine.DeleteEntitlementGraphSidecar(ctx)) + return s.markDirty(s.Engine.DeleteEntitlementGraphSidecar(ctx)) } // SetSupportsDiff marks the given sync as diff-capable, matching the diff --git a/pkg/sync/syncer.go b/pkg/sync/syncer.go index f63feef4c..d4ff22b04 100644 --- a/pkg/sync/syncer.go +++ b/pkg/sync/syncer.go @@ -1095,8 +1095,6 @@ func (s *syncer) Sync(ctx context.Context) error { } else { s.state.ClearEntitlementGraph(ctx) } - s.state.ClearExclusionGroupTracking(ctx) - err = s.Checkpoint(ctx, true) if err != nil { return s.returnSyncError(l, span, err) diff --git a/pkg/synccompactor/incremental_expansion_test.go b/pkg/synccompactor/incremental_expansion_test.go index 12266e5c4..86f830d29 100644 --- a/pkg/synccompactor/incremental_expansion_test.go +++ b/pkg/synccompactor/incremental_expansion_test.go @@ -630,7 +630,7 @@ func TestCompactor_IncrementalSealedArtifactLifecycle(t *testing.T) { // (2) by_principal index is populated and covers sam (written incrementally). eng, ok := enginepkg.AsEngine(store) require.True(t, ok, "expected a pebble engine") - it, err := eng.DB().NewIter(&pebble.IterOptions{ + it, err := eng.NewIter(&pebble.IterOptions{ LowerBound: enginepkg.GrantByPrincipalLowerBound(), UpperBound: enginepkg.GrantByPrincipalUpperBound(), }) From d6570083f90118871f4ee4174561186675837745 Mon Sep 17 00:00:00 2001 From: manojacs Date: Tue, 4 Aug 2026 06:48:30 +0000 Subject: [PATCH 06/15] feat(sync): harden incremental grant expansion Make incremental compaction derive affected memberships and edge changes from merged artifact data, validate and clone the preserved entitlement graph, and safely decline cycles, revocations, unsupported stores, and dense affected closures to normal full expansion. Persist versioned graph sidecars only after sealing and bind them to the artifact's whole-file grant digest. Reuse now requires matching sync ID, graph structure, digest ABI, grant count, and grant hash; stale, inherited, unbound, or mismatched graphs fail closed. Keep the default full-expansion behavior unchanged unless callers explicitly opt in. Add stable outcome reasons, safe finalization ordering, fold-sidecar invalidation, and the shared ingestion-invariant path needed by compaction. Co-authored-by: c1-squire-dev[bot] --- pkg/dotc1z/c1zstore/c1zstore.go | 14 ++ pkg/dotc1z/pebble_store.go | 18 +- pkg/sync/expand/graph.go | 93 +++++++++ pkg/sync/expand/graph_blob.go | 49 ++++- pkg/sync/expand/incremental.go | 157 ++++++++++++-- pkg/sync/expand/topological_merge.go | 6 +- pkg/sync/ingest_invariants.go | 52 +++-- pkg/sync/state.go | 24 ++- pkg/sync/syncer.go | 44 ++-- pkg/synccompactor/compactor.go | 282 +++++++++++++++++++++++--- pkg/synccompactor/compactor_pebble.go | 7 + 11 files changed, 655 insertions(+), 91 deletions(-) diff --git a/pkg/dotc1z/c1zstore/c1zstore.go b/pkg/dotc1z/c1zstore/c1zstore.go index 8aa50210e..87e924c51 100644 --- a/pkg/dotc1z/c1zstore/c1zstore.go +++ b/pkg/dotc1z/c1zstore/c1zstore.go @@ -61,3 +61,17 @@ type Store interface { SessionStore() sessions.SessionStore } + +// GrantGenerationDigest binds derived metadata to the exact grant generation +// stored in an artifact. +type GrantGenerationDigest struct { + Hash []byte + Count int64 + ABIVersion uint32 +} + +// GrantGenerationDigestReader is implemented by stores that persist an exact +// whole-file grant digest at seal time. +type GrantGenerationDigestReader interface { + GrantGenerationDigest(ctx context.Context) (GrantGenerationDigest, bool, error) +} diff --git a/pkg/dotc1z/pebble_store.go b/pkg/dotc1z/pebble_store.go index 93fa38972..2bc3f8223 100644 --- a/pkg/dotc1z/pebble_store.go +++ b/pkg/dotc1z/pebble_store.go @@ -364,6 +364,18 @@ func (s *pebbleStore) PebbleEngine() *pebble.Engine { return s.Engine } +func (s *pebbleStore) GrantGenerationDigest(ctx context.Context) (c1zstore.GrantGenerationDigest, bool, error) { + root, ok, err := s.GetGrantDigestGlobalRoot(ctx) + if err != nil || !ok { + return c1zstore.GrantGenerationDigest{}, ok, err + } + return c1zstore.GrantGenerationDigest{ + Hash: append([]byte(nil), root.Hash...), + Count: root.Count, + ABIVersion: pebble.GrantDigestABIVersion, + }, true, nil +} + // CloseEngineOnly closes the Pebble engine without removing the // store's unpacked temp directory, refusing to discard a dirty // writable store. Consumed by the compactor's chunk lifecycle via @@ -502,15 +514,15 @@ func (s *pebbleStore) PutAsset(ctx context.Context, assetRef *v2.AssetRef, conte // expose the entitlement-graph sidecar (see pkg/sync's EntitlementGraphStore). // The blob format is owned by pkg/sync/expand; the store treats it as opaque. func (s *pebbleStore) PutEntitlementGraphBlob(ctx context.Context, data []byte) error { - return s.markDirty(s.Engine.PutEntitlementGraphSidecar(ctx, data)) + return s.markDirty(s.PutEntitlementGraphSidecar(ctx, data)) } func (s *pebbleStore) GetEntitlementGraphBlob(ctx context.Context) ([]byte, error) { - return s.Engine.GetEntitlementGraphSidecar(ctx) + return s.GetEntitlementGraphSidecar(ctx) } func (s *pebbleStore) DeleteEntitlementGraphBlob(ctx context.Context) error { - return s.markDirty(s.Engine.DeleteEntitlementGraphSidecar(ctx)) + return s.markDirty(s.DeleteEntitlementGraphSidecar(ctx)) } // SetSupportsDiff marks the given sync as diff-capable, matching the diff --git a/pkg/sync/expand/graph.go b/pkg/sync/expand/graph.go index 23a9f9c90..c34a4c402 100644 --- a/pkg/sync/expand/graph.go +++ b/pkg/sync/expand/graph.go @@ -2,7 +2,9 @@ package expand import ( "context" + "fmt" "iter" + "slices" "sort" "strings" @@ -132,6 +134,97 @@ func (g *EntitlementGraph) IsExpanded() bool { return true } +// MarkExpansionComplete records that every edge has been evaluated and the +// graph passed cycle detection. Persisted graphs must carry these facts so the +// next expansion can safely treat them as completed bases. +func (g *EntitlementGraph) MarkExpansionComplete() { + for edgeID, edge := range g.Edges { + edge.IsExpanded = true + g.Edges[edgeID] = edge + } + g.HasNoCycles = true + g.Actions = nil +} + +// ValidateCompleted checks the durable facts required before a graph may be +// reused as an incremental-expansion base. +func (g *EntitlementGraph) ValidateCompleted() error { + if g == nil { + return fmt.Errorf("graph is nil") + } + if !g.Loaded { + return fmt.Errorf("graph is not fully loaded") + } + if !g.HasNoCycles { + return fmt.Errorf("graph has not completed cycle detection") + } + if !g.IsExpanded() { + return fmt.Errorf("graph has unexpanded edges") + } + for nodeID, node := range g.Nodes { + if node.Id != nodeID { + return fmt.Errorf("node map key %d does not match node id %d", nodeID, node.Id) + } + for _, entitlementID := range node.EntitlementIDs { + if got, ok := g.EntitlementsToNodes[entitlementID]; !ok || got != nodeID { + return fmt.Errorf("entitlement %q does not map back to node %d", entitlementID, nodeID) + } + } + } + for entitlementID, nodeID := range g.EntitlementsToNodes { + node, ok := g.Nodes[nodeID] + if !ok || !slices.Contains(node.EntitlementIDs, entitlementID) { + return fmt.Errorf("entitlement map entry %q points to inconsistent node %d", entitlementID, nodeID) + } + } + for edgeID, edge := range g.Edges { + if edge.EdgeID != edgeID { + return fmt.Errorf("edge map key %d does not match edge id %d", edgeID, edge.EdgeID) + } + if _, ok := g.Nodes[edge.SourceID]; !ok { + return fmt.Errorf("edge %d has missing source node %d", edgeID, edge.SourceID) + } + if _, ok := g.Nodes[edge.DestinationID]; !ok { + return fmt.Errorf("edge %d has missing destination node %d", edgeID, edge.DestinationID) + } + if got := g.SourcesToDestinations[edge.SourceID][edge.DestinationID]; got != edgeID { + return fmt.Errorf("edge %d missing from source adjacency", edgeID) + } + if got := g.DestinationsToSources[edge.DestinationID][edge.SourceID]; got != edgeID { + return fmt.Errorf("edge %d missing from destination adjacency", edgeID) + } + } + for sourceID, destinations := range g.SourcesToDestinations { + for destinationID, edgeID := range destinations { + edge, ok := g.Edges[edgeID] + if !ok || edge.SourceID != sourceID || edge.DestinationID != destinationID { + return fmt.Errorf("source adjacency %d->%d points to inconsistent edge %d", sourceID, destinationID, edgeID) + } + } + } + for destinationID, sources := range g.DestinationsToSources { + for sourceID, edgeID := range sources { + edge, ok := g.Edges[edgeID] + if !ok || edge.SourceID != sourceID || edge.DestinationID != destinationID { + return fmt.Errorf("destination adjacency %d<-%d points to inconsistent edge %d", destinationID, sourceID, edgeID) + } + } + } + return nil +} + +// HasCollapsedCycles reports whether full expansion collapsed an SCC into a +// multi-entitlement node. Such a graph no longer records its internal edges, +// so it cannot safely detect an edge removal that splits the SCC. +func (g *EntitlementGraph) HasCollapsedCycles() bool { + for _, node := range g.Nodes { + if len(node.EntitlementIDs) > 1 { + return true + } + } + return false +} + // IsEntitlementExpanded returns true if all the outgoing edges for the given entitlement have been expanded. func (g *EntitlementGraph) IsEntitlementExpanded(entitlementID string) bool { node := g.GetNode(entitlementID) diff --git a/pkg/sync/expand/graph_blob.go b/pkg/sync/expand/graph_blob.go index f188c7445..910029ee5 100644 --- a/pkg/sync/expand/graph_blob.go +++ b/pkg/sync/expand/graph_blob.go @@ -1,8 +1,11 @@ package expand import ( + "bytes" "encoding/json" "fmt" + + "github.com/conductorone/baton-sdk/pkg/dotc1z/c1zstore" ) // graphBlobEnvelope is the serialized form of the entitlement-graph sidecar @@ -10,18 +13,36 @@ import ( // against reading a graph inherited from a different sync (e.g. a fold-copied // compaction base). type graphBlobEnvelope struct { - SyncID string `json:"sync_id"` - Graph *EntitlementGraph `json:"graph"` + FormatVersion uint32 `json:"format_version"` + SyncID string `json:"sync_id"` + GrantDigest *c1zstore.GrantGenerationDigest `json:"grant_digest,omitempty"` + Graph *EntitlementGraph `json:"graph"` } +const graphBlobFormatVersion uint32 = 2 + // MarshalGraphBlob serializes a graph for the c1z sidecar, stamped with the // sync it belongs to. Transient state is stripped first (a reload rebuilds it). func MarshalGraphBlob(syncID string, g *EntitlementGraph) ([]byte, error) { + return marshalGraphBlob(syncID, g, nil) +} + +// MarshalGraphBlobWithGrantDigest binds the graph to the exact sealed grant +// generation. Graph reuse requires this binding. +func MarshalGraphBlobWithGrantDigest(syncID string, g *EntitlementGraph, digest c1zstore.GrantGenerationDigest) ([]byte, error) { + if len(digest.Hash) == 0 || digest.ABIVersion == 0 { + return nil, fmt.Errorf("marshal graph blob: incomplete grant digest") + } + digest.Hash = append([]byte(nil), digest.Hash...) + return marshalGraphBlob(syncID, g, &digest) +} + +func marshalGraphBlob(syncID string, g *EntitlementGraph, digest *c1zstore.GrantGenerationDigest) ([]byte, error) { if g == nil { return nil, fmt.Errorf("marshal graph blob: nil graph") } g.ClearTransientState() - data, err := json.Marshal(graphBlobEnvelope{SyncID: syncID, Graph: g}) + data, err := json.Marshal(graphBlobEnvelope{FormatVersion: graphBlobFormatVersion, SyncID: syncID, GrantDigest: digest, Graph: g}) if err != nil { return nil, fmt.Errorf("marshal graph blob: %w", err) } @@ -32,16 +53,30 @@ func MarshalGraphBlob(syncID string, g *EntitlementGraph) ([]byte, error) { // belongs to a different sync than wantSyncID (stale inherited sidecar); // pass "" to skip the guard. func UnmarshalGraphBlob(data []byte, wantSyncID string) (*EntitlementGraph, error) { + graph, _, err := UnmarshalGraphBlobWithGrantDigest(data, wantSyncID) + return graph, err +} + +// UnmarshalGraphBlobWithGrantDigest returns the persisted grant-generation +// binding along with the graph. A nil digest means the blob is unbound and +// must not be reused incrementally. +func UnmarshalGraphBlobWithGrantDigest(data []byte, wantSyncID string) (*EntitlementGraph, *c1zstore.GrantGenerationDigest, error) { var env graphBlobEnvelope if err := json.Unmarshal(data, &env); err != nil { - return nil, fmt.Errorf("unmarshal graph blob: %w", err) + return nil, nil, fmt.Errorf("unmarshal graph blob: %w", err) + } + if env.FormatVersion != graphBlobFormatVersion { + return nil, nil, nil } if wantSyncID != "" && env.SyncID != wantSyncID { - return nil, nil + return nil, nil, nil } if env.Graph == nil { - return nil, nil + return nil, nil, nil } env.Graph.reinitMaps() - return env.Graph, nil + if env.GrantDigest != nil { + env.GrantDigest.Hash = bytes.Clone(env.GrantDigest.Hash) + } + return env.Graph, env.GrantDigest, nil } diff --git a/pkg/sync/expand/incremental.go b/pkg/sync/expand/incremental.go index 1268975b7..a555d86c1 100644 --- a/pkg/sync/expand/incremental.go +++ b/pkg/sync/expand/incremental.go @@ -2,7 +2,6 @@ package expand import ( "context" - "encoding/json" "errors" "fmt" "sort" @@ -26,24 +25,73 @@ func (g *EntitlementGraph) ClearTransientState() { g.ExpansionMetrics = nil } -// Clone returns a deep copy of the graph. Incremental expansion mutates the -// graph (adds edges), so callers that keep the base graph across retries must -// pass a clone — otherwise a failed run leaves the never-expanded edges in the -// caller's graph, and a retry would treat them as already present and finish -// with an unexpanded artifact. +// Clone returns a structural deep copy of the graph. Incremental expansion +// mutates the graph, so callers must not share maps or slices with the base. +// This deliberately avoids a JSON round trip: graph cloning is paid on every +// eligible incremental attempt and benchmark evidence showed serialization +// dominated both CPU and allocation at whale scale. func (g *EntitlementGraph) Clone() (*EntitlementGraph, error) { - data, err := json.Marshal(g) - if err != nil { - return nil, fmt.Errorf("clone entitlement graph: %w", err) + if g == nil { + return nil, fmt.Errorf("clone entitlement graph: nil graph") + } + out := &EntitlementGraph{ + NextNodeID: g.NextNodeID, + NextEdgeID: g.NextEdgeID, + Nodes: make(map[int]Node, len(g.Nodes)), + EntitlementsToNodes: make(map[string]int, len(g.EntitlementsToNodes)), + SourcesToDestinations: cloneNestedIntMap(g.SourcesToDestinations), + DestinationsToSources: cloneNestedIntMap(g.DestinationsToSources), + Edges: make(map[int]Edge, len(g.Edges)), + Loaded: g.Loaded, + Depth: g.Depth, + HasNoCycles: g.HasNoCycles, + } + for id, node := range g.Nodes { + node.EntitlementIDs = append([]string(nil), node.EntitlementIDs...) + out.Nodes[id] = node + } + for entitlementID, nodeID := range g.EntitlementsToNodes { + out.EntitlementsToNodes[entitlementID] = nodeID + } + for id, edge := range g.Edges { + edge.ResourceTypeIDs = append([]string(nil), edge.ResourceTypeIDs...) + out.Edges[id] = edge + } + out.Actions = make([]*EntitlementGraphAction, len(g.Actions)) + for i, action := range g.Actions { + if action == nil { + continue + } + actionCopy := *action + actionCopy.Descendants = append([]ActionDescendant(nil), action.Descendants...) + actionCopy.ResourceTypeIDs = append([]string(nil), action.ResourceTypeIDs...) + out.Actions[i] = &actionCopy + } + if g.ExpansionPlan != nil { + plan := *g.ExpansionPlan + plan.Order = append([]int(nil), g.ExpansionPlan.Order...) + plan.ProjectionSources = append([]string(nil), g.ExpansionPlan.ProjectionSources...) + out.ExpansionPlan = &plan } - out := &EntitlementGraph{} - if err := json.Unmarshal(data, out); err != nil { - return nil, fmt.Errorf("clone entitlement graph: %w", err) + if g.ExpansionMetrics != nil { + metrics := *g.ExpansionMetrics + out.ExpansionMetrics = &metrics } - out.reinitMaps() return out, nil } +func cloneNestedIntMap(source map[int]map[int]int) map[int]map[int]int { + out := make(map[int]map[int]int, len(source)) + for outer, inner := range source { + innerCopy := make(map[int]int, len(inner)) + for key, value := range inner { + innerCopy[key] = value + } + out[outer] = innerCopy + } + return out +} + // reinitMaps replaces nil maps (json leaves absent maps nil) so a // deserialized graph is immediately usable. func (g *EntitlementGraph) reinitMaps() { @@ -75,6 +123,15 @@ var ErrIncrementalFallback = errors.New("incremental expansion: change introduce // tombstone/deletion stage flips from "decline" to "apply deletions". var ErrIncrementalRevocationDecline = errors.New("incremental expansion: revocation-shaped change, fall back to full expansion") +// ErrIncrementalDenseChangeDecline means the affected closure is large enough +// that normal full expansion is the safer bounded-cost path. +var ErrIncrementalDenseChangeDecline = errors.New("incremental expansion: dense affected graph, fall back to full expansion") + +const ( + incrementalDenseGraphMinNodes = 1000 + incrementalMaxAffectedPercent = 10 +) + // NewEdge is one edge to fold in: members of Source also get Destination. type NewEdge struct { SourceEntitlementID string @@ -97,12 +154,17 @@ type IncrementalResult struct { // expanded), and store holds that expansion's grants. Additions only; a new // edge that closes a cycle returns ErrIncrementalFallback. type IncrementalExpander struct { - store ExpanderStore - graph *EntitlementGraph + store ExpanderStore + graph *EntitlementGraph + entitlementCache map[string]*v2.Entitlement } func NewIncrementalExpander(store ExpanderStore, graph *EntitlementGraph) *IncrementalExpander { - return &IncrementalExpander{store: store, graph: graph} + return &IncrementalExpander{ + store: store, + graph: graph, + entitlementCache: make(map[string]*v2.Entitlement), + } } // ExpandChanges recomputes grants for only the subgraph affected by a set of @@ -158,9 +220,15 @@ func (ie *IncrementalExpander) ExpandChanges(ctx context.Context, newEdges []New // Only nodes forward-reachable from a seed are touched. affected := ie.forwardReachable(seeds) + if len(ie.graph.Nodes) >= incrementalDenseGraphMinNodes && + len(affected)*100 > len(ie.graph.Nodes)*incrementalMaxAffectedPercent { + return nil, ErrIncrementalDenseChangeDecline + } - // Topological order so each destination reads already-finalized parents. - order, err := topologicalNodeOrder(ie.graph) + // Topological order only the affected closure. Parents outside this set + // were finalized by the base expansion and are read from the store; sorting + // the untouched graph made K=1 work scale with total graph size. + order, err := topologicalAffectedNodeOrder(ie.graph, affected) if err != nil { return nil, fmt.Errorf("incremental expansion: topological order: %w", err) } @@ -186,9 +254,53 @@ func (ie *IncrementalExpander) ExpandChanges(ctx context.Context, newEdges []New result.GrantsWritten += written } } + ie.graph.MarkExpansionComplete() return result, nil } +func topologicalAffectedNodeOrder(g *EntitlementGraph, affected map[int]struct{}) ([]int, error) { + inDegree := make(map[int]int, len(affected)) + for nodeID := range affected { + if _, ok := g.Nodes[nodeID]; ok { + inDegree[nodeID] = 0 + } + } + for sourceID := range inDegree { + for destinationID := range g.SourcesToDestinations[sourceID] { + if _, ok := inDegree[destinationID]; ok { + inDegree[destinationID]++ + } + } + } + frontier := make([]int, 0, len(inDegree)) + for nodeID, degree := range inDegree { + if degree == 0 { + frontier = append(frontier, nodeID) + } + } + sort.Ints(frontier) + order := make([]int, 0, len(inDegree)) + for len(frontier) > 0 { + nodeID := frontier[0] + frontier = frontier[1:] + order = append(order, nodeID) + for childID := range g.SourcesToDestinations[nodeID] { + if _, ok := inDegree[childID]; !ok { + continue + } + inDegree[childID]-- + if inDegree[childID] == 0 { + frontier = append(frontier, childID) + sort.Ints(frontier) + } + } + } + if len(order) != len(inDegree) { + return nil, fmt.Errorf("incremental expansion: affected graph contains a cycle or dangling edge") + } + return order, nil +} + func (ie *IncrementalExpander) forwardReachable(seeds map[int]struct{}) map[int]struct{} { reached := make(map[int]struct{}) queue := make([]int, 0, len(seeds)) @@ -377,19 +489,26 @@ func (pc *principalContribution) addSource(entitlementID string, isDirect bool) // ref (NotFound) so callers skip it — matching the full evaluator, which treats // NotFound as skip rather than a hard error. func (ie *IncrementalExpander) getEntitlement(ctx context.Context, entitlementID string) (*v2.Entitlement, error) { + if entitlement, ok := ie.entitlementCache[entitlementID]; ok { + return entitlement, nil + } resp, err := ie.store.GetEntitlement(ctx, reader_v2.EntitlementsReaderServiceGetEntitlementRequest_builder{ EntitlementId: entitlementID, }.Build()) if err != nil { if status.Code(err) == codes.NotFound { + ie.entitlementCache[entitlementID] = nil return nil, nil } return nil, fmt.Errorf("incremental expansion: get entitlement %s: %w", entitlementID, err) } if resp == nil { + ie.entitlementCache[entitlementID] = nil return nil, nil } - return resp.GetEntitlement(), nil + entitlement := resp.GetEntitlement() + ie.entitlementCache[entitlementID] = entitlement + return entitlement, nil } // forEachGrant streams an entitlement's grants (filtered by resourceTypeIDs) diff --git a/pkg/sync/expand/topological_merge.go b/pkg/sync/expand/topological_merge.go index c3b358078..37eee0201 100644 --- a/pkg/sync/expand/topological_merge.go +++ b/pkg/sync/expand/topological_merge.go @@ -429,11 +429,7 @@ func (e *Expander) driveTopologicalLayer( // queue. The topological evaluators expand the whole graph in one pass, so they // finalize all edges together at the end rather than per action. func (e *Expander) markExpansionComplete() { - for edgeID, edge := range e.graph.Edges { - edge.IsExpanded = true - e.graph.Edges[edgeID] = edge - } - e.graph.Actions = nil + e.graph.MarkExpansionComplete() } func sortedCopy(in []string) []string { diff --git a/pkg/sync/ingest_invariants.go b/pkg/sync/ingest_invariants.go index 5a2785041..bebe62d52 100644 --- a/pkg/sync/ingest_invariants.go +++ b/pkg/sync/ingest_invariants.go @@ -449,10 +449,43 @@ func ingestInvariantHaltStages() []string { // store-level function so store-producing pipelines without a syncer // (the compactor's expand pass) can enforce the same contract. func RunIngestInvariants(ctx context.Context, store connectorstore.Reader, policy IngestInvariantsPolicy) error { - _, err := runIngestInvariants(ctx, store, policy) + _, err := RunIngestInvariantsWithVerification(ctx, store, policy) return err } +// RunIngestInvariantsWithVerification evaluates the invariant pass and returns +// the verification metadata a store-producing caller must persist after the +// sync is sealed. It does not write the marker itself: publishing proof before +// EndSync would allow an unfinished artifact to claim verification. +func RunIngestInvariantsWithVerification( + ctx context.Context, + store connectorstore.Reader, + policy IngestInvariantsPolicy, +) (*c1zstore.IngestInvariantVerification, error) { + coverage, err := runIngestInvariants(ctx, store, policy) + if err != nil { + return nil, err + } + return &c1zstore.IngestInvariantVerification{ + Generation: IngestInvariantGeneration, + Coverage: coverage, + Mode: ingestInvariantVerificationMode(policy), + }, nil +} + +func ingestInvariantVerificationMode(policy IngestInvariantsPolicy) c1zstore.IngestInvariantVerificationMode { + switch { + case policy.CompactionMerge && policy.FailFast: + return c1zstore.IngestInvariantVerificationModeCompactionMergeFailFast + case policy.CompactionMerge: + return c1zstore.IngestInvariantVerificationModeCompactionMerge + case policy.FailFast: + return c1zstore.IngestInvariantVerificationModeConnectorFailFast + default: + return c1zstore.IngestInvariantVerificationModeConnector + } +} + // runIngestInvariants returns the IDs of checks that actually completed. The // public wrapper intentionally retains its existing error-only API; the syncer // consumes the coverage to persist verification provenance. @@ -608,24 +641,11 @@ func (s *syncer) runIngestionInvariants(ctx context.Context) error { if s.testIngestHaltHook != nil { policy.halt = s.testIngestHaltHook } - coverage, err := runIngestInvariants(ctx, s.store, policy) + verification, err := RunIngestInvariantsWithVerification(ctx, s.store, policy) if err != nil { return err } - mode := c1zstore.IngestInvariantVerificationModeConnector - switch { - case policy.CompactionMerge && policy.FailFast: - mode = c1zstore.IngestInvariantVerificationModeCompactionMergeFailFast - case policy.CompactionMerge: - mode = c1zstore.IngestInvariantVerificationModeCompactionMerge - case policy.FailFast: - mode = c1zstore.IngestInvariantVerificationModeConnectorFailFast - } - s.pendingInvariantVerification = &c1zstore.IngestInvariantVerification{ - Generation: IngestInvariantGeneration, - Coverage: coverage, - Mode: mode, - } + s.pendingInvariantVerification = verification return nil } diff --git a/pkg/sync/state.go b/pkg/sync/state.go index cea10f3ac..dd79dd965 100644 --- a/pkg/sync/state.go +++ b/pkg/sync/state.go @@ -1,6 +1,7 @@ package sync //nolint:revive,nolintlint // we can't change the package name for backwards compatibility import ( + "bytes" "context" "encoding/json" "errors" @@ -143,7 +144,28 @@ func GraphFromStore(ctx context.Context, store c1zstore.Store, syncID string) (* if data == nil { return nil, nil } - return expand.UnmarshalGraphBlob(data, syncID) + graph, boundDigest, err := expand.UnmarshalGraphBlobWithGrantDigest(data, syncID) + if err != nil || graph == nil { + return graph, err + } + if boundDigest == nil { + return nil, nil + } + digestReader, ok := store.(c1zstore.GrantGenerationDigestReader) + if !ok { + return nil, nil + } + currentDigest, found, err := digestReader.GrantGenerationDigest(ctx) + if err != nil { + return nil, err + } + if !found || + boundDigest.Count != currentDigest.Count || + boundDigest.ABIVersion != currentDigest.ABIVersion || + !bytes.Equal(boundDigest.Hash, currentDigest.Hash) { + return nil, nil + } + return graph, nil } // ActionOp represents a sync operation. diff --git a/pkg/sync/syncer.go b/pkg/sync/syncer.go index d4ff22b04..64236bbac 100644 --- a/pkg/sync/syncer.go +++ b/pkg/sync/syncer.go @@ -273,28 +273,34 @@ func NewExpanderStore(store c1zstore.Store) expand.ExpanderStore { return expanderStoreAdapter{store: store} } -// persistEntitlementGraphToStore moves the preserved graph from the sync token -// into the c1z sidecar when the store supports it, keeping the token skinny. -// On any failure the graph stays in the token (GraphFromToken still works). -func (s *syncer) persistEntitlementGraphToStore(ctx context.Context, syncID string) { - g := s.state.PeekEntitlementGraph() +// persistEntitlementGraphToStore binds the preserved graph to the exact sealed +// grant generation and writes both into the c1z sidecar. +func (s *syncer) persistEntitlementGraphToStore(ctx context.Context, syncID string, g *expand.EntitlementGraph) { if g == nil { return } gs, ok := s.store.(EntitlementGraphStore) if !ok { - return // no sidecar support (SQLite): graph stays in the token + return + } + digestReader, ok := s.store.(c1zstore.GrantGenerationDigestReader) + if !ok { + return } - data, err := expand.MarshalGraphBlob(syncID, g) + digest, found, err := digestReader.GrantGenerationDigest(ctx) + if err != nil || !found { + ctxzap.Extract(ctx).Warn("preserve entitlement graph: sealed grant digest unavailable; graph will not be reusable", zap.Error(err)) + return + } + data, err := expand.MarshalGraphBlobWithGrantDigest(syncID, g, digest) if err != nil { - ctxzap.Extract(ctx).Warn("preserve entitlement graph: marshal failed; keeping graph in token", zap.Error(err)) + ctxzap.Extract(ctx).Warn("preserve entitlement graph: marshal failed", zap.Error(err)) return } if err := gs.PutEntitlementGraphBlob(ctx, data); err != nil { - ctxzap.Extract(ctx).Warn("preserve entitlement graph: sidecar write failed; keeping graph in token", zap.Error(err)) + ctxzap.Extract(ctx).Warn("preserve entitlement graph: sidecar write failed", zap.Error(err)) return } - s.state.ClearEntitlementGraph(ctx) } func (a expanderStoreAdapter) GetEntitlement(ctx context.Context, req *reader_v2.EntitlementsReaderServiceGetEntitlementRequest) (*reader_v2.EntitlementsReaderServiceGetEntitlementResponse, error) { @@ -1089,9 +1095,15 @@ func (s *syncer) Sync(ctx context.Context) error { // expansion: written to the c1z sidecar when the store supports it (token // stays skinny — a whale graph is megabytes), else kept in the final token. // Transient working state is stripped either way; a reload rebuilds it. + var graphToPersist *expand.EntitlementGraph if s.preserveEntitlementGraph { s.state.ClearEntitlementGraphTransientState(ctx) - s.persistEntitlementGraphToStore(ctx, syncID) + _, hasGraphSidecar := s.store.(EntitlementGraphStore) + _, hasGrantDigest := s.store.(c1zstore.GrantGenerationDigestReader) + if hasGraphSidecar && hasGrantDigest { + graphToPersist = s.state.PeekEntitlementGraph() + s.state.ClearEntitlementGraph(ctx) + } } else { s.state.ClearEntitlementGraph(ctx) } @@ -1114,6 +1126,10 @@ func (s *syncer) Sync(ctx context.Context) error { if err != nil { return s.returnSyncError(l, span, err) } + // EndSync built the authoritative whole-file grant digest. Persisting the + // graph now binds it to that exact sealed grant generation. A crash before + // this write leaves no reusable graph and therefore fails safe. + s.persistEntitlementGraphToStore(ctx, syncID, graphToPersist) // The sync is sealed: publish the verification the invariant pass // staged. Marking only after EndSync keeps the marker off unfinished @@ -4033,9 +4049,9 @@ func WithCompactionMergedStore() SyncOpt { } } -// WithPreserveEntitlementGraph keeps the entitlement graph in the final sync -// token instead of clearing it at sync end, so a later incremental expansion -// can reload it rather than rebuilding it from scratch. +// WithPreserveEntitlementGraph preserves the entitlement graph for later +// incremental expansion. Pebble stores it in the c1z sidecar; stores without +// that capability retain it in the final sync token as a legacy fallback. func WithPreserveEntitlementGraph() SyncOpt { return func(s *syncer) { s.preserveEntitlementGraph = true diff --git a/pkg/synccompactor/compactor.go b/pkg/synccompactor/compactor.go index c9c410a44..0c3940638 100644 --- a/pkg/synccompactor/compactor.go +++ b/pkg/synccompactor/compactor.go @@ -47,15 +47,21 @@ type Compactor struct { syncLimit int c1zOptions []dotc1z.C1ZOption skipGrantExpansion bool - // incrementalBaseGraph, when set, enables diff-aware expansion: only changes - // relative to this base-sync graph are expanded. nil (default) = full expansion. + failFastInvariants bool + // incrementalExpansion enables diff-aware expansion. The compactor loads the + // graph itself from entries[0], so callers cannot pair a graph with the wrong + // artifact. incrementalBaseGraph holds that validated, store-loaded graph. // The set of changed entitlements is derived from the applied increments // during expansion, not supplied by the caller. + incrementalExpansion bool incrementalBaseGraph *expand.EntitlementGraph // incrementalExpansionRan records whether the diff-aware path actually // handled expansion (vs falling back to full). Read by tests to prove the // fast path ran rather than silently falling back. incrementalExpansionRan bool + // incrementalTestHook is a package-private fault seam used by crash/retry + // tests. Production compactions leave it nil. + incrementalTestHook func(stage string) error // foldChangedEntitlementIDs: changed-entitlement set collected by the // Pebble fold; nil when no fold ran (derive fallback). foldChangedEntitlementIDs map[string]struct{} @@ -185,7 +191,8 @@ func WithTmpDir(tempDir string) Option { } // WithIncrementalExpansion enables diff-aware grant expansion during compaction. -// baseGraph is the base sync's graph (via sync.GraphFromToken). The set of +// The compactor loads the graph from entries[0] via sync.GraphFromStore; +// missing, stale, incomplete, or inconsistent graphs safely fall back. The set of // entitlements whose membership changed is derived from the applied increments // during expansion (not supplied by the caller), so new members propagate. A // new edge that closes a cycle falls back to full expansion; nil baseGraph @@ -193,9 +200,9 @@ func WithTmpDir(tempDir string) Option { // // Additions-only: a revocation-shaped change (a narrowed edge spec) auto-declines // to full expansion; removals are not propagated incrementally. -func WithIncrementalExpansion(baseGraph *expand.EntitlementGraph) Option { +func WithIncrementalExpansion() Option { return func(c *Compactor) { - c.incrementalBaseGraph = baseGraph + c.incrementalExpansion = true } } @@ -235,6 +242,14 @@ func WithSkipGrantExpansion() Option { } } +// WithFailFastInvariants promotes every ingestion-invariant verdict to a hard +// failure on both incremental and full expansion paths. +func WithFailFastInvariants() Option { + return func(c *Compactor) { + c.failFastInvariants = true + } +} + func NewCompactor(ctx context.Context, outputDir string, compactableSyncs []*CompactableSync, opts ...Option) (*Compactor, func() error, error) { if len(compactableSyncs) < 2 { return nil, nil, ErrNotEnoughFilesToCompact @@ -491,11 +506,21 @@ func (c *Compactor) Compact(ctx context.Context) (*CompactableSync, error) { c.compactedC1z = nil } + if c.incrementalExpansionRan { + if err := c.runIncrementalTestHook("before_publish"); err != nil { + return nil, err + } + } // Move last compacted file to the destination dir finalPath := path.Join(c.destDir, fmt.Sprintf("compacted-%s.c1z", newSyncId)) if err := cpFile(ctx, destFilePath, finalPath); err != nil { return nil, err } + if c.incrementalExpansionRan { + if err := c.runIncrementalTestHook("after_publish"); err != nil { + return nil, err + } + } if !filepath.IsAbs(finalPath) { abs, err := filepath.Abs(finalPath) @@ -603,6 +628,10 @@ func (c *Compactor) doOneCompaction(ctx context.Context, cs *CompactableSync) er // was untouched or restored, and expanded-grant writes are idempotent. var errIncrementalFatal = errors.New("incremental expansion: fatal") +// errIncrementalDroppedEdgeDecline keeps the public revocation contract while +// giving observability a stable, more specific reason. +var errIncrementalDroppedEdgeDecline = fmt.Errorf("%w: dropped edge", expand.ErrIncrementalRevocationDecline) + // Returns (true, nil) when it handled expansion. Errors come in three shapes: // decline sentinels (ErrIncrementalFallback for a cycle, // ErrIncrementalRevocationDecline for a narrowed edge) and plain errors both @@ -611,11 +640,15 @@ var errIncrementalFatal = errors.New("incremental expansion: fatal") // finalization failed and the compaction must fail. Finalization always runs // on a detached context so a run-duration timeout can't abort it. func (c *Compactor) expandGrantsIncremental(ctx context.Context, newSyncId string, compactionStart time.Time) (bool, error) { - // Clone so a failed/declined run never mutates the caller-held base graph - // (a retry with the original must not see never-expanded edges as present). - base, err := c.incrementalBaseGraph.Clone() - if err != nil { - return false, fmt.Errorf("incremental expansion: clone base graph: %w", err) + // Classification only reads the caller-held graph. Defer the whale-sized + // clone until every cheap decline check passes; chronically ineligible + // inputs should not pay O(graph) memory and CPU before falling back. + base := c.incrementalBaseGraph + if err := base.ValidateCompleted(); err != nil { + return false, fmt.Errorf("incremental expansion: invalid base graph: %w", err) + } + if base.HasCollapsedCycles() { + return false, expand.ErrIncrementalFallback } // Bound the walk by the remaining run duration; the walk polls ctx.Err(). @@ -648,6 +681,7 @@ func (c *Compactor) expandGrantsIncremental(ctx context.Context, newSyncId strin // increments) yields one or more edges. New edges are those the base graph // didn't already have expanded. var newEdges []expand.NewEdge + currentBaseNodeEdges := make(map[[2]int]struct{}) for pe, err := range c.compactedC1z.Grants().PendingExpansion(walkCtx) { if err != nil { if endErr := c.restoreEndedSync(ctx); endErr != nil { @@ -660,6 +694,11 @@ func (c *Compactor) expandGrantsIncremental(ctx context.Context, newSyncId strin continue } for _, src := range anno.GetEntitlementIds() { + srcNode := base.GetNode(src) + dstNode := base.GetNode(pe.TargetEntitlementID) + if srcNode != nil && dstNode != nil && srcNode.Id != dstNode.Id { + currentBaseNodeEdges[[2]int{srcNode.Id, dstNode.Id}] = struct{}{} + } baseEdge, inBase := baseGraphEdge(base, src, pe.TargetEntitlementID) curEdge := expand.NewEdge{ SourceEntitlementID: src, @@ -689,6 +728,18 @@ func (c *Compactor) expandGrantsIncremental(ctx context.Context, newSyncId strin } } } + // PendingExpansion describes the complete current edge set. Check the + // reverse direction too: a base edge missing from current data is a + // revocation-shaped change and cannot be applied incrementally. + for _, edge := range base.Edges { + if _, ok := currentBaseNodeEdges[[2]int{edge.SourceID, edge.DestinationID}]; ok { + continue + } + if endErr := c.restoreEndedSync(ctx); endErr != nil { + return false, endErr + } + return false, errIncrementalDroppedEdgeDecline + } // Changed entitlements are derived from the applied increments (their // grants' entitlement ids), not supplied by the caller — trust the data. @@ -702,11 +753,35 @@ func (c *Compactor) expandGrantsIncremental(ctx context.Context, newSyncId strin if len(newEdges) == 0 && len(changedEntitlementIDs) == 0 { // Nothing changed relative to the base — its grants were already merged in. - c.persistGraphSidecar(ctx, base, newSyncId) - return c.finishIncrementalExpansion(ctx) + base, err = base.Clone() + if err != nil { + if endErr := c.restoreEndedSync(ctx); endErr != nil { + return false, endErr + } + return false, fmt.Errorf("incremental expansion: clone base graph: %w", err) + } + verification, err := c.runIncrementalInvariants(walkCtx, newSyncId, syncType) + if err != nil { + if endErr := c.restoreEndedSync(ctx); endErr != nil { + return false, endErr + } + return false, err + } + return c.finishIncrementalExpansion(ctx, newSyncId, base, verification) } - ie := expand.NewIncrementalExpander(sync.NewExpanderStore(c.compactedC1z), base) + base, err = base.Clone() + if err != nil { + if endErr := c.restoreEndedSync(ctx); endErr != nil { + return false, endErr + } + return false, fmt.Errorf("incremental expansion: clone base graph: %w", err) + } + incrementalStore := sync.NewExpanderStore(c.compactedC1z) + if c.incrementalTestHook != nil { + incrementalStore = &incrementalFaultStore{ExpanderStore: incrementalStore, hook: c.incrementalTestHook} + } + ie := expand.NewIncrementalExpander(incrementalStore, base) res, err := ie.ExpandChanges(walkCtx, newEdges, changedEntitlementIDs) if err != nil { // Restore the ended state so the full path re-runs against a consistent @@ -717,12 +792,72 @@ func (c *Compactor) expandGrantsIncremental(ctx context.Context, newSyncId strin } return false, err // sentinel or plain → caller falls back to full } + if err := c.runIncrementalTestHook("after_walk"); err != nil { + return false, err + } ctxzap.Extract(ctx).Info("incremental grant expansion complete", zap.Int("entitlements_walked", len(res.EntitlementsWalked)), zap.Int("grants_written", res.GrantsWritten)) - c.persistGraphSidecar(ctx, base, newSyncId) - return c.finishIncrementalExpansion(ctx) + verification, err := c.runIncrementalInvariants(walkCtx, newSyncId, syncType) + if err != nil { + if endErr := c.restoreEndedSync(ctx); endErr != nil { + return false, endErr + } + return false, err + } + return c.finishIncrementalExpansion(ctx, newSyncId, base, verification) +} + +type incrementalFaultStore struct { + expand.ExpanderStore + hook func(stage string) error + fired bool +} + +func (s *incrementalFaultStore) StoreExpandedGrants(ctx context.Context, grants ...*v2.Grant) error { + if err := s.ExpanderStore.StoreExpandedGrants(ctx, grants...); err != nil { + return err + } + if !s.fired { + s.fired = true + if err := s.hook("mid_expand_write"); err != nil { + return fmt.Errorf("%w: injected failure at mid_expand_write: %w", errIncrementalFatal, err) + } + } + return nil +} + +func (c *Compactor) runIncrementalTestHook(stage string) error { + if c.incrementalTestHook == nil { + return nil + } + if err := c.incrementalTestHook(stage); err != nil { + return fmt.Errorf("%w: injected failure at %s: %w", errIncrementalFatal, stage, err) + } + return nil +} + +func (c *Compactor) runIncrementalInvariants( + ctx context.Context, + syncID string, + syncType connectorstore.SyncType, +) (*c1zstore.IngestInvariantVerification, error) { + if writer, ok := c.compactedC1z.SyncMeta().(c1zstore.IngestInvariantVerificationWriter); ok { + if err := writer.ClearIngestInvariantVerification(ctx, syncID); err != nil { + return nil, fmt.Errorf("incremental expansion: clear invariant verification: %w", err) + } + } + verification, err := sync.RunIngestInvariantsWithVerification(ctx, c.compactedC1z, sync.IngestInvariantsPolicy{ + ActiveSyncID: syncID, + SyncType: syncType, + FailFast: c.failFastInvariants, + CompactionMerge: true, + }) + if err != nil { + return nil, fmt.Errorf("incremental expansion: ingest invariants: %w", err) + } + return verification, nil } // persistGraphSidecar writes the post-expansion graph into the compacted c1z @@ -733,7 +868,16 @@ func (c *Compactor) persistGraphSidecar(ctx context.Context, g *expand.Entitleme if !ok { return } - data, err := expand.MarshalGraphBlob(syncID, g) + digestReader, ok := c.compactedC1z.(c1zstore.GrantGenerationDigestReader) + if !ok { + return + } + digest, found, err := digestReader.GrantGenerationDigest(ctx) + if err != nil || !found { + ctxzap.Extract(ctx).Warn("incremental expansion: sealed grant digest unavailable; graph will not be reusable", zap.Error(err)) + return + } + data, err := expand.MarshalGraphBlobWithGrantDigest(syncID, g, digest) if err == nil { err = gs.PutEntitlementGraphBlob(ctx, data) } @@ -763,14 +907,42 @@ func (c *Compactor) restoreEndedSync(ctx context.Context) error { // abort finalization. All errors here are FATAL (errIncrementalFatal): the // store is being torn down, so falling back to full expansion against it is // not safe. -func (c *Compactor) finishIncrementalExpansion(ctx context.Context) (bool, error) { +func (c *Compactor) finishIncrementalExpansion( + ctx context.Context, + syncID string, + graph *expand.EntitlementGraph, + verification *c1zstore.IngestInvariantVerification, +) (bool, error) { finalizeCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), dotc1z.FinalizeTimeout()) defer cancel() + if err := c.compactedC1z.Cleanup(finalizeCtx); err != nil { + return false, fmt.Errorf("%w: cleanup: %w", errIncrementalFatal, err) + } + if err := c.runIncrementalTestHook("before_end_sync"); err != nil { + return false, err + } if err := c.compactedC1z.EndSync(finalizeCtx); err != nil { return false, fmt.Errorf("%w: end sync: %w", errIncrementalFatal, err) } - if err := c.compactedC1z.Cleanup(finalizeCtx); err != nil { - return false, fmt.Errorf("%w: cleanup: %w", errIncrementalFatal, err) + if err := c.runIncrementalTestHook("after_end_sync"); err != nil { + return false, err + } + c.persistGraphSidecar(finalizeCtx, graph, syncID) + if err := c.runIncrementalTestHook("after_sidecar"); err != nil { + return false, err + } + if verification != nil { + if writer, ok := c.compactedC1z.SyncMeta().(c1zstore.IngestInvariantVerificationWriter); ok { + if err := writer.MarkIngestInvariantsVerified(finalizeCtx, syncID, *verification); err != nil { + ctxzap.Extract(ctx).Warn("incremental expansion: persist invariant verification failed; artifact remains unverified", zap.Error(err)) + } + } + } + if err := c.runIncrementalTestHook("after_marker"); err != nil { + return false, err + } + if err := c.runIncrementalTestHook("before_close"); err != nil { + return false, err } if err := c.compactedC1z.Close(finalizeCtx); err != nil { return false, fmt.Errorf("%w: close: %w", errIncrementalFatal, err) @@ -962,34 +1134,50 @@ func (c *Compactor) expandGrants(ctx context.Context, newSyncId string, compacti // Pebble-only: it reopens the ended compacted sync to write grants, which // only Pebble supports; on other engines we degrade gracefully to full. switch { - case c.incrementalBaseGraph == nil: - // not requested + case !c.incrementalExpansion: + logIncrementalOutcome(ctx, "not_attempted", "not_requested") case c.resolvedEngine() != c1zstore.EnginePebble: - l.Info("incremental expansion is Pebble-only; using full expansion", + logIncrementalOutcome(ctx, "not_attempted", "unsupported_engine", zap.String("engine", string(c.resolvedEngine()))) default: + baseGraph, loadErr := c.loadIncrementalBaseGraph(ctx) + if loadErr != nil { + logIncrementalOutcome(ctx, "fell_back", "base_graph_error", zap.Error(loadErr)) + break + } + if baseGraph == nil { + logIncrementalOutcome(ctx, "fell_back", "base_graph_missing_or_stale") + break + } + c.incrementalBaseGraph = baseGraph done, err := c.expandGrantsIncremental(ctx, newSyncId, compactionStart) switch { case errors.Is(err, errIncrementalFatal): // The store's finalization (or restore-to-ended) failed: it is in an // unknown/torn-down state, so running full expansion against it is // unsafe. Fail the compaction. + logIncrementalOutcome(ctx, "failed", "finalization_error", zap.Error(err)) return fmt.Errorf("incremental grant expansion: %w", err) + case errors.Is(err, errIncrementalDroppedEdgeDecline): + logIncrementalOutcome(ctx, "declined", "dropped_edge") case errors.Is(err, expand.ErrIncrementalRevocationDecline): // Named revocation hook (#6): today declines to full; a future // tombstone stage flips this one site to apply deletions. - l.Info("incremental expansion declined (revocation-shaped change); falling back to full expansion") + logIncrementalOutcome(ctx, "declined", "revocation") + case errors.Is(err, expand.ErrIncrementalDenseChangeDecline): + logIncrementalOutcome(ctx, "declined", "dense_change") case errors.Is(err, expand.ErrIncrementalFallback): // New edge closed a cycle: full expansion handles cycles correctly. - l.Info("incremental expansion declined (cycle); falling back to full expansion") + logIncrementalOutcome(ctx, "declined", "cycle") case err != nil: // Pre-write or restored-state failure: the store is back in the // ended state the full path expects, so falling back is safe. - l.Warn("incremental expansion failed; falling back to full expansion", zap.Error(err)) + logIncrementalOutcome(ctx, "fell_back", "incremental_error", zap.Error(err)) case done: // Incremental path already ended + closed the store; caller clears // c.compactedC1z after return, same as the full path. c.incrementalExpansionRan = true + logIncrementalOutcome(ctx, "succeeded", "none") return nil } } @@ -1015,12 +1203,15 @@ func (c *Compactor) expandGrants(ctx context.Context, newSyncId string, compacti // pinned by TestCompactionExpandToleratesMergeManufacturedExclusionConflicts. sync.WithCompactionMergedStore(), } + if c.failFastInvariants { + syncOpts = append(syncOpts, sync.WithFailFastInvariants()) + } // Keep the artifact's graph sidecar coherent with this full expansion: // opted-in compactions preserve a fresh graph (so the incremental chain // heals after a fallback); otherwise drop any sidecar inherited from a // fold-copied base. - if c.incrementalBaseGraph != nil { + if c.incrementalExpansion { syncOpts = append(syncOpts, sync.WithPreserveEntitlementGraph()) } else if gs, ok := c.compactedC1z.(sync.EntitlementGraphStore); ok { if err := gs.DeleteEntitlementGraphBlob(ctx); err != nil { @@ -1059,3 +1250,42 @@ func (c *Compactor) expandGrants(ctx context.Context, newSyncId string, compacti } return nil } + +func logIncrementalOutcome(ctx context.Context, outcome, reason string, fields ...zap.Field) { + fields = append([]zap.Field{ + zap.String("incremental_expansion_outcome", outcome), + zap.String("incremental_expansion_reason", reason), + }, fields...) + ctxzap.Extract(ctx).Info("incremental grant expansion outcome", fields...) +} + +func (c *Compactor) loadIncrementalBaseGraph(ctx context.Context) (*expand.EntitlementGraph, error) { + if len(c.entries) == 0 || c.entries[0] == nil || c.entries[0].SyncID == "" { + return nil, fmt.Errorf("incremental expansion: compaction base is missing") + } + store, err := dotc1z.NewStore(ctx, c.entries[0].FilePath, + dotc1z.WithReadOnly(true), dotc1z.WithTmpDir(c.tmpDir)) + if err != nil { + return nil, fmt.Errorf("incremental expansion: open base graph store: %w", err) + } + run, runErr := store.SyncMeta().LatestFinishedSyncOfAnyType(ctx) + if runErr != nil { + _ = store.Close(ctx) + return nil, fmt.Errorf("incremental expansion: load base verification: %w", runErr) + } + if run.ID != c.entries[0].SyncID || + !run.IsVerified() || + run.Generation != sync.IngestInvariantGeneration { + _ = store.Close(ctx) + return nil, fmt.Errorf("incremental expansion: base grant generation is not verified") + } + graph, graphErr := sync.GraphFromStore(ctx, store, c.entries[0].SyncID) + closeErr := store.Close(ctx) + if graphErr != nil { + return nil, fmt.Errorf("incremental expansion: load base graph: %w", graphErr) + } + if closeErr != nil { + return nil, fmt.Errorf("incremental expansion: close base graph store: %w", closeErr) + } + return graph, nil +} diff --git a/pkg/synccompactor/compactor_pebble.go b/pkg/synccompactor/compactor_pebble.go index 0481f37a1..9152f284e 100644 --- a/pkg/synccompactor/compactor_pebble.go +++ b/pkg/synccompactor/compactor_pebble.go @@ -684,6 +684,13 @@ func (c *Compactor) compactPebbleFold(ctx context.Context) (string, error) { // a lineage link would dangle, and the rebuild path's compacted // output carries no parent either. newSyncID := ksuid.New().String() + // The folded store is copied from the base, but the graph sidecar is + // stamped with that base sync ID and may no longer describe merged data. + // Drop it before publishing the fresh sync; a following expansion writes a + // new graph, while skip-expansion artifacts safely fall back next time. + if err := destEng.DeleteEntitlementGraphSidecar(ctx); err != nil { + return "", fmt.Errorf("compactPebbleFold: delete inherited entitlement graph: %w", err) + } baseRec.SetSyncId(newSyncID) baseRec.SetParentSyncId("") baseRec.SetType(unionType) From 9fd339be58b8f3632dca5b13beb341fedaff56d0 Mon Sep 17 00:00:00 2001 From: manojacs Date: Tue, 4 Aug 2026 06:48:37 +0000 Subject: [PATCH 07/15] test(sync): add incremental expansion verification instruments Add full-vs-incremental differential oracles, an independent fixed-point access model, mutation-adequacy controls, multi-generation reuse, k-way parity, edge-filter transitions, graph/grant digest mismatch, compatibility healing, and SQLite fallback coverage. Exercise artifact durability with real subprocess kills and fault injection across graph persistence, seal, verification marker, close, and publication boundaries. Add bounded performance gates that keep sparse changes incremental and decline dense closures before grant writes. Wire compatibility, crash, fuzz, soak, and performance targets into the Makefile. The full repository suite, race checks, compatibility matrix, crash/retry suite, performance gate, and extended differential fuzz runs pass. Co-authored-by: c1-squire-dev[bot] --- Makefile | 19 + cmd/baton-compat-harness/driver_test.go | 224 ++++++++- cmd/baton-compat-harness/graph_modes_new.go | 274 +++++++++++ cmd/baton-compat-harness/main.go | 145 +++++- cmd/baton-crash-harness/driver_test.go | 4 +- pkg/c1zsanitize/sanitize_pebble_test.go | 42 ++ .../engine/pebble/choke_point_meta_test.go | 4 +- .../engine/pebble/errorfs_sweep_test.go | 118 ++--- pkg/sync/expand/graph_blob_test.go | 130 +++++ pkg/sync/expand/incremental_benchmark_test.go | 209 ++++++++ .../expand/incremental_differential_test.go | 461 ++++++++++++++++++ .../expand/incremental_exhaustive_test.go | 90 ++++ pkg/sync/expand/incremental_test.go | 17 + pkg/sync/graph_compatibility_matrix_test.go | 73 +++ pkg/sync/graph_from_store_test.go | 27 +- pkg/sync/graph_golden_corpus_test.go | 169 +++++++ pkg/sync/ingest_invariants.go | 3 +- pkg/synccompactor/compactor_fold_test.go | 44 ++ .../incremental_benchmark_test.go | 80 +++ pkg/synccompactor/incremental_closure_test.go | 430 ++++++++++++++++ .../incremental_differential_test.go | 243 +++++++++ .../incremental_expansion_test.go | 390 +++++++++++++-- .../incremental_hardening_test.go | 297 +++++++++++ 23 files changed, 3375 insertions(+), 118 deletions(-) create mode 100644 cmd/baton-compat-harness/graph_modes_new.go create mode 100644 pkg/sync/expand/incremental_benchmark_test.go create mode 100644 pkg/sync/expand/incremental_differential_test.go create mode 100644 pkg/sync/expand/incremental_exhaustive_test.go create mode 100644 pkg/sync/graph_compatibility_matrix_test.go create mode 100644 pkg/sync/graph_golden_corpus_test.go create mode 100644 pkg/synccompactor/incremental_benchmark_test.go create mode 100644 pkg/synccompactor/incremental_closure_test.go create mode 100644 pkg/synccompactor/incremental_differential_test.go create mode 100644 pkg/synccompactor/incremental_hardening_test.go diff --git a/Makefile b/Makefile index 863809573..b880bc793 100644 --- a/Makefile +++ b/Makefile @@ -12,6 +12,7 @@ SOAK_ITERATIONS ?= 25 CHAOS_ITERATIONS ?= 25 NIGHTLY_FUZZ_TIME ?= 5m NIGHTLY_DIFFERENTIAL_TIME ?= 10m +INCREMENTAL_SOAK_TIME ?= 10m .DEFAULT_GOAL := help @@ -74,7 +75,14 @@ test: ## Run the Go test suite used by CI. # sets, not a number this is expected to approach. .PHONY: compat-check compat-check: ## Exchange checkpoints with a pinned older SDK. + go test -count=1 -run 'Test(EntitlementGraphTokenCompatibilityMatrix|GraphFromStore)' ./pkg/sync + go test -count=1 -run 'TestCompactorGraphCompatibilityHealing' ./pkg/synccompactor BATON_COMPAT=1 go test -v -count=1 -timeout=45m -run TestCheckpointCompatAcrossSDKVersions ./cmd/baton-compat-harness + $(MAKE) graph-compat-check + +.PHONY: graph-compat-check +graph-compat-check: ## Exchange graph-sidecar c1z artifacts with a pinned older SDK. + BATON_GRAPH_COMPAT=1 go test -v -count=1 -timeout=30m -run 'Test(GraphReuseCompatAcrossSDKVersions|DefaultPathPerformanceAgainstPinnedMain)' ./cmd/baton-compat-harness # Real-binary interruption instrument: builds a deterministic connector from # this tree, runs budget-bounded sync sessions, SIGKILLs them at varied @@ -83,6 +91,7 @@ compat-check: ## Exchange checkpoints with a pinned older SDK. .PHONY: crash-check crash-check: ## Exercise cross-process checkpoint/resume under hard kills. BATON_DEMO_CRASH=1 go test -v -count=1 -timeout=30m -run TestCrashResumeRealConnector ./cmd/baton-crash-harness + go test -v -count=1 -run 'TestIncrementalExpansion(ProcessKillRetry|CrashRetry)' ./pkg/synccompactor .PHONY: demo-crash-check demo-crash-check: crash-check ## Deprecated alias for crash-check. @@ -218,6 +227,7 @@ race-shard-matrix-audit: ## Verify nightly.yaml runs exactly the declared shards .PHONY: fuzz-smoke fuzz-smoke: ## Run each native Go fuzzer for FUZZ_TIME (default 30s). + go test -run '^$$' -fuzz '^FuzzIncrementalVsFullExpansion$$' -fuzztime=$(FUZZ_TIME) ./pkg/sync/expand go test -run '^$$' -fuzz '^FuzzCondenseFWBW_Cancellation$$' -fuzztime=$(FUZZ_TIME) ./pkg/sync/expand/scc go test -run '^$$' -fuzz '^FuzzCondenseFWBW_FromBytes$$' -fuzztime=$(FUZZ_TIME) ./pkg/sync/expand/scc @@ -229,6 +239,15 @@ differential-check: ## Differential-fuzz SQLite and Pebble for DIFFERENTIAL_TIME bench-smoke: ## Run the bounded checkpoint cost benchmarks once. go test -run '^$$' -bench 'Benchmark(CheckpointToken|SpawnedCursorAdmission)' -benchtime=1x -benchmem ./pkg/sync +.PHONY: incremental-performance-check +incremental-performance-check: ## Enforce 100k-node incremental allocation/work gates. + BATON_INCREMENTAL_PERF=1 go test -v -count=1 -timeout=30m -run '^TestIncrementalPerformanceGates$$' ./pkg/sync/expand + +.PHONY: incremental-soak +incremental-soak: ## Run the incremental differential fuzzers for INCREMENTAL_SOAK_TIME. + BATON_EXPAND_FUZZ_DURATION=$(INCREMENTAL_SOAK_TIME) go test -count=1 -timeout=30m -run '^TestFullPipelineDifferentialFuzz$$' ./pkg/sync/expand + go test -run '^$$' -fuzz '^FuzzIncrementalVsFullExpansion$$' -fuzztime=$(INCREMENTAL_SOAK_TIME) ./pkg/sync/expand + .PHONY: bench bench: ## Run curated checkpoint and medium full-sync benchmarks. go test -run '^$$' -bench 'Benchmark(CheckpointToken|SpawnedCursorAdmission)' -benchmem ./pkg/sync diff --git a/cmd/baton-compat-harness/driver_test.go b/cmd/baton-compat-harness/driver_test.go index 023ba6005..579b409df 100644 --- a/cmd/baton-compat-harness/driver_test.go +++ b/cmd/baton-compat-harness/driver_test.go @@ -23,10 +23,13 @@ package main import ( "context" + "crypto/sha256" "encoding/json" + "fmt" "os" "os/exec" "path/filepath" + "sort" "strings" "testing" "time" @@ -69,10 +72,18 @@ func TestCompatHarnessBuildsAgainstHead(t *testing.T) { } func runHarness(t *testing.T, bin, mode, c1zPath string) compatDriverResult { + return runHarnessOut(t, bin, mode, c1zPath, "") +} + +func runHarnessOut(t *testing.T, bin, mode, c1zPath, outPath string) compatDriverResult { t.Helper() ctx, cancel := context.WithTimeout(context.Background(), 3*time.Minute) defer cancel() - cmd := exec.CommandContext(ctx, bin, "-mode", mode, "-c1z", c1zPath) + args := []string{"-mode", mode, "-c1z", c1zPath} + if outPath != "" { + args = append(args, "-out", outPath) + } + cmd := exec.CommandContext(ctx, bin, args...) output, err := cmd.CombinedOutput() require.NoError(t, err, "%s -mode %s:\n%s", bin, mode, output) @@ -91,17 +102,97 @@ func runHarness(t *testing.T, bin, mode, c1zPath string) compatDriverResult { // compatDriverResult mirrors compatResult in main.go (excluded from this // compilation by its build tag). type compatDriverResult struct { - Mode string `json:"mode"` - SyncErr string `json:"sync_err"` - NotComplete bool `json:"not_complete"` - Resources int `json:"resources"` - Ents int `json:"entitlements"` - Grants int `json:"grants"` - CountErr string `json:"count_err"` - UnfinishedRuns int `json:"unfinished_runs"` - TokenLen int `json:"token_len"` - TokenSpawned bool `json:"token_spawned"` - TokenTypeScoped bool `json:"token_type_scoped"` + Mode string `json:"mode"` + SyncErr string `json:"sync_err"` + NotComplete bool `json:"not_complete"` + Resources int `json:"resources"` + Ents int `json:"entitlements"` + Grants int `json:"grants"` + CountErr string `json:"count_err"` + UnfinishedRuns int `json:"unfinished_runs"` + TokenLen int `json:"token_len"` + TokenSpawned bool `json:"token_spawned"` + TokenTypeScoped bool `json:"token_type_scoped"` + GraphPresent bool `json:"graph_present"` + GraphReusable bool `json:"graph_reusable"` + GraphErr string `json:"graph_err"` + IncrementalRan bool `json:"incremental_ran"` + IncrementalOutcome string `json:"incremental_outcome"` + IncrementalReason string `json:"incremental_reason"` + IncrementalError string `json:"incremental_error"` + ArtifactPath string `json:"artifact_path"` + AllocatedBytes uint64 `json:"allocated_bytes"` +} + +func TestDefaultPathPerformanceAgainstPinnedMain(t *testing.T) { + if os.Getenv("BATON_GRAPH_COMPAT") == "" { + t.Skip("pinned-main performance ratchet; set BATON_GRAPH_COMPAT=1") + } + mainRef := os.Getenv("BATON_GRAPH_COMPAT_MAIN_REF") + if mainRef == "" { + mainRef = "10a6da053799febd092bfffeb7c5c6ec195dca0c" + } + root := repoRoot(t) + tmp := t.TempDir() + mainTree := filepath.Join(tmp, "main-tree") + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) + defer cancel() + output, err := exec.CommandContext(ctx, "git", "-C", root, "worktree", "add", "--detach", mainTree, mainRef).CombinedOutput() // #nosec G702 -- local test ref and TempDir paths only. + require.NoError(t, err, "git worktree add %s:\n%s", mainRef, output) + t.Cleanup(func() { + rmCtx, rmCancel := context.WithTimeout(context.Background(), time.Minute) + defer rmCancel() + _, _ = exec.CommandContext(rmCtx, "git", "-C", root, "worktree", "remove", "--force", mainTree).CombinedOutput() + }) + src, err := os.ReadFile(filepath.Join(root, "cmd", "baton-compat-harness", "main.go")) + require.NoError(t, err) + mainHarnessDir := filepath.Join(mainTree, "cmd", "baton-compat-harness") + require.NoError(t, os.MkdirAll(mainHarnessDir, 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(mainHarnessDir, "main.go"), src, 0o600)) // #nosec G703 -- destination is inside the test TempDir worktree. + + candidateBin := filepath.Join(tmp, "candidate") + mainBin := filepath.Join(tmp, "main") + buildHarness(t, root, candidateBin) + buildHarness(t, mainTree, mainBin) + base := filepath.Join(tmp, "base.c1z") + baseResult := runHarness(t, mainBin, "resume", base) + require.Empty(t, baseResult.SyncErr) + + measure := func(t *testing.T, bin, label string) uint64 { + t.Helper() + values := make([]uint64, 0, 5) + for i := 0; i < 5; i++ { + input := filepath.Join(tmp, fmt.Sprintf("%s-input-%d.c1z", label, i)) + copyCompatArtifact(t, base, input) + out := filepath.Join(tmp, fmt.Sprintf("%s-output-%d.c1z", label, i)) + result := runHarnessOut(t, bin, "graph-default-compact", input, out) + require.Equal(t, wantResources, result.Resources) + require.Equal(t, wantEnts, result.Ents) + require.Equal(t, wantGrants, result.Grants) + require.Positive(t, result.AllocatedBytes) + values = append(values, result.AllocatedBytes) + } + sort.Slice(values, func(i, j int) bool { return values[i] < values[j] }) + return values[len(values)/2] + } + mainAlloc := measure(t, mainBin, "main") + candidateAlloc := measure(t, candidateBin, "candidate") + require.LessOrEqual(t, candidateAlloc, mainAlloc*110/100, + "default compaction allocation regression: candidate=%d main=%d", candidateAlloc, mainAlloc) +} + +func copyCompatArtifact(t *testing.T, src, dst string) { + t.Helper() + data, err := os.ReadFile(src) + require.NoError(t, err) + require.NoError(t, os.WriteFile(dst, data, 0o600)) // #nosec G703 -- callers provide paths inside their test TempDir. +} + +func compatArtifactDigest(t *testing.T, path string) [sha256.Size]byte { + t.Helper() + data, err := os.ReadFile(path) + require.NoError(t, err) + return sha256.Sum256(data) } func TestCheckpointCompatAcrossSDKVersions(t *testing.T) { @@ -122,8 +213,7 @@ func TestCheckpointCompatAcrossSDKVersions(t *testing.T) { // Materialize the old tree. worktree (vs. clone) shares objects and // works offline; --detach avoids claiming the ref. oldTree := filepath.Join(tmp, "old-tree") - //nolint:gosec // the ref comes from the developer's own environment; this is a local test driver - wtAdd := exec.CommandContext(ctx, "git", "-C", root, "worktree", "add", "--detach", oldTree, oldRef) + wtAdd := exec.CommandContext(ctx, "git", "-C", root, "worktree", "add", "--detach", oldTree, oldRef) // #nosec G702 -- local test ref and TempDir paths only. output, err := wtAdd.CombinedOutput() require.NoError(t, err, "git worktree add %s:\n%s", oldRef, output) t.Cleanup(func() { @@ -141,8 +231,7 @@ func TestCheckpointCompatAcrossSDKVersions(t *testing.T) { require.NoError(t, err) oldHarnessDir := filepath.Join(oldTree, "cmd", "baton-compat-harness") require.NoError(t, os.MkdirAll(oldHarnessDir, 0o755)) - //nolint:gosec // the path is rooted in this test's own TempDir worktree - require.NoError(t, os.WriteFile(filepath.Join(oldHarnessDir, "main.go"), src, 0o600)) + require.NoError(t, os.WriteFile(filepath.Join(oldHarnessDir, "main.go"), src, 0o600)) // #nosec G703 -- destination is inside the test TempDir worktree. newBin := filepath.Join(tmp, "harness-new") oldBin := filepath.Join(tmp, "harness-old") @@ -189,3 +278,106 @@ func TestCheckpointCompatAcrossSDKVersions(t *testing.T) { }) } } + +func TestGraphReuseCompatAcrossSDKVersions(t *testing.T) { + if os.Getenv("BATON_GRAPH_COMPAT") == "" { + t.Skip("graph-sidecar compatibility matrix; set BATON_GRAPH_COMPAT=1") + } + oldRef := os.Getenv("BATON_GRAPH_COMPAT_OLD_REF") + if oldRef == "" { + oldRef = "v0.20.6" + } + root := repoRoot(t) + tmp := t.TempDir() + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) + defer cancel() + + oldTree := filepath.Join(tmp, "old-tree") + wtAdd := exec.CommandContext(ctx, "git", "-C", root, "worktree", "add", "--detach", oldTree, oldRef) // #nosec G702 -- local test ref and TempDir paths only. + output, err := wtAdd.CombinedOutput() + require.NoError(t, err, "git worktree add %s:\n%s", oldRef, output) + t.Cleanup(func() { + rmCtx, rmCancel := context.WithTimeout(context.Background(), time.Minute) + defer rmCancel() + _, _ = exec.CommandContext(rmCtx, "git", "-C", root, "worktree", "remove", "--force", oldTree).CombinedOutput() + }) + + src, err := os.ReadFile(filepath.Join(root, "cmd", "baton-compat-harness", "main.go")) + require.NoError(t, err) + oldHarnessDir := filepath.Join(oldTree, "cmd", "baton-compat-harness") + require.NoError(t, os.MkdirAll(oldHarnessDir, 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(oldHarnessDir, "main.go"), src, 0o600)) // #nosec G703 -- destination is inside the test TempDir worktree. + + newBin := filepath.Join(tmp, "graph-harness-new") + oldBin := filepath.Join(tmp, "graph-harness-old") + buildHarness(t, root, newBin) + buildHarness(t, oldTree, oldBin) + + requireComplete := func(t *testing.T, result compatDriverResult) { + t.Helper() + require.Empty(t, result.CountErr) + require.Equal(t, wantResources, result.Resources) + require.Equal(t, wantEnts, result.Ents) + require.Equal(t, wantGrants, result.Grants) + } + + // M2: old -> old baseline. + oldBase := filepath.Join(tmp, "old-base.c1z") + requireComplete(t, runHarness(t, oldBin, "resume", oldBase)) + + // M3: old -> new. No graph exists, so the candidate must full-fallback. + newFromOld := filepath.Join(tmp, "new-from-old.c1z") + oldToNew := runHarnessOut(t, newBin, "graph-compact", oldBase, newFromOld) + requireComplete(t, oldToNew) + require.False(t, oldToNew.IncrementalRan) + + // M6: an old reader can read the candidate output. + requireComplete(t, runHarness(t, oldBin, "graph-inspect", newFromOld)) + + // Candidate seed proves the new sidecar premise before any cross-version row. + newSeed := filepath.Join(tmp, "new-seed.c1z") + seed := runHarness(t, newBin, "graph-seed", newSeed) + requireComplete(t, seed) + require.True(t, seed.GraphPresent) + require.True(t, seed.GraphReusable) + + // M1: new -> new admits reuse and remains complete. + newFromNew := filepath.Join(tmp, "new-from-new.c1z") + newToNew := runHarnessOut(t, newBin, "graph-compact", newSeed, newFromNew) + requireComplete(t, newToNew) + require.True(t, newToNew.IncrementalRan, "valid candidate sidecar must not vacuously full-fallback: outcome=%s reason=%s error=%s", + newToNew.IncrementalOutcome, newToNew.IncrementalReason, newToNew.IncrementalError) + + // M4: new -> old read. The unknown sidecar must not break or mutate the + // old-visible artifact. + before := compatArtifactDigest(t, newSeed) + requireComplete(t, runHarness(t, oldBin, "graph-inspect", newSeed)) + require.Equal(t, before, compatArtifactDigest(t, newSeed)) + + // M5/M9: new -> old fold-style full compaction -> new. The candidate + // must not trust metadata copied or transformed by the old binary. + roundTripInput := filepath.Join(tmp, "roundtrip-input.c1z") + copyCompatArtifact(t, newSeed, roundTripInput) + oldTransformed := filepath.Join(tmp, "old-transformed.c1z") + requireComplete(t, runHarnessOut(t, oldBin, "graph-old-compact", roundTripInput, oldTransformed)) + newAfterOld := filepath.Join(tmp, "new-after-old.c1z") + afterOld := runHarnessOut(t, newBin, "graph-compact", oldTransformed, newAfterOld) + requireComplete(t, afterOld) + require.False(t, afterOld.IncrementalRan, "old transformation cannot certify the candidate graph generation") + + // M7/M8: malformed, unknown-version, and foreign-sync sidecars all + // fail closed to one complete full expansion. + for _, mutation := range []string{"graph-corrupt", "graph-unknown-version", "graph-foreign-sync"} { + t.Run(mutation, func(t *testing.T) { + mutated := filepath.Join(t.TempDir(), "mutated.c1z") + copyCompatArtifact(t, newSeed, mutated) + mutatedState := runHarness(t, newBin, mutation, mutated) + require.True(t, mutatedState.GraphPresent) + require.False(t, mutatedState.GraphReusable) + out := filepath.Join(t.TempDir(), "repaired.c1z") + repaired := runHarnessOut(t, newBin, "graph-compact", mutated, out) + requireComplete(t, repaired) + require.False(t, repaired.IncrementalRan) + }) + } +} diff --git a/cmd/baton-compat-harness/graph_modes_new.go b/cmd/baton-compat-harness/graph_modes_new.go new file mode 100644 index 000000000..1f14953e0 --- /dev/null +++ b/cmd/baton-compat-harness/graph_modes_new.go @@ -0,0 +1,274 @@ +//go:build compatharness + +package main + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "os" + "path/filepath" + + "github.com/grpc-ecosystem/go-grpc-middleware/logging/zap/ctxzap" + "go.uber.org/zap" + "go.uber.org/zap/zapcore" + + "github.com/conductorone/baton-sdk/pkg/dotc1z" + "github.com/conductorone/baton-sdk/pkg/dotc1z/c1zstore" + sdksync "github.com/conductorone/baton-sdk/pkg/sync" + "github.com/conductorone/baton-sdk/pkg/sync/expand" + "github.com/conductorone/baton-sdk/pkg/synccompactor" +) + +func init() { + graphCompatHandler = runGraphCompatMode +} + +func runGraphCompatMode(ctx context.Context, mode, c1zPath, outPath string) (compatResult, error) { + switch mode { + case "graph-seed": + return graphCompatSeed(ctx, c1zPath) + case "graph-inspect": + return graphCompatInspect(ctx, c1zPath) + case "graph-compact": + return graphCompatCompact(ctx, c1zPath, outPath, true) + case "graph-compact-full": + return graphCompatCompact(ctx, c1zPath, outPath, false) + case "graph-old-compact": + return graphCompatCompact(ctx, c1zPath, outPath, false) + case "graph-corrupt": + return graphCompatMutate(ctx, c1zPath, "corrupt") + case "graph-unknown-version": + return graphCompatMutate(ctx, c1zPath, "unknown-version") + case "graph-foreign-sync": + return graphCompatMutate(ctx, c1zPath, "foreign-sync") + default: + return compatResult{}, fmt.Errorf("unknown graph compatibility mode %q", mode) + } +} + +func graphCompatMutate(ctx context.Context, path, mutation string) (compatResult, error) { + store, err := dotc1z.NewStore(ctx, path, dotc1z.WithTmpDir(os.TempDir())) + if err != nil { + return compatResult{}, err + } + graphStore, ok := store.(sdksync.EntitlementGraphStore) + if !ok { + _ = store.Close(ctx) + return compatResult{}, fmt.Errorf("candidate store lacks graph sidecar capability") + } + data, err := graphStore.GetEntitlementGraphBlob(ctx) + if err != nil { + _ = store.Close(ctx) + return compatResult{}, err + } + switch mutation { + case "corrupt": + data = []byte("{truncated") + case "unknown-version", "foreign-sync": + var envelope map[string]any + if err := json.Unmarshal(data, &envelope); err != nil { + _ = store.Close(ctx) + return compatResult{}, err + } + if mutation == "unknown-version" { + envelope["format_version"] = float64(999) + } else { + envelope["sync_id"] = "foreign-sync" + } + data, err = json.Marshal(envelope) + if err != nil { + _ = store.Close(ctx) + return compatResult{}, err + } + } + if err := graphStore.PutEntitlementGraphBlob(ctx, data); err != nil { + _ = store.Close(ctx) + return compatResult{}, err + } + if err := store.Close(ctx); err != nil { + return compatResult{}, err + } + return graphCompatInspect(ctx, path) +} + +func graphCompatSeed(ctx context.Context, path string) (compatResult, error) { + connector, err := newCompatConnector() + if err != nil { + return compatResult{}, err + } + store, err := dotc1z.NewStore(ctx, path, + dotc1z.WithEngine(c1zstore.EnginePebble), dotc1z.WithTmpDir(os.TempDir())) + if err != nil { + return compatResult{}, err + } + syncer, err := sdksync.NewSyncer(ctx, connector, + sdksync.WithConnectorStore(store), + sdksync.WithTmpDir(os.TempDir()), + sdksync.WithWorkerCount(2), + ) + if err != nil { + return compatResult{}, err + } + if err := syncer.Sync(ctx); err != nil { + return compatResult{}, err + } + if err := syncer.Close(ctx); err != nil { + return compatResult{}, err + } + + // Attach the candidate graph envelope to a real sealed artifact. The graph + // is deliberately edge-free: every stored grant is direct, so this is the + // exact completed graph for the fixture rather than a synthetic mismatch. + store, err = dotc1z.NewStore(ctx, path, dotc1z.WithTmpDir(os.TempDir())) + if err != nil { + return compatResult{}, err + } + run, err := store.SyncMeta().LatestFinishedSyncOfAnyType(ctx) + if err != nil { + _ = store.Close(ctx) + return compatResult{}, err + } + graph := expand.NewEntitlementGraph(ctx) + for _, entitlement := range connector.entsByRes { + graph.AddEntitlementID(entitlement.GetId()) + } + graph.MarkExpansionComplete() + graph.Loaded = true + graph.HasNoCycles = true + digestReader, ok := store.(c1zstore.GrantGenerationDigestReader) + if !ok { + _ = store.Close(ctx) + return compatResult{}, fmt.Errorf("candidate store lacks grant generation digest") + } + digest, found, err := digestReader.GrantGenerationDigest(ctx) + if err != nil || !found { + _ = store.Close(ctx) + return compatResult{}, fmt.Errorf("candidate grant digest unavailable: found=%t err=%w", found, err) + } + data, err := expand.MarshalGraphBlobWithGrantDigest(run.ID, graph, digest) + if err != nil { + _ = store.Close(ctx) + return compatResult{}, err + } + graphStore, ok := store.(sdksync.EntitlementGraphStore) + if !ok { + _ = store.Close(ctx) + return compatResult{}, fmt.Errorf("candidate store lacks graph sidecar capability") + } + if err := graphStore.PutEntitlementGraphBlob(ctx, data); err != nil { + _ = store.Close(ctx) + return compatResult{}, err + } + if err := store.Close(ctx); err != nil { + return compatResult{}, err + } + return graphCompatInspect(ctx, path) +} + +func graphCompatInspect(ctx context.Context, path string) (compatResult, error) { + result := compatResult{Mode: "graph-inspect", ArtifactPath: path} + store, err := dotc1z.NewStore(ctx, path, dotc1z.WithTmpDir(os.TempDir())) + if err != nil { + return result, err + } + run, err := store.SyncMeta().LatestFinishedSyncOfAnyType(ctx) + if err != nil { + _ = store.Close(ctx) + return result, err + } + if graphStore, ok := store.(sdksync.EntitlementGraphStore); ok { + data, graphErr := graphStore.GetEntitlementGraphBlob(ctx) + if graphErr != nil { + _ = store.Close(ctx) + return result, graphErr + } + result.GraphPresent = len(data) > 0 + } + graph, err := sdksync.GraphFromStore(ctx, store, run.ID) + if err != nil { + result.GraphErr = err.Error() + err = nil + } + result.GraphReusable = graph != nil + result.Resources, result.Ents, result.Grants, err = countList(ctx, store) + if closeErr := store.Close(ctx); err == nil { + err = closeErr + } + return result, err +} + +func graphCompatCompact(ctx context.Context, inputPath, outPath string, incremental bool) (compatResult, error) { + if outPath == "" { + return compatResult{}, fmt.Errorf("graph compact mode requires -out") + } + input, err := dotc1z.NewStore(ctx, inputPath, + dotc1z.WithReadOnly(true), dotc1z.WithTmpDir(os.TempDir())) + if err != nil { + return compatResult{}, err + } + run, err := input.SyncMeta().LatestFinishedSyncOfAnyType(ctx) + if closeErr := input.Close(ctx); err == nil { + err = closeErr + } + if err != nil { + return compatResult{}, err + } + + var logBytes bytes.Buffer + core := zapcore.NewCore(zapcore.NewJSONEncoder(zap.NewProductionEncoderConfig()), + zapcore.AddSync(&logBytes), zap.InfoLevel) + compactCtx := ctxzap.ToContext(ctx, zap.New(core)) + opts := []synccompactor.Option{ + synccompactor.WithTmpDir(os.TempDir()), + synccompactor.WithEngine(c1zstore.EnginePebble), + } + if incremental { + opts = append(opts, synccompactor.WithIncrementalExpansion()) + } + outputDir := filepath.Dir(outPath) + empty, err := graphCompatEmptyPartial(ctx, outputDir) + if err != nil { + return compatResult{}, err + } + compactor, cleanup, err := synccompactor.NewCompactor(compactCtx, outputDir, + []*synccompactor.CompactableSync{{FilePath: inputPath, SyncID: run.ID}, empty}, opts...) + if err != nil { + return compatResult{}, err + } + defer cleanup() + out, err := compactor.Compact(compactCtx) + if err != nil { + return compatResult{}, err + } + if out.FilePath != outPath { + _ = os.Remove(outPath) + if err := os.Rename(out.FilePath, outPath); err != nil { + return compatResult{}, err + } + } + result, err := graphCompatInspect(ctx, outPath) + if err != nil { + return compatResult{}, err + } + result.Mode = "graph-compact" + result.ArtifactPath = outPath + for _, line := range bytes.Split(bytes.TrimSpace(logBytes.Bytes()), []byte{'\n'}) { + var entry map[string]any + if json.Unmarshal(line, &entry) != nil || entry["msg"] != "incremental grant expansion outcome" { + continue + } + result.IncrementalRan = entry["incremental_expansion_outcome"] == "succeeded" + if value, ok := entry["incremental_expansion_outcome"].(string); ok { + result.IncrementalOutcome = value + } + if value, ok := entry["incremental_expansion_reason"].(string); ok { + result.IncrementalReason = value + } + if value, ok := entry["error"].(string); ok { + result.IncrementalError = value + } + } + return result, nil +} diff --git a/cmd/baton-compat-harness/main.go b/cmd/baton-compat-harness/main.go index 1c54f8581..bb6c32203 100644 --- a/cmd/baton-compat-harness/main.go +++ b/cmd/baton-compat-harness/main.go @@ -38,6 +38,8 @@ import ( "flag" "fmt" "os" + "path/filepath" + "runtime" "strings" "time" @@ -47,9 +49,11 @@ import ( v2 "github.com/conductorone/baton-sdk/pb/c1/connector/v2" "github.com/conductorone/baton-sdk/pkg/annotations" + "github.com/conductorone/baton-sdk/pkg/connectorstore" "github.com/conductorone/baton-sdk/pkg/dotc1z" "github.com/conductorone/baton-sdk/pkg/dotc1z/c1zstore" sdksync "github.com/conductorone/baton-sdk/pkg/sync" + "github.com/conductorone/baton-sdk/pkg/synccompactor" et "github.com/conductorone/baton-sdk/pkg/types/entitlement" gt "github.com/conductorone/baton-sdk/pkg/types/grant" rs "github.com/conductorone/baton-sdk/pkg/types/resource" @@ -380,15 +384,106 @@ type compatResult struct { Grants int `json:"grants"` CountErr string `json:"count_err,omitempty"` // Checkpoint evidence (see inspectRuns). - UnfinishedRuns int `json:"unfinished_runs"` - TokenLen int `json:"token_len"` - TokenSpawned bool `json:"token_spawned"` - TokenTypeScoped bool `json:"token_type_scoped"` + UnfinishedRuns int `json:"unfinished_runs"` + TokenLen int `json:"token_len"` + TokenSpawned bool `json:"token_spawned"` + TokenTypeScoped bool `json:"token_type_scoped"` + GraphPresent bool `json:"graph_present,omitempty"` + GraphReusable bool `json:"graph_reusable,omitempty"` + GraphErr string `json:"graph_err,omitempty"` + IncrementalRan bool `json:"incremental_ran,omitempty"` + IncrementalOutcome string `json:"incremental_outcome,omitempty"` + IncrementalReason string `json:"incremental_reason,omitempty"` + IncrementalError string `json:"incremental_error,omitempty"` + ArtifactPath string `json:"artifact_path,omitempty"` + AllocatedBytes uint64 `json:"allocated_bytes,omitempty"` +} + +// graphCompatHandler is installed by graph_modes_new.go in the candidate +// binary. The old binary is built from this file alone, so it can inspect new +// artifacts without compiling against APIs that did not exist at the old ref. +var graphCompatHandler func(context.Context, string, string, string) (compatResult, error) + +func graphCompatEmptyPartial(ctx context.Context, dir string) (*synccompactor.CompactableSync, error) { + path := filepath.Join(dir, "empty-partial.c1z") + store, err := dotc1z.NewStore(ctx, path, dotc1z.WithTmpDir(os.TempDir())) + if err != nil { + return nil, err + } + syncID, err := store.StartNewSync(ctx, connectorstore.SyncTypePartial, "") + if err == nil { + err = store.EndSync(ctx) + } + if closeErr := store.Close(ctx); err == nil { + err = closeErr + } + if err != nil { + return nil, err + } + return &synccompactor.CompactableSync{FilePath: path, SyncID: syncID}, nil +} + +func graphCompatFullCompact(ctx context.Context, inputPath, outPath string) (compatResult, error) { + if outPath == "" { + return compatResult{}, fmt.Errorf("graph-old-compact requires -out") + } + store, err := dotc1z.NewStore(ctx, inputPath, + dotc1z.WithReadOnly(true), dotc1z.WithTmpDir(os.TempDir())) + if err != nil { + return compatResult{}, err + } + run, err := store.SyncMeta().LatestFinishedSyncOfAnyType(ctx) + if closeErr := store.Close(ctx); err == nil { + err = closeErr + } + if err != nil { + return compatResult{}, err + } + empty, err := graphCompatEmptyPartial(ctx, filepath.Dir(outPath)) + if err != nil { + return compatResult{}, err + } + runtime.GC() + var before runtime.MemStats + runtime.ReadMemStats(&before) + compactor, cleanup, err := synccompactor.NewCompactor(ctx, filepath.Dir(outPath), + []*synccompactor.CompactableSync{{FilePath: inputPath, SyncID: run.ID}, empty}, + synccompactor.WithTmpDir(os.TempDir())) + if err != nil { + return compatResult{}, err + } + defer cleanup() + out, err := compactor.Compact(ctx) + if err != nil { + return compatResult{}, err + } + if out.FilePath != outPath { + _ = os.Remove(outPath) + if err := os.Rename(out.FilePath, outPath); err != nil { + return compatResult{}, err + } + } + result := compatResult{Mode: "graph-old-compact", ArtifactPath: outPath} + var after runtime.MemStats + runtime.ReadMemStats(&after) + result.AllocatedBytes = after.TotalAlloc - before.TotalAlloc + result.Resources, result.Ents, result.Grants, err = countRows(ctx, outPath, os.TempDir(), &result) + return result, err +} + +func printCompatResult(result compatResult) error { + encoded, err := json.Marshal(result) + if err != nil { + return err + } + fmt.Printf("COMPAT_RESULT %s\n", encoded) + return nil } func run() error { mode := flag.String("mode", "", "gen or resume") c1zPath := flag.String("c1z", "", "path to c1z file") + outPath := flag.String("out", "", "optional output path for graph compatibility modes") runDuration := flag.Duration("run-duration", 2*time.Second, "gen-mode run duration") flag.Parse() if *mode == "" || *c1zPath == "" { @@ -402,6 +497,41 @@ func run() error { } ctx = ctxzap.ToContext(ctx, logger) + if strings.HasPrefix(*mode, "graph-") { + if *mode == "graph-default-compact" { + result, err := graphCompatFullCompact(ctx, *c1zPath, *outPath) + if err != nil { + return err + } + result.Mode = *mode + return printCompatResult(result) + } + if graphCompatHandler != nil { + result, err := graphCompatHandler(ctx, *mode, *c1zPath, *outPath) + if err != nil { + return err + } + return printCompatResult(result) + } + if *mode == "graph-old-compact" { + result, err := graphCompatFullCompact(ctx, *c1zPath, *outPath) + if err != nil { + return err + } + return printCompatResult(result) + } + if *mode != "graph-inspect" { + return fmt.Errorf("graph compatibility mode %q unsupported by this SDK", *mode) + } + result := compatResult{Mode: *mode, ArtifactPath: *c1zPath} + var countErr error + result.Resources, result.Ents, result.Grants, countErr = countRows(ctx, *c1zPath, os.TempDir(), &result) + if countErr != nil { + result.CountErr = countErr.Error() + } + return printCompatResult(result) + } + connector, err := newCompatConnector() if err != nil { return err @@ -440,12 +570,7 @@ func run() error { if countErr != nil { result.CountErr = countErr.Error() } - encoded, err := json.Marshal(result) - if err != nil { - return err - } - fmt.Printf("COMPAT_RESULT %s\n", encoded) - return nil + return printCompatResult(result) } func main() { diff --git a/cmd/baton-crash-harness/driver_test.go b/cmd/baton-crash-harness/driver_test.go index 8755472a0..bc03a2295 100644 --- a/cmd/baton-crash-harness/driver_test.go +++ b/cmd/baton-crash-harness/driver_test.go @@ -125,7 +125,7 @@ func TestCrashResumeRealConnector(t *testing.T) { cells := []cellConfig{ {engine: "sqlite", workers: 4, budgetMs: 1000}, - {engine: "pebble", workers: 4, budgetMs: 1000}, + {engine: "pebble", workers: 4, budgetMs: 500}, {engine: "pebble", workers: 0, budgetMs: 2000}, } for _, cell := range cells { @@ -288,7 +288,7 @@ func runSession( if cell.mode != "" { args = append(args, "-mode", cell.mode) } - cmd := exec.CommandContext(ctx, bin, args...) + cmd := exec.CommandContext(ctx, bin, args...) // #nosec G204 -- bin is built by this test into its private TempDir. cmd.Dir = cellTmp var out bytes.Buffer cmd.Stdout = &out diff --git a/pkg/c1zsanitize/sanitize_pebble_test.go b/pkg/c1zsanitize/sanitize_pebble_test.go index 2c66745b2..48cf10219 100644 --- a/pkg/c1zsanitize/sanitize_pebble_test.go +++ b/pkg/c1zsanitize/sanitize_pebble_test.go @@ -11,6 +11,8 @@ import ( "github.com/conductorone/baton-sdk/pkg/connectorstore" "github.com/conductorone/baton-sdk/pkg/dotc1z" "github.com/conductorone/baton-sdk/pkg/dotc1z/c1zstore" + sdksync "github.com/conductorone/baton-sdk/pkg/sync" + "github.com/conductorone/baton-sdk/pkg/sync/expand" ) // TestSanitizePebbleEndToEnd is the core-invariant check on a @@ -74,6 +76,46 @@ func TestSanitizePebbleEndToEnd(t *testing.T) { require.True(t, dstRuns[0].SupportsDiff, "supports_diff marker must carry to the pebble output") } +func TestSanitizeDropsEntitlementGraphSidecar(t *testing.T) { + ctx := context.Background() + dir := t.TempDir() + srcPath := filepath.Join(dir, "src.c1z") + dstPath := filepath.Join(dir, "dst.c1z") + + src := newEngineStore(t, ctx, srcPath, c1zstore.EnginePebble) + buildParityFixture(t, ctx, src) + runs, _, err := src.(syncRunMetadataReader).ListSyncRuns(ctx, "", 100) + require.NoError(t, err) + require.Len(t, runs, 1) + syncID := runs[0].ID + digest, found, err := src.(c1zstore.GrantGenerationDigestReader).GrantGenerationDigest(ctx) + require.NoError(t, err) + require.True(t, found) + graph := expand.NewEntitlementGraph(ctx) + graph.MarkExpansionComplete() + graph.Loaded = true + graph.HasNoCycles = true + blob, err := expand.MarshalGraphBlobWithGrantDigest(syncID, graph, digest) + require.NoError(t, err) + require.NoError(t, src.(sdksync.EntitlementGraphStore).PutEntitlementGraphBlob(ctx, blob)) + require.NoError(t, src.Close(ctx)) + + source := openEngineStoreRO(t, ctx, srcPath) + destination := newEngineStore(t, ctx, dstPath, c1zstore.EnginePebble) + require.NoError(t, Sanitize(ctx, source, destination, Options{Secret: bytes32("graph-sidecar"), TimestampAnchor: fixedAnchor})) + require.NoError(t, destination.Close(ctx)) + require.NoError(t, source.Close(ctx)) + + ro := openEngineStoreRO(t, ctx, dstPath) + dstRuns, _, err := ro.(syncRunMetadataReader).ListSyncRuns(ctx, "", 100) + require.NoError(t, err) + require.Len(t, dstRuns, 1) + got, err := sdksync.GraphFromStore(ctx, ro, dstRuns[0].ID) + require.NoError(t, err) + require.Nil(t, got, "sanitize changes identities, so it must not carry the source graph") + require.NoError(t, ro.Close(ctx)) +} + // TestSanitizeMultiSyncIntoPebbleIsRejected proves the live-dst guard: a // multi-sync SQLite source cannot be sanitized into a single-sync Pebble // destination, while a single-sync source into Pebble succeeds. diff --git a/pkg/dotc1z/engine/pebble/choke_point_meta_test.go b/pkg/dotc1z/engine/pebble/choke_point_meta_test.go index b72425868..7bee6123f 100644 --- a/pkg/dotc1z/engine/pebble/choke_point_meta_test.go +++ b/pkg/dotc1z/engine/pebble/choke_point_meta_test.go @@ -67,7 +67,7 @@ func TestRawWriteSignalsOnlyInsideRawDB(t *testing.T) { if !strings.HasSuffix(name, ".go") || strings.HasSuffix(name, "_test.go") { return nil } - src, readErr := os.ReadFile(path) //nolint:gosec // meta-test walking the repo's own source tree + src, readErr := os.ReadFile(path) // #nosec G122 -- read-only meta-test walks the trusted repository tree. if readErr != nil { return readErr } @@ -197,7 +197,7 @@ func walkClientTreeProductionGoFiles(t *testing.T, visit func(root, path string, if !strings.HasSuffix(name, ".go") || strings.HasSuffix(name, "_test.go") { return nil } - src, readErr := os.ReadFile(path) //nolint:gosec // meta-test walking the repo's own source tree; no untrusted paths + src, readErr := os.ReadFile(path) // #nosec G122 -- read-only meta-test walks the trusted repository tree. if readErr != nil { return readErr } diff --git a/pkg/dotc1z/engine/pebble/errorfs_sweep_test.go b/pkg/dotc1z/engine/pebble/errorfs_sweep_test.go index 723bc3ac7..da25bbb85 100644 --- a/pkg/dotc1z/engine/pebble/errorfs_sweep_test.go +++ b/pkg/dotc1z/engine/pebble/errorfs_sweep_test.go @@ -65,7 +65,6 @@ package pebble import ( "context" "fmt" - "math/rand" "os" "runtime" "sync" @@ -727,9 +726,10 @@ func TestErrorFSEndSyncWindowSweep(t *testing.T) { k-1, fatalRuns, outcomes) } -// TestErrorFSWholeSyncRandomSweepSoak randomizes the failure point -// across the whole sync (open → pages → EndSync) with the strictest -// crash image (no unsynced survival). Env-gated like the other soaks. +// TestErrorFSWholeSyncRandomSweepSoak exhaustively advances the failure point +// across the whole pre-EndSync write path. It stops only after one armed run +// completes without an injection, proving every earlier write-class operation +// was exercised. The historical name is kept for Makefile compatibility. func TestErrorFSWholeSyncRandomSweepSoak(t *testing.T) { skipOnWindowsMemFS(t) if os.Getenv("BATON_SOAK") == "" { @@ -740,61 +740,67 @@ func TestErrorFSWholeSyncRandomSweepSoak(t *testing.T) { cache := pebble.NewCache(8 << 20) defer cache.Unref() - for seed := int64(1); seed <= 60; seed++ { - t.Run(fmt.Sprintf("seed=%d", seed), func(t *testing.T) { - rng := rand.New(rand.NewSource(seed)) //nolint:gosec // deterministic seeded sweep, not cryptography - k := int64(rng.Intn(400)) - - fs := vfs.NewCrashableMem() - inj := &failFromInjector{failOnce: true} - efs := errorfs.Wrap(fs, inj) - - gate := newFatalGate() - e, err := Open(ctx, "sweep-db", WithVFS(efs), WithSharedCache(cache), withFatalGate(gate)) - require.NoError(t, err, "open before arming") - a := NewAdapter(e) - - var syncID string - res := runInjected(fs, inj, e, gate, k, func() error { - var err error - syncID, err = a.StartNewSync(ctx, connectorstore.SyncTypeFull, "") - if err != nil { - return err - } - if err := w.write(ctx, a); err != nil { - return err - } - // Disarm before EndSync: the soak owns the OPEN + PAGES - // phase (undurable, memtable-resident state — the fresh-sync - // crash surface the window sweep's durable baseline can't - // reach). EndSync itself is the window sweep's territory — - // and with pages still in the memtable, EndSync's ingests - // take pebble's flushable-ingest path, where ANY injected - // failure is a raw panic(err) whose unwind faults the - // runtime (crash-only by design; not recoverable in-process; - // see the coverage-gap note in the file header). - inj.disarm() - return a.EndSync(ctx) - }) - - label := fmt.Sprintf("seed=%d k=%d", seed, k) - if res.err == nil { - // Failure point beyond the sync's op count (or only - // post-completion background work was hit): the finished - // artifact was flushed, so the strict crash image must - // verify complete. - require.NotEmpty(t, syncID) - verifyCrashImage(ctx, t, w, res.image, cache, syncID, true, res.injected > 0, label+" (clean run)") - return + const maxK = 5000 + var injectedRuns int64 + for k := int64(0); ; k++ { + require.Less(t, k, int64(maxK), "whole-sync sweep exceeded %d write ops", maxK) + fs := vfs.NewCrashableMem() + inj := &failFromInjector{failOnce: true} + efs := errorfs.Wrap(fs, inj) + + gate := newFatalGate() + e, err := Open(ctx, "sweep-db", WithVFS(efs), WithSharedCache(cache), withFatalGate(gate)) + require.NoError(t, err, "open before arming") + a := NewAdapter(e) + + var syncID string + res := runInjected(fs, inj, e, gate, k, func() error { + var err error + syncID, err = a.StartNewSync(ctx, connectorstore.SyncTypeFull, "") + if err != nil { + return err } - if syncID == "" { - // StartNewSync itself failed; the image owes nothing beyond - // reopening clean and supporting a fresh sync. - verifyEmptyImageRestarts(ctx, t, w, res.image, cache, label+" (pre-sync)") - return + if err := w.write(ctx, a); err != nil { + return err } - verifyCrashImage(ctx, t, w, res.image, cache, syncID, true, true, label) + // Disarm before EndSync: the soak owns the OPEN + PAGES + // phase (undurable, memtable-resident state — the fresh-sync + // crash surface the window sweep's durable baseline can't + // reach). EndSync itself is the window sweep's territory — + // and with pages still in the memtable, EndSync's ingests + // take pebble's flushable-ingest path, where ANY injected + // failure is a raw panic(err) whose unwind faults the + // runtime (crash-only by design; not recoverable in-process; + // see the coverage-gap note in the file header). + inj.disarm() + return a.EndSync(ctx) }) + + label := fmt.Sprintf("k=%d", k) + if res.err == nil { + // Failure point beyond the sync's op count (or only + // post-completion background work was hit): the finished + // artifact was flushed, so the strict crash image must + // verify complete. + require.NotEmpty(t, syncID) + verifyCrashImage(ctx, t, w, res.image, cache, syncID, true, res.injected > 0, label+" (clean run)") + if res.injected == 0 { + require.Equal(t, k, injectedRuns, "every earlier write point must inject") + t.Logf("whole pre-EndSync path covered: %d write-op failure points", injectedRuns) + break + } + injectedRuns++ + continue + } + require.Positive(t, res.injected, "failed run must contain an injected write fault") + injectedRuns++ + if syncID == "" { + // StartNewSync itself failed; the image owes nothing beyond + // reopening clean and supporting a fresh sync. + verifyEmptyImageRestarts(ctx, t, w, res.image, cache, label+" (pre-sync)") + continue + } + verifyCrashImage(ctx, t, w, res.image, cache, syncID, true, true, label) } } diff --git a/pkg/sync/expand/graph_blob_test.go b/pkg/sync/expand/graph_blob_test.go index ecc1766bc..ae82c346b 100644 --- a/pkg/sync/expand/graph_blob_test.go +++ b/pkg/sync/expand/graph_blob_test.go @@ -2,12 +2,45 @@ package expand import ( "context" + "encoding/json" "fmt" "testing" "github.com/stretchr/testify/require" ) +func TestGraphBlobRejectsMissingAndUnknownVersions(t *testing.T) { + ctx := context.Background() + g := NewEntitlementGraph(ctx) + g.AddEntitlementID("ent-a") + + legacy, err := json.Marshal(map[string]any{"sync_id": "sync-1", "graph": g}) + require.NoError(t, err) + got, err := UnmarshalGraphBlob(legacy, "sync-1") + require.NoError(t, err) + require.Nil(t, got, "an unversioned graph must fall back to full expansion") + + future, err := json.Marshal(map[string]any{"format_version": 999, "sync_id": "sync-1", "graph": g}) + require.NoError(t, err) + got, err = UnmarshalGraphBlob(future, "sync-1") + require.NoError(t, err) + require.Nil(t, got, "an unknown graph version must fall back to full expansion") +} + +func TestValidateCompletedRejectsInconsistentAdjacency(t *testing.T) { + ctx := context.Background() + g := NewEntitlementGraph(ctx) + g.AddEntitlementID("ent-a") + g.AddEntitlementID("ent-b") + require.NoError(t, g.AddEdge(ctx, "ent-a", "ent-b", false, nil)) + g.Loaded = true + g.MarkExpansionComplete() + require.NoError(t, g.ValidateCompleted()) + + delete(g.SourcesToDestinations[g.GetNode("ent-a").Id], g.GetNode("ent-b").Id) + require.ErrorContains(t, g.ValidateCompleted(), "missing from source adjacency") +} + // TestGraphBlobRoundTrip: marshal/unmarshal preserves the graph; the sync-id // guard rejects a blob from a different sync. func TestGraphBlobRoundTrip(t *testing.T) { @@ -77,6 +110,103 @@ func TestGraphBlobSizeAtScale(t *testing.T) { } } +func BenchmarkGraphClone(b *testing.B) { + ctx := context.Background() + for _, n := range []int{1_000, 10_000, 100_000} { + b.Run(fmt.Sprintf("nodes=%d", n), func(b *testing.B) { + g := NewEntitlementGraph(ctx) + for i := 0; i < n; i++ { + g.AddEntitlementID(entName(i)) + } + for i := 0; i+1 < n; i++ { + require.NoError(b, g.AddEdge(ctx, entName(i), entName(i+1), false, nil)) + } + g.MarkExpansionComplete() + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + clone, err := g.Clone() + if err != nil { + b.Fatal(err) + } + if !clone.IsExpanded() { + b.Fatal("clone lost completion state") + } + } + }) + } +} + +func TestGraphCloneIsStructurallyIndependent(t *testing.T) { + ctx := context.Background() + graph := NewEntitlementGraph(ctx) + graph.AddEntitlementID("a") + graph.AddEntitlementID("b") + require.NoError(t, graph.AddEdge(ctx, "a", "b", false, []string{"user"})) + graph.Loaded = true + graph.MarkExpansionComplete() + graph.Actions = []*EntitlementGraphAction{{ + SourceEntitlementID: "a", + Descendants: []ActionDescendant{{EntitlementID: "b"}}, + ResourceTypeIDs: []string{"user"}, + }} + graph.ExpansionPlan = &EntitlementGraphPlan{Order: []int{0, 1}, ProjectionSources: []string{"a"}} + graph.ExpansionMetrics = &EntitlementGraphMetrics{Algorithm: "test"} + + clone, err := graph.Clone() + require.NoError(t, err) + clone.Nodes[graph.GetNode("a").Id] = Node{Id: 99, EntitlementIDs: []string{"changed"}} + clone.EntitlementsToNodes["a"] = 99 + for edgeID, edge := range clone.Edges { + edge.ResourceTypeIDs[0] = "service" + clone.Edges[edgeID] = edge + } + clone.Actions[0].Descendants[0].EntitlementID = "changed" + clone.Actions[0].ResourceTypeIDs[0] = "service" + clone.ExpansionPlan.Order[0] = 99 + clone.ExpansionPlan.ProjectionSources[0] = "changed" + clone.ExpansionMetrics.Algorithm = "changed" + + require.Equal(t, "a", graph.Nodes[graph.GetNode("a").Id].EntitlementIDs[0]) + require.Equal(t, graph.GetNode("a").Id, graph.EntitlementsToNodes["a"]) + for _, edge := range graph.Edges { + require.Equal(t, []string{"user"}, edge.ResourceTypeIDs) + } + require.Equal(t, "b", graph.Actions[0].Descendants[0].EntitlementID) + require.Equal(t, []string{"user"}, graph.Actions[0].ResourceTypeIDs) + require.Equal(t, 0, graph.ExpansionPlan.Order[0]) + require.Equal(t, "a", graph.ExpansionPlan.ProjectionSources[0]) + require.Equal(t, "test", graph.ExpansionMetrics.Algorithm) +} + +func BenchmarkMarshalGraphBlob(b *testing.B) { + ctx := context.Background() + for _, n := range []int{1_000, 10_000, 100_000} { + b.Run(fmt.Sprintf("nodes=%d", n), func(b *testing.B) { + g := NewEntitlementGraph(ctx) + for i := 0; i < n; i++ { + g.AddEntitlementID(entName(i)) + } + for i := 0; i+1 < n; i++ { + require.NoError(b, g.AddEdge(ctx, entName(i), entName(i+1), false, nil)) + } + g.Loaded = true + g.MarkExpansionComplete() + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + data, err := MarshalGraphBlob("benchmark-sync", g) + if err != nil { + b.Fatal(err) + } + if len(data) == 0 { + b.Fatal("empty graph blob") + } + } + }) + } +} + func entName(i int) string { return fmt.Sprintf("group:g%06d:member", i) } diff --git a/pkg/sync/expand/incremental_benchmark_test.go b/pkg/sync/expand/incremental_benchmark_test.go new file mode 100644 index 000000000..ff9cd8ea6 --- /dev/null +++ b/pkg/sync/expand/incremental_benchmark_test.go @@ -0,0 +1,209 @@ +package expand + +import ( + "context" + "errors" + "fmt" + "os" + "runtime" + "testing" + + "github.com/stretchr/testify/require" +) + +type incrementalBenchFixture struct { + store *MockExpanderStore + graph *EntitlementGraph + changed []string +} + +// TestIncrementalPerformanceGates is intentionally opt-in because its +// 100k-entitlement/100k-principal fixtures allocate hundreds of megabytes. +// It enforces allocation/work gates; wall time remains benchmark evidence. +func TestIncrementalPerformanceGates(t *testing.T) { + if os.Getenv("BATON_INCREMENTAL_PERF") == "" { + t.Skip("set BATON_INCREMENTAL_PERF=1 to run production-scale performance gates") + } + const ( + entitlements = 100_000 + principals = 100_000 + ) + ctx := context.Background() + + sparse := buildIncrementalBenchFixture(t, entitlements, principals, 1, true) + var sparseResult *IncrementalResult + sparseAlloc := measuredTotalAlloc(func() { + var err error + sparseResult, err = NewIncrementalExpander(sparse.store, sparse.graph). + ExpandChanges(ctx, nil, sparse.changed) + require.NoError(t, err) + }) + require.LessOrEqual(t, len(sparseResult.EntitlementsWalked), 3, + "sparse work must stay bounded by its modeled affected component") + + dense := buildIncrementalBenchFixture(t, entitlements, principals, 10_000, true) + denseEligibilityAlloc := measuredTotalAlloc(func() { + _, err := NewIncrementalExpander(dense.store, dense.graph). + ExpandChanges(ctx, nil, dense.changed) + require.ErrorIs(t, err, ErrIncrementalDenseChangeDecline) + }) + + full := buildIncrementalBenchFixture(t, entitlements, principals, 10_000, false) + fullAlloc := measuredTotalAlloc(func() { + require.NoError(t, NewExpander(full.store, full.graph).Run(ctx)) + }) + + // P3: deciding to fall back must cost at most 10% of full expansion. + require.LessOrEqual(t, denseEligibilityAlloc*10, fullAlloc) + // P4: dense eligibility plus full fallback must stay within 1.15x full. + require.LessOrEqual(t, (denseEligibilityAlloc+fullAlloc)*100, fullAlloc*115) + // P5: a truly sparse incremental run must allocate less than full. + require.Less(t, sparseAlloc, fullAlloc) + t.Logf("allocations: sparse=%d dense-eligibility=%d full=%d", sparseAlloc, denseEligibilityAlloc, fullAlloc) +} + +func measuredTotalAlloc(run func()) uint64 { + runtime.GC() + var before, after runtime.MemStats + runtime.ReadMemStats(&before) + run() + runtime.ReadMemStats(&after) + return after.TotalAlloc - before.TotalAlloc +} + +// BenchmarkIncrementalExpand measures only the change application. Fixture +// construction and the prior completed expansion are outside the timer. +func BenchmarkIncrementalExpand(b *testing.B) { + for _, entitlements := range []int{1_000, 10_000, 100_000} { + for _, principals := range []int{10_000, 100_000} { + for _, delta := range []int{1, 100, 250, 500, 1_000, 2_500, 5_000, 10_000} { + if delta > principals { + continue + } + name := fmt.Sprintf("E=%d/P=%d/K=%d", entitlements, principals, delta) + b.Run(name, func(b *testing.B) { + b.ReportAllocs() + for i := 0; i < b.N; i++ { + b.StopTimer() + fixture := buildIncrementalBenchFixture(b, entitlements, principals, delta, true) + runtime.GC() + var before runtime.MemStats + runtime.ReadMemStats(&before) + b.StartTimer() + _, err := NewIncrementalExpander(fixture.store, fixture.graph). + ExpandChanges(context.Background(), nil, fixture.changed) + b.StopTimer() + if errors.Is(err, ErrIncrementalDenseChangeDecline) { + b.ReportMetric(1, "dense-decline/op") + } else if err != nil { + b.Fatal(err) + } + reportHeapDelta(b, &before) + } + }) + } + } + } +} + +// BenchmarkFullExpand is the fresh-rebuild oracle for the same post-delta +// states used by BenchmarkIncrementalExpand. +func BenchmarkFullExpand(b *testing.B) { + for _, entitlements := range []int{1_000, 10_000, 100_000} { + for _, principals := range []int{10_000, 100_000} { + for _, delta := range []int{1, 100, 250, 500, 1_000, 2_500, 5_000, 10_000} { + if delta > principals { + continue + } + name := fmt.Sprintf("E=%d/P=%d/K=%d", entitlements, principals, delta) + b.Run(name, func(b *testing.B) { + b.ReportAllocs() + for i := 0; i < b.N; i++ { + b.StopTimer() + fixture := buildIncrementalBenchFixture(b, entitlements, principals, delta, false) + runtime.GC() + var before runtime.MemStats + runtime.ReadMemStats(&before) + b.StartTimer() + err := NewExpander(fixture.store, fixture.graph).Run(context.Background()) + b.StopTimer() + if err != nil { + b.Fatal(err) + } + reportHeapDelta(b, &before) + } + }) + } + } + } +} + +func buildIncrementalBenchFixture( + b testing.TB, + entitlementCount int, + principalCount int, + deltaCount int, + prepareBase bool, +) incrementalBenchFixture { + b.Helper() + ctx := context.Background() + // Three nodes per component: source -> left and source -> right. This gives + // about 2E/3 edges and bounds total expanded rows near 3P. + components := max(1, entitlementCount/3) + store := NewMockExpanderStore() + graph := NewEntitlementGraph(ctx) + sources := make([]string, components) + for i := 0; i < components; i++ { + source := fmt.Sprintf("source:%06d", i) + left := fmt.Sprintf("left:%06d", i) + right := fmt.Sprintf("right:%06d", i) + sources[i] = source + for _, id := range []string{source, left, right} { + store.AddEntitlement(makeEntitlement(id, makeResource("group", id))) + graph.AddEntitlementID(id) + } + if err := graph.AddEdge(ctx, source, left, false, nil); err != nil { + b.Fatal(err) + } + if err := graph.AddEdge(ctx, source, right, false, nil); err != nil { + b.Fatal(err) + } + } + for i := 0; i < principalCount; i++ { + source := sources[i%len(sources)] + store.AddGrant(directGrant(source, makeResource("user", fmt.Sprintf("p:%07d", i)))) + } + + if prepareBase { + if err := graph.FixCycles(ctx); err != nil { + b.Fatal(err) + } + if err := NewExpander(store, graph).Run(ctx); err != nil { + b.Fatal(err) + } + graph.Loaded = true + } + + changedSet := make(map[string]struct{}, deltaCount) + for i := 0; i < deltaCount; i++ { + source := sources[i%len(sources)] + store.AddGrant(directGrant(source, makeResource("user", fmt.Sprintf("delta:%07d", i)))) + changedSet[source] = struct{}{} + } + changed := make([]string, 0, len(changedSet)) + for source := range changedSet { + changed = append(changed, source) + } + return incrementalBenchFixture{store: store, graph: graph, changed: changed} +} + +func reportHeapDelta(b *testing.B, before *runtime.MemStats) { + var after runtime.MemStats + runtime.ReadMemStats(&after) + if after.TotalAlloc >= before.TotalAlloc { + b.ReportMetric(float64(after.TotalAlloc-before.TotalAlloc), "total-alloc-bytes/op") + } + if after.HeapInuse >= before.HeapInuse { + b.ReportMetric(float64(after.HeapInuse-before.HeapInuse), "heap-inuse-delta-bytes/op") + } +} diff --git a/pkg/sync/expand/incremental_differential_test.go b/pkg/sync/expand/incremental_differential_test.go new file mode 100644 index 000000000..9b967f342 --- /dev/null +++ b/pkg/sync/expand/incremental_differential_test.go @@ -0,0 +1,461 @@ +package expand + +import ( + "context" + "fmt" + "math/rand" + "slices" + "sort" + "testing" + + "github.com/stretchr/testify/require" +) + +// Stage-2a's oracle is a fresh full expansion. Each generated script mutates +// an already-expanded store one step at a time. Additive changes must match +// the oracle after every step; revocation-shaped changes must decline before +// the additions-only incremental writer is called. +func TestIncrementalDifferentialRandom(t *testing.T) { + coverage := make(map[differentialMutation]int) + incrementalRuns := 0 + declines := 0 + for seed := int64(0); seed < 200; seed++ { + seed := seed + t.Run(fmt.Sprintf("seed=%d", seed), func(t *testing.T) { + runs, declined, seen := runDifferentialScript(t, seed) + incrementalRuns += runs + declines += declined + for mutation, count := range seen { + coverage[mutation] += count + } + }) + } + + for mutation := differentialMutation(0); mutation < mutationCount; mutation++ { + require.Positive(t, coverage[mutation], "mutation %s was not generated", mutation) + } + require.Positive(t, declines, "the corpus must exercise safe-decline paths") + require.Greater(t, incrementalRuns, declines, + "more than half of exercised decisions should run incrementally") +} + +// FuzzIncrementalVsFullExpansion extends the deterministic corpus with Go's +// native mutation engine. A failure reports one replayable script seed. +func FuzzIncrementalVsFullExpansion(f *testing.F) { + for seed := int64(0); seed < 40; seed++ { + f.Add(seed) + } + f.Fuzz(func(t *testing.T, seed int64) { + runDifferentialScript(t, seed) + }) +} + +// TestIncrementalDifferentialMutationAdequacy proves the oracle detects the +// three representative mutants required by the review brief. These tests pass +// only when the deliberately wrong result differs from the fresh-full oracle. +func TestIncrementalDifferentialMutationAdequacy(t *testing.T) { + t.Run("ghost edge", func(t *testing.T) { + ctx := context.Background() + base := sqliteParityCase{ + entitlementIDs: []string{"a", "b"}, + grants: []sqliteGrantSpec{{ + id: "alice-a", entitlementID: "a", principalRT: "user", principalID: "alice", + }}, + edges: []sqliteEdgeSpec{{src: "a", dst: "b"}}, + } + wrongStore, wrongGraph := mockStoreFromCase(t, ctx, base) + require.NoError(t, NewExpander(wrongStore, wrongGraph).Run(ctx)) + + // Mutant: the current rule set removed a->b, but incremental wrongly + // keeps the old graph and old synthesized grant. + current := base + current.edges = nil + fullStore, fullGraph := mockStoreFromCase(t, ctx, current) + require.NoError(t, NewExpander(fullStore, fullGraph).Run(ctx)) + require.NotEqual(t, snapshotStoreGrants(fullStore), snapshotStoreGrants(wrongStore), + "the oracle must kill a retained ghost-edge mutant") + }) + + t.Run("skipped re-expansion", func(t *testing.T) { + ctx := context.Background() + current := sqliteParityCase{ + entitlementIDs: []string{"a", "b"}, + grants: []sqliteGrantSpec{{ + id: "alice-a", entitlementID: "a", principalRT: "user", principalID: "alice", + }}, + edges: []sqliteEdgeSpec{{src: "a", dst: "b"}}, + } + wrongStore, wrongGraph := mockStoreFromCase(t, ctx, current) + require.NoError(t, NewExpander(wrongStore, wrongGraph).Run(ctx)) + + // Mutant: Bob is added to A, but the incremental walk is skipped. + bob := sqliteGrantSpec{id: "bob-a", entitlementID: "a", principalRT: "user", principalID: "bob"} + current.grants = append(current.grants, bob) + wrongStore.AddGrant(directGrant("a", makeResource("user", "bob"))) + fullStore, fullGraph := mockStoreFromCase(t, ctx, current) + require.NoError(t, NewExpander(fullStore, fullGraph).Run(ctx)) + require.NotEqual(t, snapshotStoreGrants(fullStore), snapshotStoreGrants(wrongStore), + "the oracle must kill a skipped-re-expansion mutant") + }) + + t.Run("stale sidecar", func(t *testing.T) { + ctx := context.Background() + store := NewMockExpanderStore() + graph := buildExpandedChain(t, ctx, store, "alice", "a", "b") + graph.Loaded = true + data, err := MarshalGraphBlob("old-sync", graph) + require.NoError(t, err) + got, err := UnmarshalGraphBlob(data, "current-sync") + require.NoError(t, err) + require.Nil(t, got, "the guard must kill stale-sidecar acceptance") + }) + + t.Run("one hop affected walk", func(t *testing.T) { + ctx := context.Background() + current := sqliteParityCase{ + entitlementIDs: []string{"a", "b", "c"}, + grants: []sqliteGrantSpec{{ + id: "alice-a", entitlementID: "a", principalRT: "user", principalID: "alice", + }}, + edges: []sqliteEdgeSpec{{src: "a", dst: "b"}, {src: "b", dst: "c"}}, + } + wrongStore, wrongGraph := mockStoreFromCase(t, ctx, current) + require.NoError(t, NewExpander(wrongStore, wrongGraph).Run(ctx)) + bob := sqliteGrantSpec{id: "bob-a", entitlementID: "a", principalRT: "user", principalID: "bob"} + current.grants = append(current.grants, bob) + wrongStore.AddGrant(directGrant("a", makeResource("user", "bob"))) + // Mutant: stop affected traversal after B by hiding B->C. + bNode, cNode := wrongGraph.GetNode("b"), wrongGraph.GetNode("c") + edgeID := wrongGraph.SourcesToDestinations[bNode.Id][cNode.Id] + delete(wrongGraph.SourcesToDestinations[bNode.Id], cNode.Id) + delete(wrongGraph.DestinationsToSources[cNode.Id], bNode.Id) + delete(wrongGraph.Edges, edgeID) + _, err := NewIncrementalExpander(wrongStore, wrongGraph).ExpandChanges(ctx, nil, []string{"a"}) + require.NoError(t, err) + + fullStore, fullGraph := mockStoreFromCase(t, ctx, current) + require.NoError(t, NewExpander(fullStore, fullGraph).Run(ctx)) + require.NotEqual(t, snapshotStoreGrants(fullStore), snapshotStoreGrants(wrongStore), + "the oracle must kill a one-hop traversal mutant") + }) + + t.Run("stale provenance", func(t *testing.T) { + ctx := context.Background() + current := sqliteParityCase{ + entitlementIDs: []string{"a", "b", "c"}, + grants: []sqliteGrantSpec{ + {id: "carol-a", entitlementID: "a", principalRT: "user", principalID: "carol"}, + {id: "carol-b", entitlementID: "b", principalRT: "user", principalID: "carol"}, + }, + edges: []sqliteEdgeSpec{{src: "a", dst: "c"}, {src: "b", dst: "c"}}, + } + fullStore, fullGraph := mockStoreFromCase(t, ctx, current) + require.NoError(t, NewExpander(fullStore, fullGraph).Run(ctx)) + + wrongStore, _ := mockStoreFromCase(t, ctx, current) + wrongStore.AddGrant(expandedGrantWithSource("c", makeResource("user", "carol"), "a")) + require.NotEqual(t, snapshotStoreGrants(fullStore), snapshotStoreGrants(wrongStore), + "the oracle must kill a missing-contributor provenance mutant") + }) +} + +type differentialMutation int + +const ( + mutationAddMember differentialMutation = iota + mutationAddEdge + mutationWidenShallow + mutationWidenFilter + mutationRemoveMember + mutationRemoveEdge + mutationNarrowShallow + mutationNarrowFilter + mutationCloseCycle + mutationNoOp + mutationCount +) + +func (m differentialMutation) String() string { + return [...]string{ + "add-member", "add-edge", "widen-shallow", "widen-filter", + "remove-member", "remove-edge", "narrow-shallow", "narrow-filter", + "close-cycle", "no-op", + }[m] +} + +func runDifferentialScript(t *testing.T, seed int64) (int, int, map[differentialMutation]int) { + t.Helper() + ctx := context.Background() + rng := rand.New(rand.NewSource(seed ^ 0x5eed)) //nolint:gosec // deterministic fixture generation + model := randomExpansionCase(seed) + incStore, incGraph := mockStoreFromCase(t, ctx, model) + require.NoError(t, NewExpander(incStore, incGraph).Run(ctx)) + incGraph.Loaded = true + + incrementalRuns := 0 + declines := 0 + seen := make(map[differentialMutation]int) + steps := 1 + rng.Intn(10) + for step := 0; step < steps; step++ { + mutation := differentialMutation((int(seed) + step) % int(mutationCount)) + seen[mutation]++ + + newEdges, changed, decline := applyDifferentialMutation(t, rng, &model, incStore, mutation, seed, step) + if decline { + // Compaction must route these changes to full expansion. Calling the + // additions-only writer would leave stale grants or provenance. + declines++ + return incrementalRuns, declines, seen + } + + _, err := NewIncrementalExpander(incStore, incGraph).ExpandChanges(ctx, newEdges, changed) + if err != nil { + if mutation == mutationCloseCycle { + require.ErrorIs(t, err, ErrIncrementalFallback) + declines++ + return incrementalRuns, declines, seen + } + require.NoError(t, err, "seed=%d step=%d mutation=%s", seed, step, mutation) + } + incrementalRuns++ + + fullStore, fullGraph := mockStoreFromCase(t, ctx, model) + require.NoError(t, NewExpander(fullStore, fullGraph).Run(ctx)) + require.Equal(t, snapshotStoreGrants(fullStore), snapshotStoreGrants(incStore), + "seed=%d step=%d mutation=%s", seed, step, mutation) + require.Equal(t, independentAccessOracle(model), snapshotAccessPairs(incStore), + "seed=%d step=%d mutation=%s independent access oracle", seed, step, mutation) + assertReusableGraph(t, incGraph, seed, step, mutation) + } + return incrementalRuns, declines, seen +} + +type modelMembership struct { + resourceType string + principalID string + direct bool +} + +// independentAccessOracle is deliberately graph-library-free. It computes a +// fixed point over plain maps and sets, providing a second oracle independent +// of both IncrementalExpander and the production full Expander. +func independentAccessOracle(model sqliteParityCase) map[string]struct{} { + members := make(map[string]map[string]modelMembership, len(model.entitlementIDs)) + for _, entitlementID := range model.entitlementIDs { + members[entitlementID] = make(map[string]modelMembership) + } + for _, grant := range model.grants { + key := grant.principalRT + "\x00" + grant.principalID + direct := len(grant.sources) == 0 || grant.sources[grant.entitlementID] + current, exists := members[grant.entitlementID][key] + if !exists || direct { + members[grant.entitlementID][key] = modelMembership{ + resourceType: grant.principalRT, principalID: grant.principalID, direct: current.direct || direct, + } + } + } + changed := true + for changed { + changed = false + for _, edge := range model.edges { + for key, member := range members[edge.src] { + if edge.shallow && !member.direct { + continue + } + if len(edge.rtids) > 0 && !slices.Contains(edge.rtids, member.resourceType) { + continue + } + if _, exists := members[edge.dst][key]; exists { + continue + } + members[edge.dst][key] = modelMembership{ + resourceType: member.resourceType, principalID: member.principalID, + } + changed = true + } + } + } + out := make(map[string]struct{}) + for entitlementID, entitlementMembers := range members { + for _, member := range entitlementMembers { + out[entitlementID+"\x00"+member.resourceType+"\x00"+member.principalID] = struct{}{} + } + } + return out +} + +func snapshotAccessPairs(store *MockExpanderStore) map[string]struct{} { + out := make(map[string]struct{}) + for entitlementID, grants := range store.grants { + for _, grant := range grants { + principal := grant.GetPrincipal().GetId() + out[entitlementID+"\x00"+principal.GetResourceType()+"\x00"+principal.GetResource()] = struct{}{} + } + } + return out +} + +func applyDifferentialMutation( + t *testing.T, + rng *rand.Rand, + model *sqliteParityCase, + store *MockExpanderStore, + mutation differentialMutation, + seed int64, + step int, +) ([]NewEdge, []string, bool) { + t.Helper() + ents := model.entitlementIDs + switch mutation { + case mutationAddMember: + entitlementID := ents[rng.Intn(len(ents))] + principalID := fmt.Sprintf("delta-%d-%d", seed, step) + grant := directGrant(entitlementID, makeResource("user", principalID)) + store.AddGrant(grant) + model.grants = append(model.grants, sqliteGrantSpec{ + id: grant.GetId(), entitlementID: entitlementID, principalRT: "user", principalID: principalID, + }) + return nil, []string{entitlementID}, false + + case mutationAddEdge: + edge, ok := missingForwardEdge(*model) + if !ok { + return nil, nil, false + } + edge.shallow = rng.Intn(2) == 0 + if rng.Intn(2) == 0 { + edge.rtids = []string{"user"} + } + model.edges = append(model.edges, edge) + return []NewEdge{newEdgeFromSpec(edge)}, nil, false + + case mutationWidenShallow: + index := findEdge(model.edges, func(edge sqliteEdgeSpec) bool { return edge.shallow }) + if index < 0 { + return nil, nil, false + } + model.edges[index].shallow = false + return []NewEdge{newEdgeFromSpec(model.edges[index])}, nil, false + + case mutationWidenFilter: + index := findEdge(model.edges, func(edge sqliteEdgeSpec) bool { return len(edge.rtids) > 0 }) + if index < 0 { + return nil, nil, false + } + model.edges[index].rtids = nil // no filter means all principal types + return []NewEdge{newEdgeFromSpec(model.edges[index])}, nil, false + + case mutationRemoveMember: + // Membership removals need deletion-aware writes. The current contract + // declines them to a fresh full expansion. + return nil, nil, true + + case mutationRemoveEdge: + if len(model.edges) == 0 { + return nil, nil, false + } + return nil, nil, true + + case mutationNarrowShallow: + index := findEdge(model.edges, func(edge sqliteEdgeSpec) bool { return !edge.shallow }) + if index < 0 { + return nil, nil, false + } + return nil, nil, true + + case mutationNarrowFilter: + index := findEdge(model.edges, func(edge sqliteEdgeSpec) bool { + return len(edge.rtids) == 0 || !slices.Equal(edge.rtids, []string{"user"}) + }) + if index < 0 { + return nil, nil, false + } + return nil, nil, true + + case mutationCloseCycle: + if len(model.edges) == 0 { + return nil, nil, false + } + base := model.edges[rng.Intn(len(model.edges))] + reverse := sqliteEdgeSpec{src: base.dst, dst: base.src} + model.edges = append(model.edges, reverse) + return []NewEdge{newEdgeFromSpec(reverse)}, nil, false + + case mutationNoOp: + if len(model.edges) == 0 { + return nil, nil, false + } + return []NewEdge{newEdgeFromSpec(model.edges[rng.Intn(len(model.edges))])}, nil, false + case mutationCount: + return nil, nil, false + } + return nil, nil, false +} + +func findEdge(edges []sqliteEdgeSpec, match func(sqliteEdgeSpec) bool) int { + for i, edge := range edges { + if match(edge) { + return i + } + } + return -1 +} + +func newEdgeFromSpec(edge sqliteEdgeSpec) NewEdge { + return NewEdge{ + SourceEntitlementID: edge.src, + DestEntitlementID: edge.dst, + Shallow: edge.shallow, + ResourceTypeIDs: append([]string(nil), edge.rtids...), + } +} + +func assertReusableGraph(t *testing.T, graph *EntitlementGraph, seed int64, step int, mutation differentialMutation) { + t.Helper() + require.True(t, graph.IsExpanded(), "seed=%d step=%d mutation=%s", seed, step, mutation) + require.True(t, graph.HasNoCycles, "seed=%d step=%d mutation=%s", seed, step, mutation) + require.NoError(t, graph.ValidateCompleted(), "seed=%d step=%d mutation=%s", seed, step, mutation) + + data, err := MarshalGraphBlob("differential", graph) + require.NoError(t, err) + roundTrip, err := UnmarshalGraphBlob(data, "differential") + require.NoError(t, err) + require.NotNil(t, roundTrip) + require.NoError(t, roundTrip.ValidateCompleted()) + require.Equal(t, canonicalDifferentialGraph(graph), canonicalDifferentialGraph(roundTrip)) +} + +// canonicalGraph removes map iteration from diagnostics and checks the +// durable graph structure, not transient action/metric state. +func canonicalDifferentialGraph(graph *EntitlementGraph) []string { + out := make([]string, 0, len(graph.Edges)+len(graph.EntitlementsToNodes)) + for entitlementID, nodeID := range graph.EntitlementsToNodes { + out = append(out, fmt.Sprintf("node:%s=%d", entitlementID, nodeID)) + } + for _, edge := range graph.Edges { + source := graph.Nodes[edge.SourceID] + destination := graph.Nodes[edge.DestinationID] + out = append(out, fmt.Sprintf("edge:%v->%v:%t:%v:%t", + source.EntitlementIDs, destination.EntitlementIDs, + edge.IsShallow, edge.ResourceTypeIDs, edge.IsExpanded)) + } + sort.Strings(out) + return out +} + +func missingForwardEdge(tc sqliteParityCase) (sqliteEdgeSpec, bool) { + existing := make(map[string]struct{}, len(tc.edges)) + for _, edge := range tc.edges { + existing[edge.src+"\x00"+edge.dst] = struct{}{} + } + for i := 0; i < len(tc.entitlementIDs); i++ { + for j := i + 1; j < len(tc.entitlementIDs); j++ { + src, dst := tc.entitlementIDs[i], tc.entitlementIDs[j] + if _, ok := existing[src+"\x00"+dst]; ok { + continue + } + return sqliteEdgeSpec{src: src, dst: dst}, true + } + } + return sqliteEdgeSpec{}, false +} diff --git a/pkg/sync/expand/incremental_exhaustive_test.go b/pkg/sync/expand/incremental_exhaustive_test.go new file mode 100644 index 000000000..8390a6a2d --- /dev/null +++ b/pkg/sync/expand/incremental_exhaustive_test.go @@ -0,0 +1,90 @@ +package expand + +import ( + "context" + "fmt" + "testing" + + "github.com/stretchr/testify/require" +) + +// TestIncrementalExhaustiveFourNodeGraphs gives bounded closure for the +// additions-only contract over every simple directed graph with 0..4 nodes, +// every possible edge set, and every seed node. Each non-empty cell adds one +// new direct member, then compares incremental output with both a fresh full +// expansion and the graph-library-free fixed-point oracle. +func TestIncrementalExhaustiveFourNodeGraphs(t *testing.T) { + const wantCells = 16_586 + ctx := context.Background() + cells := 0 + for nodeCount := 0; nodeCount <= 4; nodeCount++ { + entitlements := make([]string, nodeCount) + var possibleEdges []sqliteEdgeSpec + for i := 0; i < nodeCount; i++ { + entitlements[i] = fmt.Sprintf("e%d", i) + for j := 0; j < nodeCount; j++ { + if i != j { + possibleEdges = append(possibleEdges, sqliteEdgeSpec{ + src: fmt.Sprintf("e%d", i), + dst: fmt.Sprintf("e%d", j), + }) + } + } + } + edgeSets := 1 << len(possibleEdges) + if nodeCount == 0 { + cells++ // the single empty graph + continue + } + for edgeMask := 0; edgeMask < edgeSets; edgeMask++ { + edges := make([]sqliteEdgeSpec, 0, len(possibleEdges)) + for edgeIndex, edge := range possibleEdges { + if edgeMask&(1< 0 { + require.NoError(t, graph.AddEdge(ctx, "dense:"+itoa(i-1), id, false, nil)) + } + } + + _, err := NewIncrementalExpander(store, graph).ExpandChanges(ctx, nil, []string{"dense:0"}) + require.ErrorIs(t, err, ErrIncrementalDenseChangeDecline) +} + func itoa(i int) string { if i == 0 { return "0" diff --git a/pkg/sync/graph_compatibility_matrix_test.go b/pkg/sync/graph_compatibility_matrix_test.go new file mode 100644 index 000000000..30a6b2ebd --- /dev/null +++ b/pkg/sync/graph_compatibility_matrix_test.go @@ -0,0 +1,73 @@ +package sync //nolint:revive,nolintlint // package name kept for compatibility + +import ( + "context" + "testing" + + "github.com/stretchr/testify/require" +) + +// TestEntitlementGraphTokenCompatibilityMatrix records every token-side +// compatibility disposition. Pebble sidecar cells are exercised by +// TestGraphFromStore and synccompactor's TestCompactorGraphCompatibilityHealing. +func TestEntitlementGraphTokenCompatibilityMatrix(t *testing.T) { + ctx := context.Background() + stateWithGraph := newState() + graph := stateWithGraph.EntitlementGraph(ctx) + graph.AddEntitlementID("a") + graph.Loaded = true + graph.MarkExpansionComplete() + legacyToken, err := stateWithGraph.Marshal() + require.NoError(t, err) + + emptyState := newState() + emptyToken, err := emptyState.Marshal() + require.NoError(t, err) + + tests := []struct { + name string + token string + prepare bool + wantGraph bool + disposition string + }{ + { + name: "legacy graph in final token", token: legacyToken, wantGraph: true, + disposition: "legacy/SQLite reader may load graph from token", + }, + { + name: "final token without graph", token: emptyToken, + disposition: "missing graph selects full expansion", + }, + { + name: "replay clears legacy graph", token: legacyToken, prepare: true, + disposition: "replay rebuilds graph and cannot silently no-op", + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + token := tc.token + if tc.prepare { + token, err = PrepareExpansionReplayToken(token) + require.NoError(t, err) + } + got, err := GraphFromToken(token) + require.NoError(t, err) + require.Equal(t, tc.wantGraph, got != nil, tc.disposition) + }) + } +} + +// Cross-version dispositions that require a pinned external binary: +// +// - old writer, new reader: old artifact has no sidecar; new code full-expands +// and heals by writing a current sidecar (executable coverage: +// TestCompactorGraphCompatibilityHealing/old_pebble_without_sidecar). +// - new writer, old reader: the sidecar is an unknown engine-meta key. The old +// reader's grant data remains readable; it cannot opt into this new feature. +// This repository does not promise that an old SDK can perform incremental +// expansion on a new artifact. +// - SQLite old/new: graph-in-token remains readable, but compaction explicitly +// selects full expansion; SQLite-to-Pebble conversion heals and the next +// generation reuses the sidecar (executable coverage: +// TestCompactorGraphCompatibilityHealing/sqlite_converted_to_pebble). diff --git a/pkg/sync/graph_from_store_test.go b/pkg/sync/graph_from_store_test.go index 9f92e346a..fb91a8d78 100644 --- a/pkg/sync/graph_from_store_test.go +++ b/pkg/sync/graph_from_store_test.go @@ -35,7 +35,12 @@ func TestGraphFromStore(t *testing.T) { g.AddEntitlementID("ent-a") g.AddEntitlementID("ent-b") require.NoError(t, g.AddEdge(ctx, "ent-a", "ent-b", false, nil)) - data, err := expand.MarshalGraphBlob(syncID, g) + digestReader, ok := store.(c1zstore.GrantGenerationDigestReader) + require.True(t, ok) + digest, found, err := digestReader.GrantGenerationDigest(ctx) + require.NoError(t, err) + require.True(t, found) + data, err := expand.MarshalGraphBlobWithGrantDigest(syncID, g, digest) require.NoError(t, err) gs, ok := store.(EntitlementGraphStore) require.True(t, ok, "pebble store must implement EntitlementGraphStore") @@ -52,6 +57,26 @@ func TestGraphFromStore(t *testing.T) { require.NoError(t, err) require.Nil(t, got) + // A structurally valid graph bound to a different grant generation must + // fail closed. + wrongDigest := digest + wrongDigest.Hash = append([]byte(nil), digest.Hash...) + wrongDigest.Hash[0] ^= 0xff + wrongData, err := expand.MarshalGraphBlobWithGrantDigest(syncID, g, wrongDigest) + require.NoError(t, err) + require.NoError(t, gs.PutEntitlementGraphBlob(ctx, wrongData)) + got, err = GraphFromStore(ctx, store, syncID) + require.NoError(t, err) + require.Nil(t, got) + + // Current-format but unbound blobs are also not reusable. + unboundData, err := expand.MarshalGraphBlob(syncID, g) + require.NoError(t, err) + require.NoError(t, gs.PutEntitlementGraphBlob(ctx, unboundData)) + got, err = GraphFromStore(ctx, store, syncID) + require.NoError(t, err) + require.Nil(t, got) + // StartNewSync resets the keyspace, wiping the sidecar. _, err = store.StartNewSync(ctx, connectorstore.SyncTypeFull, "") require.NoError(t, err) diff --git a/pkg/sync/graph_golden_corpus_test.go b/pkg/sync/graph_golden_corpus_test.go new file mode 100644 index 000000000..49c76e92f --- /dev/null +++ b/pkg/sync/graph_golden_corpus_test.go @@ -0,0 +1,169 @@ +package sync //nolint:revive,nolintlint // package name kept for compatibility + +import ( + "bytes" + "context" + "encoding/json" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/conductorone/baton-sdk/pkg/connectorstore" + "github.com/conductorone/baton-sdk/pkg/dotc1z" + "github.com/conductorone/baton-sdk/pkg/dotc1z/c1zstore" + "github.com/conductorone/baton-sdk/pkg/sync/expand" +) + +// TestGraphSidecarGoldenCorpus is the non-aging, semantic form of the seven +// artifact corpus: each row generates one sealed c1z from its stated premise, +// closes it, reopens it read-only, and proves inspection never changes bytes. +// The old-binary physical artifact cells live in baton-compat-harness. +func TestGraphSidecarGoldenCorpus(t *testing.T) { + type corpusCase struct { + name string + mutate func(*testing.T, []byte) []byte + delete bool + wantGraph bool + wantError bool + } + tests := []corpusCase{ + {name: "old artifact without graph"}, + {name: "valid current graph", mutate: func(_ *testing.T, data []byte) []byte { return data }, wantGraph: true}, + {name: "foreign sync binding", mutate: mutateGraphEnvelope("sync_id", "foreign-sync")}, + {name: "unknown future version", mutate: mutateGraphEnvelope("format_version", float64(999))}, + {name: "truncated graph", mutate: func(_ *testing.T, _ []byte) []byte { return []byte("{truncated") }, wantError: true}, + {name: "graph grant digest mismatch", mutate: mutateGraphDigest}, + {name: "rollback invalidated graph", mutate: func(_ *testing.T, data []byte) []byte { return data }, delete: true}, + } + require.Len(t, tests, 7, "corpus size is a coverage guard") + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + ctx := context.Background() + path := filepath.Join(t.TempDir(), "corpus.c1z") + store, err := dotc1z.NewStore(ctx, path, + dotc1z.WithEngine(c1zstore.EnginePebble), dotc1z.WithTmpDir(t.TempDir())) + require.NoError(t, err) + syncID, err := store.StartNewSync(ctx, connectorstore.SyncTypeFull, "") + require.NoError(t, err) + require.NoError(t, store.EndSync(ctx)) + + if tc.mutate != nil { + digestReader, ok := store.(c1zstore.GrantGenerationDigestReader) + require.True(t, ok) + digest, found, digestErr := digestReader.GrantGenerationDigest(ctx) + require.NoError(t, digestErr) + require.True(t, found) + graph := expand.NewEntitlementGraph(ctx) + graph.AddEntitlementID("ent-a") + graph.MarkExpansionComplete() + graph.Loaded = true + graph.HasNoCycles = true + data, marshalErr := expand.MarshalGraphBlobWithGrantDigest(syncID, graph, digest) + require.NoError(t, marshalErr) + graphStore, ok := store.(EntitlementGraphStore) + require.True(t, ok) + require.NoError(t, graphStore.PutEntitlementGraphBlob(ctx, tc.mutate(t, data))) + if tc.delete { + require.NoError(t, graphStore.DeleteEntitlementGraphBlob(ctx)) + } + } + require.NoError(t, store.Close(ctx)) + + before, err := os.ReadFile(path) + require.NoError(t, err) + reader, err := dotc1z.NewStore(ctx, path, + dotc1z.WithReadOnly(true), dotc1z.WithTmpDir(t.TempDir())) + require.NoError(t, err) + graph, graphErr := GraphFromStore(ctx, reader, syncID) + if tc.wantError { + require.Error(t, graphErr) + } else { + require.NoError(t, graphErr) + } + require.Equal(t, tc.wantGraph, graph != nil) + require.NoError(t, reader.Close(ctx)) + after, err := os.ReadFile(path) + require.NoError(t, err) + require.True(t, bytes.Equal(before, after), "read-only corpus inspection mutated the artifact") + }) + } +} + +func TestGraphSidecarCloneAndCopyIsolation(t *testing.T) { + for _, operation := range []struct { + name string + copy func(context.Context, c1zstore.Store, string, string) error + }{ + {name: "clone", copy: func(ctx context.Context, source c1zstore.Store, path, syncID string) error { + return source.FileOps().CloneSync(ctx, path, syncID) + }}, + {name: "copy isolate", copy: func(ctx context.Context, source c1zstore.Store, path, syncID string) error { + return source.FileOps().CopyIsolateSync(ctx, path, syncID) + }}, + } { + t.Run(operation.name, func(t *testing.T) { + ctx := context.Background() + dir := t.TempDir() + sourcePath := filepath.Join(dir, "source.c1z") + source, err := dotc1z.NewStore(ctx, sourcePath, + dotc1z.WithEngine(c1zstore.EnginePebble), dotc1z.WithTmpDir(t.TempDir())) + require.NoError(t, err) + syncID, err := source.StartNewSync(ctx, connectorstore.SyncTypeFull, "") + require.NoError(t, err) + require.NoError(t, source.EndSync(ctx)) + + digestReader := source.(c1zstore.GrantGenerationDigestReader) + digest, found, err := digestReader.GrantGenerationDigest(ctx) + require.NoError(t, err) + require.True(t, found) + graph := expand.NewEntitlementGraph(ctx) + graph.AddEntitlementID("ent-a") + graph.MarkExpansionComplete() + graph.Loaded = true + graph.HasNoCycles = true + blob, err := expand.MarshalGraphBlobWithGrantDigest(syncID, graph, digest) + require.NoError(t, err) + require.NoError(t, source.(EntitlementGraphStore).PutEntitlementGraphBlob(ctx, blob)) + + outPath := filepath.Join(dir, "copy.c1z") + require.NoError(t, operation.copy(ctx, source, outPath, syncID)) + require.NoError(t, source.Close(ctx)) + + clone, err := dotc1z.NewStore(ctx, outPath, + dotc1z.WithReadOnly(true), dotc1z.WithTmpDir(t.TempDir())) + require.NoError(t, err) + clonedGraph, err := GraphFromStore(ctx, clone, syncID) + require.NoError(t, err) + require.NotNil(t, clonedGraph, "an exact clone must preserve its valid graph") + require.NoError(t, clonedGraph.ValidateCompleted()) + require.NoError(t, clone.Close(ctx)) + }) + } +} + +func mutateGraphEnvelope(key string, value any) func(*testing.T, []byte) []byte { + return func(t *testing.T, data []byte) []byte { + t.Helper() + var envelope map[string]any + require.NoError(t, json.Unmarshal(data, &envelope)) + envelope[key] = value + out, err := json.Marshal(envelope) + require.NoError(t, err) + return out + } +} + +func mutateGraphDigest(t *testing.T, data []byte) []byte { + t.Helper() + var envelope map[string]any + require.NoError(t, json.Unmarshal(data, &envelope)) + digest, ok := envelope["grant_digest"].(map[string]any) + require.True(t, ok) + digest["Count"] = digest["Count"].(float64) + 1 + out, err := json.Marshal(envelope) + require.NoError(t, err) + return out +} diff --git a/pkg/sync/ingest_invariants.go b/pkg/sync/ingest_invariants.go index bebe62d52..3bb9efbe6 100644 --- a/pkg/sync/ingest_invariants.go +++ b/pkg/sync/ingest_invariants.go @@ -163,8 +163,7 @@ func invariantVerdict(err error) error { return &invariantVerdictError{err: err} // TypeScopedGrants) land, they register here against I7 and I8 // respectively — type-granularity scopes are exactly the shapes those // referential checks exist for. -// -//nolint:gosec // G101 false positive: "PageTokens" is an annotation name, not a credential. +// #nosec G101 -- "PageTokens" is an annotation name, not a credential. var sideEffectAnnotationCoverage = map[string]string{ "c1.connector.v2.GrantExpandable": "I1: response-loop expansion arming (SetNeedsExpansion) + needs_expansion column persistence; store-derived probe arrives with replay", "c1.connector.v2.ExternalResourceMatch": "I2: response-loop match arming (SetHasExternalResourcesGrants); store-derived existence-bit repair arrives with replay", diff --git a/pkg/synccompactor/compactor_fold_test.go b/pkg/synccompactor/compactor_fold_test.go index 79d331ca0..a1f77e887 100644 --- a/pkg/synccompactor/compactor_fold_test.go +++ b/pkg/synccompactor/compactor_fold_test.go @@ -15,8 +15,24 @@ import ( "github.com/conductorone/baton-sdk/pkg/dotc1z/c1zstore" enginepkg "github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble" formatv3 "github.com/conductorone/baton-sdk/pkg/dotc1z/format/v3" + sdksync "github.com/conductorone/baton-sdk/pkg/sync" + "github.com/conductorone/baton-sdk/pkg/sync/expand" ) +func markFoldInputWithGraph(t *testing.T, ctx context.Context, path, syncID string) { + t.Helper() + store, err := dotc1z.NewStore(ctx, path, dotc1z.WithTmpDir(t.TempDir())) + require.NoError(t, err) + gs, ok := store.(sdksync.EntitlementGraphStore) + require.True(t, ok) + g := expand.NewEntitlementGraph(ctx) + g.AddEntitlementID("ent-a") + data, err := expand.MarshalGraphBlob(syncID, g) + require.NoError(t, err) + require.NoError(t, gs.PutEntitlementGraphBlob(ctx, data)) + require.NoError(t, store.Close(ctx)) +} + // grantDiscoveredAt reads one grant's discovered_at from the Pebble c1z // at path, under syncID. Fixture grant ids are connector-custom (no concat // shape), so the row is addressed by refs via the by_principal index. @@ -168,6 +184,34 @@ func TestCompactPebbleFoldMintsFreshSync(t *testing.T) { require.Positive(t, m.GetFoldDeadBytes(), "fold must record the overridden incumbent's bytes in fold_dead_bytes") } +func TestCompactPebbleFoldDoesNotPublishStaleGraphSidecar(t *testing.T) { + ctx := context.Background() + basePath := filepath.Join(t.TempDir(), "base.c1z") + partialPath := filepath.Join(t.TempDir(), "partial.c1z") + baseSyncID := buildPebbleInput(t, ctx, basePath, connectorstore.SyncTypeFull, "g-base") + partialSyncID := buildPebbleInput(t, ctx, partialPath, connectorstore.SyncTypePartial, "g-partial") + markFoldInputWithGraph(t, ctx, basePath, baseSyncID) + + c, cleanup, err := NewCompactor(ctx, t.TempDir(), []*CompactableSync{ + {FilePath: basePath, SyncID: baseSyncID}, + {FilePath: partialPath, SyncID: partialSyncID}, + }, WithTmpDir(t.TempDir()), WithEngine(c1zstore.EnginePebble), + WithPebbleCompactorMode(PebbleCompactorModeFold), WithSkipGrantExpansion()) + require.NoError(t, err) + defer func() { require.NoError(t, cleanup()) }() + out, err := c.Compact(ctx) + require.NoError(t, err) + + store, err := dotc1z.NewStore(ctx, out.FilePath, dotc1z.WithReadOnly(true), dotc1z.WithTmpDir(t.TempDir())) + require.NoError(t, err) + defer func() { require.NoError(t, store.Close(ctx)) }() + gs, ok := store.(sdksync.EntitlementGraphStore) + require.True(t, ok) + raw, err := gs.GetEntitlementGraphBlob(ctx) + require.NoError(t, err) + require.Nil(t, raw, "fold must delete an inherited graph stamped for the base sync") +} + // TestCompactPebbleFoldWasteAccumulates covers the fold-waste // accounting lifecycle: each fold adds the raw bytes it shadowed to // the manifest's fold_dead_bytes (carried forward from the base it diff --git a/pkg/synccompactor/incremental_benchmark_test.go b/pkg/synccompactor/incremental_benchmark_test.go new file mode 100644 index 000000000..1be674a3b --- /dev/null +++ b/pkg/synccompactor/incremental_benchmark_test.go @@ -0,0 +1,80 @@ +package synccompactor + +import ( + "context" + "runtime" + "testing" + + "github.com/conductorone/baton-sdk/pkg/dotc1z/c1zstore" +) + +func BenchmarkIncrementalCompactionHappy(b *testing.B) { + for _, incremental := range []bool{true, false} { + name := "full" + if incremental { + name = "incremental" + } + b.Run(name, func(b *testing.B) { + benchmarkIncrementalCompaction(b, incremental, false) + }) + } +} + +func BenchmarkIncrementalCompactionDecline(b *testing.B) { + for _, requested := range []bool{true, false} { + name := "full" + if requested { + name = "incremental-requested-declined" + } + b.Run(name, func(b *testing.B) { + benchmarkIncrementalCompaction(b, requested, true) + }) + } +} + +func benchmarkIncrementalCompaction(b *testing.B, requestIncremental, declining bool) { + ctx := context.Background() + b.ReportAllocs() + for i := 0; i < b.N; i++ { + b.StopTimer() + inputDir := b.TempDir() + var entries []*CompactableSync + if declining { + entries = buildSpecChangeFixtures(b, ctx, inputDir, false, true) + } else { + entries = buildIncrementalFixtures(b, ctx, inputDir) + } + options := []Option{WithTmpDir(b.TempDir()), WithEngine(c1zstore.EnginePebble)} + if requestIncremental { + options = append(options, WithIncrementalExpansion()) + } + compactor, cleanup, err := NewCompactor(ctx, b.TempDir(), entries, options...) + if err != nil { + b.Fatal(err) + } + runtime.GC() + var before runtime.MemStats + runtime.ReadMemStats(&before) + b.StartTimer() + _, err = compactor.Compact(ctx) + b.StopTimer() + if err != nil { + b.Fatal(err) + } + if err := cleanup(); err != nil { + b.Fatal(err) + } + reportCompactorHeapDelta(b, &before) + } +} + +func reportCompactorHeapDelta(b *testing.B, before *runtime.MemStats) { + var after runtime.MemStats + runtime.ReadMemStats(&after) + if after.TotalAlloc >= before.TotalAlloc { + b.ReportMetric(float64(after.TotalAlloc-before.TotalAlloc), "total-alloc-bytes/op") + } + if after.HeapInuse >= before.HeapInuse { + b.ReportMetric(float64(after.HeapInuse-before.HeapInuse), "heap-inuse-delta-bytes/op") + } +} diff --git a/pkg/synccompactor/incremental_closure_test.go b/pkg/synccompactor/incremental_closure_test.go new file mode 100644 index 000000000..a4828ae71 --- /dev/null +++ b/pkg/synccompactor/incremental_closure_test.go @@ -0,0 +1,430 @@ +package synccompactor + +import ( + "context" + "errors" + "path/filepath" + "sync" + "testing" + + "github.com/stretchr/testify/require" + + v2 "github.com/conductorone/baton-sdk/pb/c1/connector/v2" + "github.com/conductorone/baton-sdk/pkg/annotations" + "github.com/conductorone/baton-sdk/pkg/connectorstore" + "github.com/conductorone/baton-sdk/pkg/dotc1z" + "github.com/conductorone/baton-sdk/pkg/dotc1z/c1zstore" + sdksync "github.com/conductorone/baton-sdk/pkg/sync" + "github.com/conductorone/baton-sdk/pkg/sync/expand" + batonGrant "github.com/conductorone/baton-sdk/pkg/types/grant" +) + +func TestConcurrentDuplicateIncrementalCompactions(t *testing.T) { + ctx := context.Background() + entries := buildIncrementalFixtures(t, ctx, t.TempDir()) + type result struct { + grants []string + err error + } + results := make(chan result, 2) + var wg sync.WaitGroup + for i := 0; i < 2; i++ { + wg.Add(1) + go func() { + defer wg.Done() + compactor, cleanup, err := NewCompactor(ctx, t.TempDir(), entries, + WithTmpDir(t.TempDir()), WithEngine(c1zstore.EnginePebble), WithIncrementalExpansion()) + if err != nil { + results <- result{err: err} + return + } + defer cleanup() //nolint:errcheck // the result below verifies the artifact. + out, err := compactor.Compact(ctx) + if err != nil { + results <- result{err: err} + return + } + if !compactor.incrementalExpansionRan { + results <- result{err: errors.New("incremental expansion did not run")} + return + } + results <- result{grants: grantOutcome(t, ctx, out.FilePath, out.SyncID)} + }() + } + wg.Wait() + close(results) + var outcomes [][]string + for got := range results { + require.NoError(t, got.err) + outcomes = append(outcomes, got.grants) + } + require.Len(t, outcomes, 2) + require.Equal(t, outcomes[0], outcomes[1]) +} + +// The compactor has no session-store input. This pins the production default: +// incremental graph reuse must work when connector sessions are disabled. +func TestIncrementalCompactionWithNoSessionStore(t *testing.T) { + ctx := context.Background() + entries := buildIncrementalFixtures(t, ctx, t.TempDir()) + compactor, cleanup, err := NewCompactor(ctx, t.TempDir(), entries, + WithTmpDir(t.TempDir()), WithEngine(c1zstore.EnginePebble), WithIncrementalExpansion()) + require.NoError(t, err) + defer func() { require.NoError(t, cleanup()) }() + out, err := compactor.Compact(ctx) + require.NoError(t, err) + require.True(t, compactor.incrementalExpansionRan) + assertSealedCompactionArtifact(t, ctx, out, true) +} + +func TestCompactorIncrementalResourceTypeFilterChanges(t *testing.T) { + ctx := context.Background() + t.Run("widen", func(t *testing.T) { + incEntries := buildFilterChangeFixtures(t, ctx, t.TempDir(), []string{"user"}, nil) + inc, cleanupInc, err := NewCompactor(ctx, t.TempDir(), incEntries, + WithTmpDir(t.TempDir()), WithEngine(c1zstore.EnginePebble), WithIncrementalExpansion()) + require.NoError(t, err) + defer func() { require.NoError(t, cleanupInc()) }() + incOut, err := inc.Compact(ctx) + require.NoError(t, err) + require.True(t, inc.incrementalExpansionRan) + + fullEntries := buildFilterChangeFixtures(t, ctx, t.TempDir(), []string{"user"}, nil) + full, cleanupFull, err := NewCompactor(ctx, t.TempDir(), fullEntries, + WithTmpDir(t.TempDir()), WithEngine(c1zstore.EnginePebble)) + require.NoError(t, err) + defer func() { require.NoError(t, cleanupFull()) }() + fullOut, err := full.Compact(ctx) + require.NoError(t, err) + require.Equal(t, grantOutcome(t, ctx, fullOut.FilePath, fullOut.SyncID), + grantOutcome(t, ctx, incOut.FilePath, incOut.SyncID)) + hasGrant(t, grantOutcome(t, ctx, incOut.FilePath, incOut.SyncID), "ent-c|group|nested") + }) + + t.Run("narrow", func(t *testing.T) { + entries := buildFilterChangeFixtures(t, ctx, t.TempDir(), nil, []string{"user"}) + compactor, cleanup, err := NewCompactor(ctx, t.TempDir(), entries, + WithTmpDir(t.TempDir()), WithEngine(c1zstore.EnginePebble), WithIncrementalExpansion()) + require.NoError(t, err) + defer func() { require.NoError(t, cleanup()) }() + _, err = compactor.Compact(ctx) + require.NoError(t, err) + require.False(t, compactor.incrementalExpansionRan, "filter narrowing must decline") + }) +} + +func buildFilterChangeFixtures( + t testing.TB, + ctx context.Context, + dir string, + baseFilter, currentFilter []string, +) []*CompactableSync { + t.Helper() + groupB, groupC, nested := grp("grpB"), grp("grpC"), grp("nested") + alice := usr("alice") + entB, entC := ent("ent-b", groupB), ent("ent-c", groupC) + userRT := v2.ResourceType_builder{Id: "user", DisplayName: "User"}.Build() + groupRT := v2.ResourceType_builder{Id: "group", DisplayName: "Group"}.Build() + + basePath := filepath.Join(dir, "base.c1z") + base, err := dotc1z.NewStore(ctx, basePath, dotc1z.WithEngine(c1zstore.EnginePebble)) + require.NoError(t, err) + baseSyncID, err := base.StartNewSync(ctx, connectorstore.SyncTypeFull, "") + require.NoError(t, err) + require.NoError(t, base.PutResourceTypes(ctx, userRT, groupRT)) + require.NoError(t, base.PutResources(ctx, groupB, groupC, nested, alice)) + require.NoError(t, base.PutEntitlements(ctx, entB, entC)) + graph := expand.NewEntitlementGraph(ctx) + graph.AddEntitlementID("ent-b") + graph.AddEntitlementID("ent-c") + require.NoError(t, graph.AddEdge(ctx, "ent-b", "ent-c", false, baseFilter)) + require.NoError(t, base.PutGrants(ctx, + memberGrant(entB, alice), + memberGrant(entB, nested), + ruleGrantFilter(entC, groupB, "ent-b", baseFilter), + )) + require.NoError(t, graph.FixCycles(ctx)) + require.NoError(t, expand.NewExpander(sdksync.NewExpanderStore(base), graph).Run(ctx)) + graph.Loaded = true + graph.MarkExpansionComplete() + require.NoError(t, base.EndSync(ctx)) + persistFixtureGraph(t, ctx, base, baseSyncID, graph) + require.NoError(t, base.Close(ctx)) + + incPath := filepath.Join(dir, "inc.c1z") + inc, err := dotc1z.NewStore(ctx, incPath, dotc1z.WithEngine(c1zstore.EnginePebble)) + require.NoError(t, err) + incSyncID, err := inc.StartNewSync(ctx, connectorstore.SyncTypePartial, "") + require.NoError(t, err) + require.NoError(t, inc.PutResourceTypes(ctx, userRT, groupRT)) + require.NoError(t, inc.PutResources(ctx, groupB, groupC)) + require.NoError(t, inc.PutEntitlements(ctx, entB, entC)) + require.NoError(t, inc.PutGrants(ctx, ruleGrantFilter(entC, groupB, "ent-b", currentFilter))) + require.NoError(t, inc.EndSync(ctx)) + require.NoError(t, inc.Close(ctx)) + return []*CompactableSync{{FilePath: basePath, SyncID: baseSyncID}, {FilePath: incPath, SyncID: incSyncID}} +} + +func ruleGrantFilter(dest *v2.Entitlement, source *v2.Resource, sourceEntitlementID string, filter []string) *v2.Grant { + grant := v2.Grant_builder{ + Id: batonGrant.NewGrantID(source, dest), Entitlement: dest, Principal: source, + }.Build() + grant.SetAnnotations(annotations.New(v2.GrantExpandable_builder{ + EntitlementIds: []string{sourceEntitlementID}, ResourceTypeIds: filter, + }.Build())) + return grant +} + +func TestCompactorIncrementalKWayParity(t *testing.T) { + ctx := context.Background() + incEntries := buildNewMemberFixtures(t, ctx, t.TempDir()) + inc, cleanupInc, err := NewCompactor(ctx, t.TempDir(), incEntries, + WithTmpDir(t.TempDir()), WithEngine(c1zstore.EnginePebble), + WithPebbleCompactorMode(PebbleCompactorModeKWay), WithIncrementalExpansion()) + require.NoError(t, err) + defer func() { require.NoError(t, cleanupInc()) }() + incOut, err := inc.Compact(ctx) + require.NoError(t, err) + require.True(t, inc.incrementalExpansionRan) + + fullEntries := buildNewMemberFixtures(t, ctx, t.TempDir()) + full, cleanupFull, err := NewCompactor(ctx, t.TempDir(), fullEntries, + WithTmpDir(t.TempDir()), WithEngine(c1zstore.EnginePebble), + WithPebbleCompactorMode(PebbleCompactorModeKWay)) + require.NoError(t, err) + defer func() { require.NoError(t, cleanupFull()) }() + fullOut, err := full.Compact(ctx) + require.NoError(t, err) + require.Equal(t, grantOutcome(t, ctx, fullOut.FilePath, fullOut.SyncID), + grantOutcome(t, ctx, incOut.FilePath, incOut.SyncID)) +} + +func TestCompactorIncrementalThreeGenerationChain(t *testing.T) { + ctx := context.Background() + initial := buildIncrementalFixtures(t, ctx, t.TempDir()) + allInputs := append([]*CompactableSync(nil), initial...) + currentEntries := initial + var last *CompactableSync + for generation, user := range []string{"zoe", "yuki", "xavier"} { + if generation > 0 { + partial := buildMemberPartial(t, ctx, filepath.Join(t.TempDir(), user+".c1z"), "ent-b", "grpB", user) + allInputs = append(allInputs, partial) + currentEntries = []*CompactableSync{last, partial} + } + compactor, cleanup, err := NewCompactor(ctx, t.TempDir(), currentEntries, + WithTmpDir(t.TempDir()), WithEngine(c1zstore.EnginePebble), WithIncrementalExpansion()) + require.NoError(t, err) + out, err := compactor.Compact(ctx) + require.NoError(t, err) + require.True(t, compactor.incrementalExpansionRan) + require.NoError(t, cleanup()) + last = out + } + + full, cleanupFull, err := NewCompactor(ctx, t.TempDir(), allInputs, + WithTmpDir(t.TempDir()), WithEngine(c1zstore.EnginePebble)) + require.NoError(t, err) + defer func() { require.NoError(t, cleanupFull()) }() + fullOut, err := full.Compact(ctx) + require.NoError(t, err) + require.Equal(t, grantOutcome(t, ctx, fullOut.FilePath, fullOut.SyncID), + grantOutcome(t, ctx, last.FilePath, last.SyncID)) + assertSealedCompactionArtifact(t, ctx, last, true) +} + +// A partial can replace an existing grant record even though the format has no +// tombstones. This test distinguishes that case from a grant merely being +// absent from a partial sync. +func TestCompactorIncrementalDirectToIndirectReplacement(t *testing.T) { + ctx := context.Background() + build := func(t *testing.T, dir string) []*CompactableSync { + groupA, groupB := grp("group-a"), grp("group-b") + alice := usr("alice") + entA, entB := ent("ent-a", groupA), ent("ent-b", groupB) + userRT := v2.ResourceType_builder{Id: "user", DisplayName: "User"}.Build() + groupRT := v2.ResourceType_builder{Id: "group", DisplayName: "Group"}.Build() + + basePath := filepath.Join(dir, "base.c1z") + base, err := dotc1z.NewStore(ctx, basePath, dotc1z.WithEngine(c1zstore.EnginePebble)) + require.NoError(t, err) + baseSyncID, err := base.StartNewSync(ctx, connectorstore.SyncTypeFull, "") + require.NoError(t, err) + require.NoError(t, base.PutResourceTypes(ctx, userRT, groupRT)) + require.NoError(t, base.PutResources(ctx, groupA, groupB, alice)) + require.NoError(t, base.PutEntitlements(ctx, entA, entB)) + require.NoError(t, base.PutGrants(ctx, + memberGrant(entA, alice), + expandedGrant(entB, alice, entA.GetId()), + ruleGrantSpec(entB, groupA, entA.GetId(), true), + )) + require.NoError(t, base.EndSync(ctx)) + graph := expand.NewEntitlementGraph(ctx) + graph.AddEntitlementID(entA.GetId()) + graph.AddEntitlementID(entB.GetId()) + require.NoError(t, graph.AddEdge(ctx, entA.GetId(), entB.GetId(), true, nil)) + graph.MarkEdgeExpanded(entA.GetId(), entB.GetId()) + graph.Loaded = true + graph.HasNoCycles = true + persistFixtureGraph(t, ctx, base, baseSyncID, graph) + require.NoError(t, base.Close(ctx)) + + partialPath := filepath.Join(dir, "partial.c1z") + partial, err := dotc1z.NewStore(ctx, partialPath, dotc1z.WithEngine(c1zstore.EnginePebble)) + require.NoError(t, err) + partialSyncID, err := partial.StartNewSync(ctx, connectorstore.SyncTypePartial, "") + require.NoError(t, err) + require.NoError(t, partial.PutResourceTypes(ctx, userRT, groupRT)) + require.NoError(t, partial.PutResources(ctx, groupA, alice)) + require.NoError(t, partial.PutEntitlements(ctx, entA)) + replacement := memberGrant(entA, alice) + replacement.SetSources(v2.GrantSources_builder{ + Sources: map[string]*v2.GrantSources_GrantSource{"some-other-source": {}}, + }.Build()) + require.NoError(t, partial.PutGrants(ctx, replacement)) + require.NoError(t, partial.EndSync(ctx)) + require.NoError(t, partial.Close(ctx)) + + return []*CompactableSync{ + {FilePath: basePath, SyncID: baseSyncID}, + {FilePath: partialPath, SyncID: partialSyncID}, + } + } + + incEntries := build(t, t.TempDir()) + inc, incCleanup, err := NewCompactor(ctx, t.TempDir(), incEntries, + WithTmpDir(t.TempDir()), WithEngine(c1zstore.EnginePebble), WithIncrementalExpansion()) + require.NoError(t, err) + defer func() { require.NoError(t, incCleanup()) }() + incOut, err := inc.Compact(ctx) + require.NoError(t, err) + require.True(t, inc.incrementalExpansionRan, "the replacement currently remains eligible for incremental expansion") + + fullEntries := build(t, t.TempDir()) + full, fullCleanup, err := NewCompactor(ctx, t.TempDir(), fullEntries, + WithTmpDir(t.TempDir()), WithEngine(c1zstore.EnginePebble)) + require.NoError(t, err) + defer func() { require.NoError(t, fullCleanup()) }() + fullOut, err := full.Compact(ctx) + require.NoError(t, err) + + incGrants := grantOutcome(t, ctx, incOut.FilePath, incOut.SyncID) + fullGrants := grantOutcome(t, ctx, fullOut.FilePath, fullOut.SyncID) + require.Equal(t, + fullGrants, + incGrants, + "incremental replacement handling must equal full expansion", + ) +} + +func TestCompactorIncrementalRejectsGraphIncoherentWithBaseGrants(t *testing.T) { + ctx := context.Background() + build := func(t *testing.T, dir string) []*CompactableSync { + groupA, groupB := grp("group-a"), grp("group-b") + alice := usr("alice") + entA, entB := ent("ent-a", groupA), ent("ent-b", groupB) + userRT := v2.ResourceType_builder{Id: "user", DisplayName: "User"}.Build() + groupRT := v2.ResourceType_builder{Id: "group", DisplayName: "Group"}.Build() + + basePath := filepath.Join(dir, "base.c1z") + base, err := dotc1z.NewStore(ctx, basePath, dotc1z.WithEngine(c1zstore.EnginePebble)) + require.NoError(t, err) + baseSyncID, err := base.StartNewSync(ctx, connectorstore.SyncTypeFull, "") + require.NoError(t, err) + require.NoError(t, base.PutResourceTypes(ctx, userRT, groupRT)) + require.NoError(t, base.PutResources(ctx, groupA, groupB, alice)) + require.NoError(t, base.PutEntitlements(ctx, entA, entB)) + // The sidecar claims A -> B was expanded, but Alice's derived B grant + // is deliberately absent. + require.NoError(t, base.PutGrants(ctx, + memberGrant(entA, alice), + ruleGrant(entB, groupA, entA.GetId()), + )) + require.NoError(t, base.EndSync(ctx)) + graph := expand.NewEntitlementGraph(ctx) + graph.AddEntitlementID(entA.GetId()) + graph.AddEntitlementID(entB.GetId()) + require.NoError(t, graph.AddEdge(ctx, entA.GetId(), entB.GetId(), false, nil)) + graph.MarkEdgeExpanded(entA.GetId(), entB.GetId()) + graph.Loaded = true + graph.HasNoCycles = true + digestReader, ok := base.(c1zstore.GrantGenerationDigestReader) + require.True(t, ok) + digest, found, err := digestReader.GrantGenerationDigest(ctx) + require.NoError(t, err) + require.True(t, found) + // Simulate a graph copied from a different grant generation while + // retaining a valid structure, sync ID, and verification marker. + digest.Hash[0] ^= 0xff + data, err := expand.MarshalGraphBlobWithGrantDigest(baseSyncID, graph, digest) + require.NoError(t, err) + graphStore, ok := base.(sdksync.EntitlementGraphStore) + require.True(t, ok) + require.NoError(t, graphStore.PutEntitlementGraphBlob(ctx, data)) + verificationWriter, ok := base.SyncMeta().(c1zstore.IngestInvariantVerificationWriter) + require.True(t, ok) + require.NoError(t, verificationWriter.MarkIngestInvariantsVerified(ctx, baseSyncID, c1zstore.IngestInvariantVerification{ + Generation: sdksync.IngestInvariantGeneration, + Coverage: []string{"test-fixture"}, + Mode: c1zstore.IngestInvariantVerificationModeConnector, + })) + require.NoError(t, base.Close(ctx)) + + partial := buildEmptyPartial(t, ctx, filepath.Join(dir, "partial.c1z")) + return []*CompactableSync{ + {FilePath: basePath, SyncID: baseSyncID}, + partial, + } + } + + entries := build(t, t.TempDir()) + compactor, cleanup, err := NewCompactor(ctx, t.TempDir(), entries, + WithTmpDir(t.TempDir()), WithEngine(c1zstore.EnginePebble), WithIncrementalExpansion()) + require.NoError(t, err) + defer func() { require.NoError(t, cleanup()) }() + out, err := compactor.Compact(ctx) + require.NoError(t, err) + require.False(t, compactor.incrementalExpansionRan, + "a sidecar that is not coherent with its base grants must fall back to full expansion") + hasGrant(t, grantOutcome(t, ctx, out.FilePath, out.SyncID), "ent-b|user|alice") +} + +func TestCompactorGraphCompatibilityHealing(t *testing.T) { + ctx := context.Background() + t.Run("old pebble without sidecar", func(t *testing.T) { + entries := buildIncrementalFixtures(t, ctx, t.TempDir()) + store, err := dotc1z.NewStore(ctx, entries[0].FilePath, dotc1z.WithTmpDir(t.TempDir())) + require.NoError(t, err) + graphStore, ok := store.(interface{ DeleteEntitlementGraphBlob(context.Context) error }) + require.True(t, ok) + require.NoError(t, graphStore.DeleteEntitlementGraphBlob(ctx)) + require.NoError(t, store.Close(ctx)) + assertFallbackHealsAndReuses(t, ctx, entries) + }) + + t.Run("sqlite converted to pebble", func(t *testing.T) { + entries := buildIncrementalFixturesEngine(t, ctx, t.TempDir(), c1zstore.EngineSQLite) + assertFallbackHealsAndReuses(t, ctx, entries) + }) +} + +func assertFallbackHealsAndReuses(t *testing.T, ctx context.Context, entries []*CompactableSync) { + t.Helper() + first, cleanupFirst, err := NewCompactor(ctx, t.TempDir(), entries, + WithTmpDir(t.TempDir()), WithEngine(c1zstore.EnginePebble), WithIncrementalExpansion()) + require.NoError(t, err) + firstOut, err := first.Compact(ctx) + require.NoError(t, err) + require.False(t, first.incrementalExpansionRan) + require.NoError(t, cleanupFirst()) + assertSealedCompactionArtifact(t, ctx, firstOut, true) + + partial := buildMemberPartial(t, ctx, filepath.Join(t.TempDir(), "next.c1z"), "ent-b", "grpB", "next-user") + second, cleanupSecond, err := NewCompactor(ctx, t.TempDir(), []*CompactableSync{firstOut, partial}, + WithTmpDir(t.TempDir()), WithEngine(c1zstore.EnginePebble), WithIncrementalExpansion()) + require.NoError(t, err) + defer func() { require.NoError(t, cleanupSecond()) }() + secondOut, err := second.Compact(ctx) + require.NoError(t, err) + require.True(t, second.incrementalExpansionRan) + assertSealedCompactionArtifact(t, ctx, secondOut, true) +} diff --git a/pkg/synccompactor/incremental_differential_test.go b/pkg/synccompactor/incremental_differential_test.go new file mode 100644 index 000000000..c42e173b7 --- /dev/null +++ b/pkg/synccompactor/incremental_differential_test.go @@ -0,0 +1,243 @@ +package synccompactor + +import ( + "context" + "fmt" + "math/rand" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" + + v2 "github.com/conductorone/baton-sdk/pb/c1/connector/v2" + reader_v2 "github.com/conductorone/baton-sdk/pb/c1/reader/v2" + "github.com/conductorone/baton-sdk/pkg/connectorstore" + "github.com/conductorone/baton-sdk/pkg/dotc1z" + "github.com/conductorone/baton-sdk/pkg/dotc1z/c1zstore" + sdksync "github.com/conductorone/baton-sdk/pkg/sync" + "github.com/conductorone/baton-sdk/pkg/sync/expand" +) + +// TestCompactorIncrementalDifferentialRandom is Stage 2b's real-artifact +// oracle. For each deterministic seed it writes equivalent base+partial +// Pebble c1z inputs, compacts one through incremental expansion and one through +// full expansion, and compares complete grant rows (including provenance). +func TestCompactorIncrementalDifferentialRandom(t *testing.T) { + const cases = 10 + incrementalRuns := 0 + for seed := int64(0); seed < cases; seed++ { + seed := seed + t.Run(fmt.Sprintf("seed=%d", seed), func(t *testing.T) { + ctx := context.Background() + + incEntries := buildRandomDifferentialFixtures(t, ctx, t.TempDir(), seed) + incCompactor, incCleanup, err := NewCompactor(ctx, t.TempDir(), incEntries, + WithTmpDir(t.TempDir()), + WithEngine(c1zstore.EnginePebble), + WithIncrementalExpansion(), + ) + require.NoError(t, err) + defer func() { require.NoError(t, incCleanup()) }() + incOut, err := incCompactor.Compact(ctx) + require.NoError(t, err) + require.NotNil(t, incOut) + require.True(t, incCompactor.incrementalExpansionRan, + "seed=%d silently fell back instead of exercising Stage 2b", seed) + incrementalRuns++ + + fullEntries := buildRandomDifferentialFixtures(t, ctx, t.TempDir(), seed) + fullCompactor, fullCleanup, err := NewCompactor(ctx, t.TempDir(), fullEntries, + WithTmpDir(t.TempDir()), + WithEngine(c1zstore.EnginePebble), + ) + require.NoError(t, err) + defer func() { require.NoError(t, fullCleanup()) }() + fullOut, err := fullCompactor.Compact(ctx) + require.NoError(t, err) + require.NotNil(t, fullOut) + + require.Equal(t, + grantOutcome(t, ctx, fullOut.FilePath, fullOut.SyncID), + grantOutcome(t, ctx, incOut.FilePath, incOut.SyncID), + "seed=%d incremental grants/provenance differ from full expansion", seed, + ) + assertSealedCompactionArtifact(t, ctx, incOut, true) + assertSealedCompactionArtifact(t, ctx, fullOut, false) + }) + } + require.Equal(t, cases, incrementalRuns, "every additive case must use the incremental path") +} + +func buildRandomDifferentialFixtures( + t *testing.T, + ctx context.Context, + dir string, + seed int64, +) []*CompactableSync { + t.Helper() + rng := rand.New(rand.NewSource(seed)) //nolint:gosec // deterministic test fixture + nodeCount := 4 + rng.Intn(4) + + groups := make([]*v2.Resource, nodeCount) + entitlements := make([]*v2.Entitlement, nodeCount) + for i := 0; i < nodeCount; i++ { + groups[i] = grp(fmt.Sprintf("seed-%d-group-%d", seed, i)) + entitlements[i] = ent(fmt.Sprintf("seed-%d-ent-%d", seed, i), groups[i]) + } + + type edge struct{ source, destination int } + edges := make([]edge, 0, nodeCount*2) + edgeSet := make(map[[2]int]struct{}) + addEdge := func(source, destination int) { + key := [2]int{source, destination} + if _, exists := edgeSet[key]; exists { + return + } + edgeSet[key] = struct{}{} + edges = append(edges, edge{source: source, destination: destination}) + } + // A spine guarantees useful transitive expansion. Extra forward edges add + // diamonds and multi-parent provenance while keeping the graph acyclic. + for i := 0; i+1 < nodeCount; i++ { + addEdge(i, i+1) + } + for i := 0; i < nodeCount; i++ { + for j := i + 2; j < nodeCount; j++ { + if rng.Intn(3) == 0 { + addEdge(i, j) + } + } + } + + userRT := v2.ResourceType_builder{Id: "user", DisplayName: "User"}.Build() + groupRT := v2.ResourceType_builder{Id: "group", DisplayName: "Group"}.Build() + users := make([]*v2.Resource, nodeCount) + for i := range users { + users[i] = usr(fmt.Sprintf("seed-%d-user-%d", seed, i)) + } + + basePath := filepath.Join(dir, "base.c1z") + base, err := dotc1z.NewStore(ctx, basePath, dotc1z.WithEngine(c1zstore.EnginePebble)) + require.NoError(t, err) + baseSyncID, err := base.StartNewSync(ctx, connectorstore.SyncTypeFull, "") + require.NoError(t, err) + require.NoError(t, base.PutResourceTypes(ctx, userRT, groupRT)) + baseResources := append(append([]*v2.Resource(nil), groups...), users...) + require.NoError(t, base.PutResources(ctx, baseResources...)) + require.NoError(t, base.PutEntitlements(ctx, entitlements...)) + + baseGraph := expand.NewEntitlementGraph(ctx) + for _, entitlement := range entitlements { + baseGraph.AddEntitlementID(entitlement.GetId()) + } + grants := make([]*v2.Grant, 0, len(users)+len(edges)) + for i, user := range users { + // Every node gets one direct member; repeated downstream paths exercise + // union and de-duplication of provenance. + grants = append(grants, memberGrant(entitlements[i], user)) + } + for _, e := range edges { + grants = append(grants, ruleGrant(entitlements[e.destination], groups[e.source], entitlements[e.source].GetId())) + require.NoError(t, baseGraph.AddEdge(ctx, + entitlements[e.source].GetId(), entitlements[e.destination].GetId(), false, nil)) + } + require.NoError(t, base.PutGrants(ctx, grants...)) + require.NoError(t, baseGraph.FixCycles(ctx)) + require.NoError(t, expand.NewExpander(sdksync.NewExpanderStore(base), baseGraph).Run(ctx)) + baseGraph.Loaded = true + baseGraph.MarkExpansionComplete() + require.NoError(t, base.EndSync(ctx)) + persistFixtureGraph(t, ctx, base, baseSyncID, baseGraph) + require.NoError(t, base.Close(ctx)) + + incPath := filepath.Join(dir, "increment.c1z") + inc, err := dotc1z.NewStore(ctx, incPath, dotc1z.WithEngine(c1zstore.EnginePebble)) + require.NoError(t, err) + incSyncID, err := inc.StartNewSync(ctx, connectorstore.SyncTypePartial, "") + require.NoError(t, err) + require.NoError(t, inc.PutResourceTypes(ctx, userRT, groupRT)) + + if seed%2 == 0 { + // Membership addition on an existing entitlement. + target := rng.Intn(nodeCount - 1) + newUser := usr(fmt.Sprintf("seed-%d-delta-user", seed)) + require.NoError(t, inc.PutResources(ctx, groups[target], newUser)) + require.NoError(t, inc.PutEntitlements(ctx, entitlements[target])) + require.NoError(t, inc.PutGrants(ctx, memberGrant(entitlements[target], newUser))) + } else { + // Brand-new forward edge. A missing direct edge may already have a + // transitive path; that still exercises provenance reconciliation. + source, destination, ok := missingRandomEdge(nodeCount, edgeSet) + require.True(t, ok) + require.NoError(t, inc.PutResources(ctx, groups[source], groups[destination])) + require.NoError(t, inc.PutEntitlements(ctx, entitlements[source], entitlements[destination])) + require.NoError(t, inc.PutGrants(ctx, + ruleGrant(entitlements[destination], groups[source], entitlements[source].GetId()))) + } + require.NoError(t, inc.EndSync(ctx)) + require.NoError(t, inc.Close(ctx)) + + return []*CompactableSync{ + {FilePath: basePath, SyncID: baseSyncID}, + {FilePath: incPath, SyncID: incSyncID}, + } +} + +func missingRandomEdge(nodeCount int, existing map[[2]int]struct{}) (int, int, bool) { + for distance := 2; distance < nodeCount; distance++ { + for source := 0; source+distance < nodeCount; source++ { + destination := source + distance + if _, found := existing[[2]int{source, destination}]; !found { + return source, destination, true + } + } + } + return 0, 0, false +} + +// assertSealedCompactionArtifact is the Stage 3.2 artifact fsck. It validates +// the sealed sync record and invariant marker for every compaction path, and +// validates graph ownership, completion, and serialization when the path +// promises a sidecar. +func assertSealedCompactionArtifact( + t *testing.T, + ctx context.Context, + out *CompactableSync, + expectGraph bool, +) { + t.Helper() + store, err := dotc1z.NewStore(ctx, out.FilePath, + dotc1z.WithReadOnly(true), dotc1z.WithTmpDir(t.TempDir())) + require.NoError(t, err) + defer func() { require.NoError(t, store.Close(ctx)) }() + + finished, err := store.GetLatestFinishedSync(ctx, + reader_v2.SyncsReaderServiceGetLatestFinishedSyncRequest_builder{}.Build()) + require.NoError(t, err) + require.Equal(t, out.SyncID, finished.GetSync().GetId(), "artifact must be sealed") + + run, err := store.SyncMeta().LatestFinishedSyncOfAnyType(ctx) + require.NoError(t, err) + require.Equal(t, out.SyncID, run.ID) + require.NotNil(t, run.EndedAt) + require.True(t, run.IsVerified()) + require.Equal(t, sdksync.IngestInvariantGeneration, run.Generation) + require.Equal(t, c1zstore.IngestInvariantVerificationModeCompactionMerge, run.Mode) + + graph, err := sdksync.GraphFromStore(ctx, store, out.SyncID) + require.NoError(t, err) + if !expectGraph { + require.Nil(t, graph, "non-incremental artifact must not inherit a stale graph") + return + } + require.NotNil(t, graph, "artifact must carry a graph stamped for its output sync") + require.NoError(t, graph.ValidateCompleted()) + + data, err := expand.MarshalGraphBlob(out.SyncID, graph) + require.NoError(t, err) + roundTrip, err := expand.UnmarshalGraphBlob(data, out.SyncID) + require.NoError(t, err) + require.NotNil(t, roundTrip) + require.NoError(t, roundTrip.ValidateCompleted()) + require.Equal(t, graph, roundTrip, "graph must survive a serialization round trip") +} diff --git a/pkg/synccompactor/incremental_expansion_test.go b/pkg/synccompactor/incremental_expansion_test.go index 86f830d29..fba338d7b 100644 --- a/pkg/synccompactor/incremental_expansion_test.go +++ b/pkg/synccompactor/incremental_expansion_test.go @@ -72,17 +72,131 @@ func ent(id string, resource *v2.Resource) *v2.Entitlement { // destination entitlement, with a GrantExpandable annotation naming the source // entitlement — i.e. "members of sourceEntID also get destEnt". func ruleGrant(destEnt *v2.Entitlement, sourceGroup *v2.Resource, sourceEntID string) *v2.Grant { + return ruleGrantSources(destEnt, sourceGroup, []string{sourceEntID}) +} + +func ruleGrantSources(destEnt *v2.Entitlement, sourceGroup *v2.Resource, sourceEntIDs []string) *v2.Grant { g := v2.Grant_builder{ Id: batonGrant.NewGrantID(sourceGroup, destEnt), Entitlement: destEnt, Principal: sourceGroup, }.Build() g.SetAnnotations(annotations.New(v2.GrantExpandable_builder{ - EntitlementIds: []string{sourceEntID}, + EntitlementIds: sourceEntIDs, }.Build())) return g } +func persistFixtureGraph(t testing.TB, ctx context.Context, store c1zstore.Store, syncID string, graph *expand.EntitlementGraph) { + t.Helper() + gs, ok := store.(sdksync.EntitlementGraphStore) + if !ok { + return + } + digestReader, ok := store.(c1zstore.GrantGenerationDigestReader) + require.True(t, ok) + digest, found, err := digestReader.GrantGenerationDigest(ctx) + require.NoError(t, err) + require.True(t, found) + data, err := expand.MarshalGraphBlobWithGrantDigest(syncID, graph, digest) + require.NoError(t, err) + require.NoError(t, gs.PutEntitlementGraphBlob(ctx, data)) + verificationWriter, ok := store.SyncMeta().(c1zstore.IngestInvariantVerificationWriter) + require.True(t, ok) + require.NoError(t, verificationWriter.MarkIngestInvariantsVerified(ctx, syncID, c1zstore.IngestInvariantVerification{ + Generation: sdksync.IngestInvariantGeneration, + Coverage: []string{"test-fixture"}, + Mode: c1zstore.IngestInvariantVerificationModeConnector, + })) +} + +func overwriteFixtureGraph(t *testing.T, ctx context.Context, path, stampedSyncID string, graph *expand.EntitlementGraph) { + t.Helper() + store, err := dotc1z.NewStore(ctx, path, dotc1z.WithTmpDir(t.TempDir())) + require.NoError(t, err) + data, err := expand.MarshalGraphBlob(stampedSyncID, graph) + require.NoError(t, err) + graphStore, ok := store.(sdksync.EntitlementGraphStore) + require.True(t, ok) + require.NoError(t, graphStore.PutEntitlementGraphBlob(ctx, data)) + require.NoError(t, store.Close(ctx)) +} + +func overwriteFixtureGraphRaw(t *testing.T, ctx context.Context, path string, data []byte) { + t.Helper() + store, err := dotc1z.NewStore(ctx, path, dotc1z.WithTmpDir(t.TempDir())) + require.NoError(t, err) + gs, ok := store.(sdksync.EntitlementGraphStore) + require.True(t, ok) + require.NoError(t, gs.PutEntitlementGraphBlob(ctx, data)) + require.NoError(t, store.Close(ctx)) +} + +func buildEmptyPartial(t *testing.T, ctx context.Context, path string) *CompactableSync { + t.Helper() + store, err := dotc1z.NewStore(ctx, path, dotc1z.WithEngine(c1zstore.EnginePebble)) + require.NoError(t, err) + syncID, err := store.StartNewSync(ctx, connectorstore.SyncTypePartial, "") + require.NoError(t, err) + require.NoError(t, store.EndSync(ctx)) + require.NoError(t, store.Close(ctx)) + return &CompactableSync{FilePath: path, SyncID: syncID} +} + +func buildDroppedEdgeFixtures(t *testing.T, ctx context.Context, dir string) []*CompactableSync { + t.Helper() + grpA, grpB, grpC := grp("grpA"), grp("grpB"), grp("grpC") + bob := usr("bob") + entA, entB, entC := ent("ent-a", grpA), ent("ent-b", grpB), ent("ent-c", grpC) + userRT := v2.ResourceType_builder{Id: "user", DisplayName: "User"}.Build() + groupRT := v2.ResourceType_builder{Id: "group", DisplayName: "Group"}.Build() + + basePath := filepath.Join(dir, "base.c1z") + base, err := dotc1z.NewStore(ctx, basePath, dotc1z.WithEngine(c1zstore.EnginePebble)) + require.NoError(t, err) + baseSyncID, err := base.StartNewSync(ctx, connectorstore.SyncTypeFull, "") + require.NoError(t, err) + require.NoError(t, base.PutResourceTypes(ctx, userRT, groupRT)) + require.NoError(t, base.PutResources(ctx, grpA, grpB, grpC)) + require.NoError(t, base.PutEntitlements(ctx, entA, entB, entC)) + require.NoError(t, base.PutGrants(ctx, ruleGrantSources(entC, grpA, []string{"ent-a", "ent-b"}))) + require.NoError(t, base.EndSync(ctx)) + persistFixtureGraph(t, ctx, base, baseSyncID, droppedEdgeBaseGraph(t, ctx)) + require.NoError(t, base.Close(ctx)) + + incPath := filepath.Join(dir, "inc.c1z") + inc, err := dotc1z.NewStore(ctx, incPath, dotc1z.WithEngine(c1zstore.EnginePebble)) + require.NoError(t, err) + incSyncID, err := inc.StartNewSync(ctx, connectorstore.SyncTypePartial, "") + require.NoError(t, err) + require.NoError(t, inc.PutResourceTypes(ctx, userRT, groupRT)) + require.NoError(t, inc.PutResources(ctx, grpA, grpB, grpC, bob)) + require.NoError(t, inc.PutEntitlements(ctx, entA, entB, entC)) + require.NoError(t, inc.PutGrants(ctx, + ruleGrantSources(entC, grpA, []string{"ent-a"}), // drops ent-b -> ent-c + memberGrant(entB, bob), + )) + require.NoError(t, inc.EndSync(ctx)) + require.NoError(t, inc.Close(ctx)) + + return []*CompactableSync{{FilePath: basePath, SyncID: baseSyncID}, {FilePath: incPath, SyncID: incSyncID}} +} + +func droppedEdgeBaseGraph(t *testing.T, ctx context.Context) *expand.EntitlementGraph { + t.Helper() + g := expand.NewEntitlementGraph(ctx) + for _, id := range []string{"ent-a", "ent-b", "ent-c"} { + g.AddEntitlementID(id) + } + for _, src := range []string{"ent-a", "ent-b"} { + require.NoError(t, g.AddEdge(ctx, src, "ent-c", false, nil)) + g.MarkEdgeExpanded(src, "ent-c") + } + g.Loaded = true + g.HasNoCycles = true + return g +} + // ruleGrantSpec is ruleGrant with an explicit shallow flag, for edge-spec // change tests. func ruleGrantSpec(destEnt *v2.Entitlement, sourceGroup *v2.Resource, sourceEntID string, shallow bool) *v2.Grant { @@ -119,13 +233,13 @@ func expandedGrant(e *v2.Entitlement, principal *v2.Resource, sourceEntID string // buildIncrementalFixtures writes a base (full, pre-expanded) c1z and an // increment (partial) c1z into dir, returning the compactable entries. -func buildIncrementalFixtures(t *testing.T, ctx context.Context, dir string) []*CompactableSync { +func buildIncrementalFixtures(t testing.TB, ctx context.Context, dir string) []*CompactableSync { return buildIncrementalFixturesEngine(t, ctx, dir, c1zstore.EnginePebble) } // buildIncrementalFixturesEngine is buildIncrementalFixtures with a chosen // storage engine, so the SQLite degrade path can be exercised too. -func buildIncrementalFixturesEngine(t *testing.T, ctx context.Context, dir string, engine c1zstore.Engine) []*CompactableSync { +func buildIncrementalFixturesEngine(t testing.TB, ctx context.Context, dir string, engine c1zstore.Engine) []*CompactableSync { t.Helper() grpA, grpB, grpC := grp("grpA"), grp("grpB"), grp("grpC") @@ -150,6 +264,7 @@ func buildIncrementalFixturesEngine(t *testing.T, ctx context.Context, dir strin ruleGrant(entC, grpB, "ent-b"), // rule: members of B get C )) require.NoError(t, base.EndSync(ctx)) + persistFixtureGraph(t, ctx, base, baseSyncID, baseGraphForFixtures(t, ctx)) require.NoError(t, base.Close(ctx)) // --- increment: partial, adds ent-a -> ent-b with sam on A --- @@ -177,13 +292,15 @@ func buildIncrementalFixturesEngine(t *testing.T, ctx context.Context, dir strin // baseGraphForFixtures returns the in-memory graph the base sync would have // persisted (ent-b -> ent-c, already expanded) — what sync.GraphFromToken // would hand back in production. -func baseGraphForFixtures(t *testing.T, ctx context.Context) *expand.EntitlementGraph { +func baseGraphForFixtures(t testing.TB, ctx context.Context) *expand.EntitlementGraph { t.Helper() g := expand.NewEntitlementGraph(ctx) g.AddEntitlementID("ent-b") g.AddEntitlementID("ent-c") require.NoError(t, g.AddEdge(ctx, "ent-b", "ent-c", false, nil)) g.MarkEdgeExpanded("ent-b", "ent-c") + g.Loaded = true + g.HasNoCycles = true return g } @@ -238,7 +355,7 @@ func TestCompactor_IncrementalExpansionMatchesFull(t *testing.T) { cInc, cleanupInc, err := NewCompactor(ctx, t.TempDir(), incEntries, WithTmpDir(t.TempDir()), WithEngine(c1zstore.EnginePebble), - WithIncrementalExpansion(baseGraphForFixtures(t, ctx)), + WithIncrementalExpansion(), ) require.NoError(t, err) defer func() { _ = cleanupInc() }() @@ -300,6 +417,7 @@ func buildNewMemberFixtures(t *testing.T, ctx context.Context, dir string) []*Co ruleGrant(entC, grpB, "ent-b"), )) require.NoError(t, base.EndSync(ctx)) + persistFixtureGraph(t, ctx, base, baseSyncID, baseGraphForFixtures(t, ctx)) require.NoError(t, base.Close(ctx)) // increment: partial, adds bob as a direct member of the EXISTING ent-b. @@ -322,6 +440,25 @@ func buildNewMemberFixtures(t *testing.T, ctx context.Context, dir string) []*Co } } +func buildMemberPartial(t *testing.T, ctx context.Context, path, entitlementID, groupID, userID string) *CompactableSync { + t.Helper() + group, user := grp(groupID), usr(userID) + entitlement := ent(entitlementID, group) + store, err := dotc1z.NewStore(ctx, path, dotc1z.WithEngine(c1zstore.EnginePebble)) + require.NoError(t, err) + syncID, err := store.StartNewSync(ctx, connectorstore.SyncTypePartial, "") + require.NoError(t, err) + require.NoError(t, store.PutResourceTypes(ctx, + v2.ResourceType_builder{Id: "user", DisplayName: "User"}.Build(), + v2.ResourceType_builder{Id: "group", DisplayName: "Group"}.Build())) + require.NoError(t, store.PutResources(ctx, group, user)) + require.NoError(t, store.PutEntitlements(ctx, entitlement)) + require.NoError(t, store.PutGrants(ctx, memberGrant(entitlement, user))) + require.NoError(t, store.EndSync(ctx)) + require.NoError(t, store.Close(ctx)) + return &CompactableSync{FilePath: path, SyncID: syncID} +} + // TestCompactor_IncrementalNewMemberMatchesFull is the blocker regression at // the compactor level: an increment that adds a new member to an existing // group (no new edge) must still propagate that member downstream, and match @@ -335,7 +472,7 @@ func TestCompactor_IncrementalNewMemberMatchesFull(t *testing.T) { cInc, cleanupInc, err := NewCompactor(ctx, t.TempDir(), incEntries, WithTmpDir(t.TempDir()), WithEngine(c1zstore.EnginePebble), - WithIncrementalExpansion(baseGraphForFixtures(t, ctx)), + WithIncrementalExpansion(), ) require.NoError(t, err) defer func() { _ = cleanupInc() }() @@ -369,7 +506,7 @@ func TestCompactor_IncrementalNewMemberMatchesFull(t *testing.T) { // buildSpecChangeFixtures builds a base with a B->C rule at baseShallow, plus // mandy (direct on B) and bob (indirect on B), pre-expanded per the base spec; // and an increment that overwrites the B->C rule to incShallow (same grant id). -func buildSpecChangeFixtures(t *testing.T, ctx context.Context, dir string, baseShallow, incShallow bool) []*CompactableSync { +func buildSpecChangeFixtures(t testing.TB, ctx context.Context, dir string, baseShallow, incShallow bool) []*CompactableSync { t.Helper() grpB, grpC := grp("grpB"), grp("grpC") mandy, bob := usr("mandy"), usr("bob") @@ -396,6 +533,7 @@ func buildSpecChangeFixtures(t *testing.T, ctx context.Context, dir string, base } require.NoError(t, base.PutGrants(ctx, baseGrants...)) require.NoError(t, base.EndSync(ctx)) + persistFixtureGraph(t, ctx, base, baseSyncID, specChangeBaseGraph(t, ctx, baseShallow)) require.NoError(t, base.Close(ctx)) incPath := filepath.Join(dir, "inc.c1z") @@ -416,13 +554,15 @@ func buildSpecChangeFixtures(t *testing.T, ctx context.Context, dir string, base } } -func specChangeBaseGraph(t *testing.T, ctx context.Context, shallow bool) *expand.EntitlementGraph { +func specChangeBaseGraph(t testing.TB, ctx context.Context, shallow bool) *expand.EntitlementGraph { t.Helper() g := expand.NewEntitlementGraph(ctx) g.AddEntitlementID("ent-b") g.AddEntitlementID("ent-c") require.NoError(t, g.AddEdge(ctx, "ent-b", "ent-c", shallow, nil)) g.MarkEdgeExpanded("ent-b", "ent-c") + g.Loaded = true + g.HasNoCycles = true return g } @@ -436,7 +576,7 @@ func TestCompactor_IncrementalWidenedEdgeReExpands(t *testing.T) { cInc, cleanupInc, err := NewCompactor(ctx, t.TempDir(), incEntries, WithTmpDir(t.TempDir()), WithEngine(c1zstore.EnginePebble), - WithIncrementalExpansion(specChangeBaseGraph(t, ctx, true)), // base edge is shallow + WithIncrementalExpansion(), // base edge is shallow ) require.NoError(t, err) defer func() { _ = cleanupInc() }() @@ -470,7 +610,7 @@ func TestCompactor_IncrementalNarrowedEdgeDeclines(t *testing.T) { cInc, cleanupInc, err := NewCompactor(ctx, t.TempDir(), incEntries, WithTmpDir(t.TempDir()), WithEngine(c1zstore.EnginePebble), - WithIncrementalExpansion(specChangeBaseGraph(t, ctx, false)), // base edge is deep + WithIncrementalExpansion(), // base edge is deep ) require.NoError(t, err) defer func() { _ = cleanupInc() }() @@ -491,21 +631,20 @@ func TestCompactor_IncrementalNarrowedEdgeDeclines(t *testing.T) { require.Equal(t, fullGrants, incGrants, "declined incremental must equal full expansion") } -// TestCompactor_IncrementalDoesNotMutateBaseGraph (U1): running the incremental -// expansion must not mutate the caller-held base graph, so a retry with the same -// graph can't treat never-expanded edges as already present. +// TestCompactor_IncrementalDoesNotMutateBaseGraph (U1): running incremental +// expansion must not mutate the graph persisted in the caller's base artifact. func TestCompactor_IncrementalDoesNotMutateBaseGraph(t *testing.T) { ctx := context.Background() entries := buildIncrementalFixtures(t, ctx, t.TempDir()) // increment adds ent-a -> ent-b - base := baseGraphForFixtures(t, ctx) // holds only ent-b -> ent-c + base := artifactGraph(t, ctx, entries[0].FilePath, entries[0].SyncID) edgesBefore := len(base.Edges) nodesBefore := len(base.Nodes) c, cleanup, err := NewCompactor(ctx, t.TempDir(), entries, WithTmpDir(t.TempDir()), WithEngine(c1zstore.EnginePebble), - WithIncrementalExpansion(base), + WithIncrementalExpansion(), ) require.NoError(t, err) defer func() { _ = cleanup() }() @@ -514,11 +653,10 @@ func TestCompactor_IncrementalDoesNotMutateBaseGraph(t *testing.T) { require.NoError(t, err) require.True(t, c.incrementalExpansionRan) - // The caller's graph is untouched — the new ent-a -> ent-b edge went into a - // clone, not this graph. - require.Equal(t, edgesBefore, len(base.Edges), "base graph edges must be unchanged") - require.Equal(t, nodesBefore, len(base.Nodes), "base graph nodes must be unchanged") - require.Nil(t, base.GetNode("ent-a"), "new edge's node must not leak into the caller's graph") + after := artifactGraph(t, ctx, entries[0].FilePath, entries[0].SyncID) + require.Equal(t, edgesBefore, len(after.Edges), "base graph edges must be unchanged") + require.Equal(t, nodesBefore, len(after.Nodes), "base graph nodes must be unchanged") + require.Nil(t, after.GetNode("ent-a"), "new edge's node must not leak into the base artifact") } // buildDanglingRefFixtures builds a base (ent-b -> ent-c, mandy) and a single @@ -547,6 +685,7 @@ func buildDanglingRefFixtures(t *testing.T, ctx context.Context, dir string) []* ruleGrant(entC, grpB, "ent-b"), )) require.NoError(t, base.EndSync(ctx)) + persistFixtureGraph(t, ctx, base, baseSyncID, baseGraphForFixtures(t, ctx)) require.NoError(t, base.Close(ctx)) incPath := filepath.Join(dir, "inc.c1z") @@ -570,7 +709,8 @@ func buildDanglingRefFixtures(t *testing.T, ctx context.Context, dir string) []* // TestCompactor_IncrementalDanglingRefMatchesFull (#11a): an increment with a // grant referencing an entitlement absent from the merged set is skipped by -// both paths; incremental (skip-with-warn) must equal full (NotFound skip). +// both paths. This fixture also replaces the base rule, dropping ent-b -> +// ent-c, so the incremental path must safely decline and equal full expansion. func TestCompactor_IncrementalDanglingRefMatchesFull(t *testing.T) { ctx := context.Background() @@ -578,13 +718,13 @@ func TestCompactor_IncrementalDanglingRefMatchesFull(t *testing.T) { cInc, cleanupInc, err := NewCompactor(ctx, t.TempDir(), incEntries, WithTmpDir(t.TempDir()), WithEngine(c1zstore.EnginePebble), - WithIncrementalExpansion(baseGraphForFixtures(t, ctx)), + WithIncrementalExpansion(), ) require.NoError(t, err) defer func() { _ = cleanupInc() }() incOut, err := cInc.Compact(ctx) require.NoError(t, err) - require.True(t, cInc.incrementalExpansionRan, "dangling ref must skip, not fall back") + require.False(t, cInc.incrementalExpansionRan, "the dropped base edge must decline to full expansion") fullEntries := buildDanglingRefFixtures(t, ctx, t.TempDir()) cFull, cleanupFull, err := NewCompactor(ctx, t.TempDir(), fullEntries, @@ -610,7 +750,7 @@ func TestCompactor_IncrementalSealedArtifactLifecycle(t *testing.T) { c, cleanup, err := NewCompactor(ctx, t.TempDir(), entries, WithTmpDir(t.TempDir()), WithEngine(c1zstore.EnginePebble), - WithIncrementalExpansion(baseGraphForFixtures(t, ctx)), + WithIncrementalExpansion(), ) require.NoError(t, err) defer func() { _ = cleanup() }() @@ -626,6 +766,10 @@ func TestCompactor_IncrementalSealedArtifactLifecycle(t *testing.T) { fin, err := store.GetLatestFinishedSync(ctx, reader_v2.SyncsReaderServiceGetLatestFinishedSyncRequest_builder{}.Build()) require.NoError(t, err) require.Equal(t, out.SyncID, fin.GetSync().GetId(), "compacted sync must be sealed/finished") + run, err := store.SyncMeta().LatestFinishedSyncOfAnyType(ctx) + require.NoError(t, err) + require.Equal(t, sdksync.IngestInvariantGeneration, run.Generation) + require.Equal(t, c1zstore.IngestInvariantVerificationModeCompactionMerge, run.Mode) // (2) by_principal index is populated and covers sam (written incrementally). eng, ok := enginepkg.AsEngine(store) @@ -648,6 +792,26 @@ func TestCompactor_IncrementalSealedArtifactLifecycle(t *testing.T) { require.True(t, sawSam, "by_principal index must cover sam's incrementally-written grants") } +func TestCompactor_IncrementalFailFastInvariantMarker(t *testing.T) { + ctx := context.Background() + entries := buildIncrementalFixtures(t, ctx, t.TempDir()) + c, cleanup, err := NewCompactor(ctx, t.TempDir(), entries, + WithTmpDir(t.TempDir()), WithEngine(c1zstore.EnginePebble), + WithIncrementalExpansion(), WithFailFastInvariants()) + require.NoError(t, err) + defer func() { _ = cleanup() }() + out, err := c.Compact(ctx) + require.NoError(t, err) + require.True(t, c.incrementalExpansionRan) + + store, err := dotc1z.NewStore(ctx, out.FilePath, dotc1z.WithReadOnly(true), dotc1z.WithTmpDir(t.TempDir())) + require.NoError(t, err) + defer store.Close(ctx) + run, err := store.SyncMeta().LatestFinishedSyncOfAnyType(ctx) + require.NoError(t, err) + require.Equal(t, c1zstore.IngestInvariantVerificationModeCompactionMergeFailFast, run.Mode) +} + // TestCompactor_IncrementalDegradesGracefullyOnSQLite: the fast path is // Pebble-only (it reopens an ended sync, which SQLite refuses). On a SQLite // output, requesting incremental must degrade to full expansion — no error, @@ -659,7 +823,7 @@ func TestCompactor_IncrementalDegradesGracefullyOnSQLite(t *testing.T) { // No WithEngine → engine inferred from the SQLite inputs. c, cleanup, err := NewCompactor(ctx, t.TempDir(), entries, WithTmpDir(t.TempDir()), - WithIncrementalExpansion(baseGraphForFixtures(t, ctx)), + WithIncrementalExpansion(), ) require.NoError(t, err) defer func() { _ = cleanup() }() @@ -687,7 +851,7 @@ func TestCompactor_IncrementalNewMemberFoldCollectsChangedEnts(t *testing.T) { WithTmpDir(t.TempDir()), WithEngine(c1zstore.EnginePebble), WithPebbleCompactorMode(PebbleCompactorModeFold), - WithIncrementalExpansion(baseGraphForFixtures(t, ctx)), + WithIncrementalExpansion(), ) require.NoError(t, err) defer func() { _ = cleanupInc() }() @@ -727,7 +891,7 @@ func TestCompactor_IncrementalNewMemberRebuildFallsBackToDerive(t *testing.T) { WithTmpDir(t.TempDir()), WithEngine(c1zstore.EnginePebble), WithPebbleCompactorMode(PebbleCompactorModeOverlay), - WithIncrementalExpansion(baseGraphForFixtures(t, ctx)), + WithIncrementalExpansion(), ) require.NoError(t, err) defer func() { _ = cleanupInc() }() @@ -777,7 +941,7 @@ func TestCompactor_ArtifactCarriesGraphSidecar(t *testing.T) { cInc, cleanupInc, err := NewCompactor(ctx, t.TempDir(), entries, WithTmpDir(t.TempDir()), WithEngine(c1zstore.EnginePebble), - WithIncrementalExpansion(baseGraphForFixtures(t, ctx)), // ent-b -> ent-c + WithIncrementalExpansion(), // ent-b -> ent-c ) require.NoError(t, err) defer func() { _ = cleanupInc() }() @@ -789,6 +953,8 @@ func TestCompactor_ArtifactCarriesGraphSidecar(t *testing.T) { require.NotNil(t, g, "incremental artifact must carry its graph sidecar") require.NotNil(t, g.GetNode("ent-a"), "sidecar graph must include the increment's new edge source") require.Len(t, g.Edges, 2, "base edge + folded-in new edge") + require.True(t, g.IsExpanded(), "persisted graph must describe completed expansion") + require.True(t, g.HasNoCycles, "persisted graph must record the successful cycle check") // (b) Decline -> full (narrowed edge) with incremental requested: the full // path preserves a fresh graph so the chain heals after the fallback. @@ -796,7 +962,7 @@ func TestCompactor_ArtifactCarriesGraphSidecar(t *testing.T) { cDecl, cleanupDecl, err := NewCompactor(ctx, t.TempDir(), declEntries, WithTmpDir(t.TempDir()), WithEngine(c1zstore.EnginePebble), - WithIncrementalExpansion(specChangeBaseGraph(t, ctx, false)), + WithIncrementalExpansion(), ) require.NoError(t, err) defer func() { _ = cleanupDecl() }() @@ -822,3 +988,169 @@ func TestCompactor_ArtifactCarriesGraphSidecar(t *testing.T) { g = artifactGraph(t, ctx, plainOut.FilePath, plainOut.SyncID) require.Nil(t, g, "artifact without incremental opt-in must carry no graph sidecar") } + +func TestCompactor_IncrementalDroppedEdgeDeclinesToFull(t *testing.T) { + ctx := context.Background() + incEntries := buildDroppedEdgeFixtures(t, ctx, t.TempDir()) + cInc, cleanupInc, err := NewCompactor(ctx, t.TempDir(), incEntries, + WithTmpDir(t.TempDir()), WithEngine(c1zstore.EnginePebble), + WithIncrementalExpansion()) + require.NoError(t, err) + defer func() { _ = cleanupInc() }() + incOut, err := cInc.Compact(ctx) + require.NoError(t, err) + require.False(t, cInc.incrementalExpansionRan, "a dropped edge must decline to full expansion") + + fullEntries := buildDroppedEdgeFixtures(t, ctx, t.TempDir()) + cFull, cleanupFull, err := NewCompactor(ctx, t.TempDir(), fullEntries, + WithTmpDir(t.TempDir()), WithEngine(c1zstore.EnginePebble)) + require.NoError(t, err) + defer func() { _ = cleanupFull() }() + fullOut, err := cFull.Compact(ctx) + require.NoError(t, err) + require.Equal(t, + grantOutcome(t, ctx, fullOut.FilePath, fullOut.SyncID), + grantOutcome(t, ctx, incOut.FilePath, incOut.SyncID), + "dropped-edge fallback must equal full expansion") +} + +func TestCompactor_IncrementalRejectsWrongBaseSyncAndInvalidGraph(t *testing.T) { + ctx := context.Background() + for _, tc := range []struct { + name string + mutate func(*testing.T, context.Context, []*CompactableSync) + }{ + { + name: "wrong base sync", + mutate: func(t *testing.T, ctx context.Context, entries []*CompactableSync) { + overwriteFixtureGraph(t, ctx, entries[0].FilePath, "another-sync", baseGraphForFixtures(t, ctx)) + }, + }, + { + name: "inconsistent adjacency", + mutate: func(t *testing.T, ctx context.Context, entries []*CompactableSync) { + g := baseGraphForFixtures(t, ctx) + src, dst := g.GetNode("ent-b"), g.GetNode("ent-c") + delete(g.SourcesToDestinations[src.Id], dst.Id) + overwriteFixtureGraph(t, ctx, entries[0].FilePath, entries[0].SyncID, g) + }, + }, + { + name: "malformed sidecar", + mutate: func(t *testing.T, ctx context.Context, entries []*CompactableSync) { + overwriteFixtureGraphRaw(t, ctx, entries[0].FilePath, []byte("{")) + }, + }, + } { + t.Run(tc.name, func(t *testing.T) { + entries := buildIncrementalFixtures(t, ctx, t.TempDir()) + tc.mutate(t, ctx, entries) + c, cleanup, err := NewCompactor(ctx, t.TempDir(), entries, + WithTmpDir(t.TempDir()), WithEngine(c1zstore.EnginePebble), + WithIncrementalExpansion()) + require.NoError(t, err) + defer func() { _ = cleanup() }() + out, err := c.Compact(ctx) + require.NoError(t, err) + require.False(t, c.incrementalExpansionRan) + grants := grantOutcome(t, ctx, out.FilePath, out.SyncID) + hasGrant(t, grants, "ent-c|user|sam") + }) + } +} + +func TestCompactor_AbsentPartialMembershipIsNotADeletion(t *testing.T) { + ctx := context.Background() + baseFixtures := buildIncrementalFixtures(t, ctx, t.TempDir()) + base := baseFixtures[0] + + incEmpty := buildEmptyPartial(t, ctx, filepath.Join(t.TempDir(), "inc-empty.c1z")) + inc, cleanupInc, err := NewCompactor(ctx, t.TempDir(), []*CompactableSync{base, incEmpty}, + WithTmpDir(t.TempDir()), WithEngine(c1zstore.EnginePebble), WithIncrementalExpansion()) + require.NoError(t, err) + defer func() { _ = cleanupInc() }() + incOut, err := inc.Compact(ctx) + require.NoError(t, err) + require.True(t, inc.incrementalExpansionRan) + + fullEmpty := buildEmptyPartial(t, ctx, filepath.Join(t.TempDir(), "full-empty.c1z")) + full, cleanupFull, err := NewCompactor(ctx, t.TempDir(), []*CompactableSync{base, fullEmpty}, + WithTmpDir(t.TempDir()), WithEngine(c1zstore.EnginePebble)) + require.NoError(t, err) + defer func() { _ = cleanupFull() }() + fullOut, err := full.Compact(ctx) + require.NoError(t, err) + + require.Equal(t, + grantOutcome(t, ctx, fullOut.FilePath, fullOut.SyncID), + grantOutcome(t, ctx, incOut.FilePath, incOut.SyncID)) + grants := grantOutcome(t, ctx, incOut.FilePath, incOut.SyncID) + hasGrant(t, grants, "ent-c|user|mandy") +} + +func TestCompactor_IncrementalExistingCollapsedCycleFallsBack(t *testing.T) { + ctx := context.Background() + entries := buildIncrementalFixtures(t, ctx, t.TempDir()) + g := expand.NewEntitlementGraph(ctx) + g.AddEntitlementID("cycle-a") + g.AddEntitlementID("cycle-b") + require.NoError(t, g.AddEdge(ctx, "cycle-a", "cycle-b", false, nil)) + require.NoError(t, g.AddEdge(ctx, "cycle-b", "cycle-a", false, nil)) + require.NoError(t, g.FixCycles(ctx)) + g.Loaded = true + g.MarkExpansionComplete() + require.True(t, g.HasCollapsedCycles()) + overwriteFixtureGraph(t, ctx, entries[0].FilePath, entries[0].SyncID, g) + + c, cleanup, err := NewCompactor(ctx, t.TempDir(), entries, + WithTmpDir(t.TempDir()), WithEngine(c1zstore.EnginePebble), + WithIncrementalExpansion()) + require.NoError(t, err) + defer func() { _ = cleanup() }() + out, err := c.Compact(ctx) + require.NoError(t, err) + require.False(t, c.incrementalExpansionRan) + grants := grantOutcome(t, ctx, out.FilePath, out.SyncID) + hasGrant(t, grants, "ent-c|user|sam") +} + +func TestCompactor_IncrementalGraphReusedByNextGeneration(t *testing.T) { + ctx := context.Background() + firstEntries := buildIncrementalFixtures(t, ctx, t.TempDir()) + first, cleanupFirst, err := NewCompactor(ctx, t.TempDir(), firstEntries, + WithTmpDir(t.TempDir()), WithEngine(c1zstore.EnginePebble), + WithIncrementalExpansion()) + require.NoError(t, err) + defer func() { _ = cleanupFirst() }() + firstOut, err := first.Compact(ctx) + require.NoError(t, err) + require.True(t, first.incrementalExpansionRan) + baseGraph := artifactGraph(t, ctx, firstOut.FilePath, firstOut.SyncID) + require.NoError(t, baseGraph.ValidateCompleted()) + + incPartial := buildMemberPartial(t, ctx, filepath.Join(t.TempDir(), "inc-next.c1z"), "ent-b", "grpB", "zoe") + incEntries := []*CompactableSync{{FilePath: firstOut.FilePath, SyncID: firstOut.SyncID}, incPartial} + inc, cleanupInc, err := NewCompactor(ctx, t.TempDir(), incEntries, + WithTmpDir(t.TempDir()), WithEngine(c1zstore.EnginePebble), + WithIncrementalExpansion()) + require.NoError(t, err) + defer func() { _ = cleanupInc() }() + incOut, err := inc.Compact(ctx) + require.NoError(t, err) + require.True(t, inc.incrementalExpansionRan) + + fullPartial := buildMemberPartial(t, ctx, filepath.Join(t.TempDir(), "full-next.c1z"), "ent-b", "grpB", "zoe") + fullEntries := []*CompactableSync{{FilePath: firstOut.FilePath, SyncID: firstOut.SyncID}, fullPartial} + full, cleanupFull, err := NewCompactor(ctx, t.TempDir(), fullEntries, + WithTmpDir(t.TempDir()), WithEngine(c1zstore.EnginePebble)) + require.NoError(t, err) + defer func() { _ = cleanupFull() }() + fullOut, err := full.Compact(ctx) + require.NoError(t, err) + + require.Equal(t, + grantOutcome(t, ctx, fullOut.FilePath, fullOut.SyncID), + grantOutcome(t, ctx, incOut.FilePath, incOut.SyncID)) + nextGraph := artifactGraph(t, ctx, incOut.FilePath, incOut.SyncID) + require.NoError(t, nextGraph.ValidateCompleted()) +} diff --git a/pkg/synccompactor/incremental_hardening_test.go b/pkg/synccompactor/incremental_hardening_test.go new file mode 100644 index 000000000..3435270ff --- /dev/null +++ b/pkg/synccompactor/incremental_hardening_test.go @@ -0,0 +1,297 @@ +package synccompactor + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "os" + "os/exec" + "path/filepath" + "regexp" + "runtime" + "sort" + "strings" + "testing" + + "github.com/grpc-ecosystem/go-grpc-middleware/logging/zap/ctxzap" + "github.com/stretchr/testify/require" + "go.uber.org/zap" + "go.uber.org/zap/zapcore" + + v2 "github.com/conductorone/baton-sdk/pb/c1/connector/v2" + "github.com/conductorone/baton-sdk/pkg/connectorstore" + "github.com/conductorone/baton-sdk/pkg/dotc1z" + "github.com/conductorone/baton-sdk/pkg/dotc1z/c1zstore" +) + +var allIncrementalFaultStages = []string{ + "mid_expand_write", + "after_walk", + "after_sidecar", + "before_end_sync", + "after_end_sync", + "after_marker", + "before_close", + "before_publish", + "after_publish", +} + +func TestIncrementalFaultInventoryIsComplete(t *testing.T) { + _, thisFile, _, ok := runtime.Caller(0) + require.True(t, ok) + source, err := os.ReadFile(filepath.Join(filepath.Dir(thisFile), "compactor.go")) + require.NoError(t, err) + re := regexp.MustCompile(`(?:runIncrementalTestHook|hook)\("([a-z_]+)"`) + matches := re.FindAllSubmatch(source, -1) + actualSet := make(map[string]struct{}, len(matches)) + for _, match := range matches { + actualSet[string(match[1])] = struct{}{} + } + actual := make([]string, 0, len(actualSet)) + for stage := range actualSet { + actual = append(actual, stage) + } + sort.Strings(actual) + want := append([]string(nil), allIncrementalFaultStages...) + sort.Strings(want) + require.Equal(t, want, actual, + "every production fault cut must be present in the exhaustive crash/retry sweep") +} + +func TestIncrementalExpansionOutcomeLogging(t *testing.T) { + tests := []struct { + name string + build func(*testing.T, context.Context, string) []*CompactableSync + options []Option + wantOutcome string + wantReason string + }{ + { + name: "success", build: func(t *testing.T, ctx context.Context, dir string) []*CompactableSync { + return buildIncrementalFixtures(t, ctx, dir) + }, + options: []Option{WithEngine(c1zstore.EnginePebble), WithIncrementalExpansion()}, + wantOutcome: "succeeded", wantReason: "none", + }, + { + name: "revocation decline", + build: func(t *testing.T, ctx context.Context, dir string) []*CompactableSync { + return buildSpecChangeFixtures(t, ctx, dir, false, true) + }, + options: []Option{WithEngine(c1zstore.EnginePebble), WithIncrementalExpansion()}, + wantOutcome: "declined", wantReason: "revocation", + }, + { + name: "dropped edge decline", build: buildDroppedEdgeFixtures, + options: []Option{WithEngine(c1zstore.EnginePebble), WithIncrementalExpansion()}, + wantOutcome: "declined", wantReason: "dropped_edge", + }, + { + name: "cycle decline", build: buildCycleLoggingFixtures, + options: []Option{WithEngine(c1zstore.EnginePebble), WithIncrementalExpansion()}, + wantOutcome: "declined", wantReason: "cycle", + }, + { + name: "unsupported engine", build: func(t *testing.T, ctx context.Context, dir string) []*CompactableSync { + return buildIncrementalFixturesEngine(t, ctx, dir, c1zstore.EngineSQLite) + }, + options: []Option{WithIncrementalExpansion()}, + wantOutcome: "not_attempted", wantReason: "unsupported_engine", + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + var logs bytes.Buffer + core := zapcore.NewCore(zapcore.NewJSONEncoder(zap.NewProductionEncoderConfig()), + zapcore.AddSync(&logs), zap.InfoLevel) + ctx := ctxzap.ToContext(context.Background(), zap.New(core)) + entries := tc.build(t, ctx, t.TempDir()) + options := append([]Option{WithTmpDir(t.TempDir())}, tc.options...) + compactor, cleanup, err := NewCompactor(ctx, t.TempDir(), entries, options...) + require.NoError(t, err) + defer func() { require.NoError(t, cleanup()) }() + _, err = compactor.Compact(ctx) + require.NoError(t, err) + + found := false + for _, line := range strings.Split(strings.TrimSpace(logs.String()), "\n") { + var fields map[string]any + require.NoError(t, json.Unmarshal([]byte(line), &fields)) + if fields["msg"] == "incremental grant expansion outcome" && + fields["incremental_expansion_outcome"] == tc.wantOutcome && + fields["incremental_expansion_reason"] == tc.wantReason { + found = true + } + } + require.True(t, found, "missing stable outcome=%s reason=%s in logs:\n%s", + tc.wantOutcome, tc.wantReason, logs.String()) + }) + } +} + +func buildCycleLoggingFixtures(t *testing.T, ctx context.Context, dir string) []*CompactableSync { + t.Helper() + entries := buildIncrementalFixtures(t, ctx, dir) + groupB, groupC := grp("grpB"), grp("grpC") + entB := ent("ent-b", groupB) + path := filepath.Join(dir, "cycle.c1z") + store, err := dotc1z.NewStore(ctx, path, dotc1z.WithEngine(c1zstore.EnginePebble)) + require.NoError(t, err) + syncID, err := store.StartNewSync(ctx, connectorstore.SyncTypePartial, "") + require.NoError(t, err) + require.NoError(t, store.PutResourceTypes(ctx, + v2.ResourceType_builder{Id: "group", DisplayName: "Group"}.Build())) + require.NoError(t, store.PutResources(ctx, groupB, groupC)) + require.NoError(t, store.PutEntitlements(ctx, entB)) + require.NoError(t, store.PutGrants(ctx, ruleGrant(entB, groupC, "ent-c"))) + require.NoError(t, store.EndSync(ctx)) + require.NoError(t, store.Close(ctx)) + return append(entries, &CompactableSync{FilePath: path, SyncID: syncID}) +} + +func TestIncrementalExpansionProcessKillRetry(t *testing.T) { + for _, stage := range []string{"after_sidecar", "before_end_sync"} { + t.Run(stage, func(t *testing.T) { + ctx := context.Background() + entries := buildIncrementalFixtures(t, ctx, t.TempDir()) + failedDest := t.TempDir() + cmd := exec.CommandContext(ctx, os.Args[0], "-test.run=^TestIncrementalExpansionProcessKillHelper$") //nolint:gosec // os.Args[0] is the current test binary. + cmd.Env = append(os.Environ(), + "BATON_INCREMENTAL_KILL_HELPER=1", + "BATON_INCREMENTAL_KILL_STAGE="+stage, + "BATON_INCREMENTAL_BASE_PATH="+entries[0].FilePath, + "BATON_INCREMENTAL_BASE_SYNC="+entries[0].SyncID, + "BATON_INCREMENTAL_INC_PATH="+entries[1].FilePath, + "BATON_INCREMENTAL_INC_SYNC="+entries[1].SyncID, + "BATON_INCREMENTAL_DEST="+failedDest, + "BATON_INCREMENTAL_TMP="+t.TempDir(), + ) + err := cmd.Run() + require.Error(t, err, "helper must be killed at %s", stage) + published, readErr := os.ReadDir(failedDest) + require.NoError(t, readErr) + require.Empty(t, published, "killed attempt must publish nothing") + + var outcomes [][]string + for retryNumber := 0; retryNumber < 2; retryNumber++ { + retry, cleanup, newErr := NewCompactor(ctx, t.TempDir(), entries, + WithTmpDir(t.TempDir()), WithEngine(c1zstore.EnginePebble), WithIncrementalExpansion()) + require.NoError(t, newErr) + out, compactErr := retry.Compact(ctx) + require.NoError(t, compactErr) + require.True(t, retry.incrementalExpansionRan) + outcomes = append(outcomes, grantOutcome(t, ctx, out.FilePath, out.SyncID)) + assertSealedCompactionArtifact(t, ctx, out, true) + require.NoError(t, cleanup()) + } + require.Equal(t, outcomes[0], outcomes[1], "two fresh retries must converge identically") + }) + } +} + +func TestIncrementalExpansionProcessKillHelper(t *testing.T) { + if os.Getenv("BATON_INCREMENTAL_KILL_HELPER") != "1" { + t.Skip("subprocess helper") + } + ctx := context.Background() + entries := []*CompactableSync{ + {FilePath: os.Getenv("BATON_INCREMENTAL_BASE_PATH"), SyncID: os.Getenv("BATON_INCREMENTAL_BASE_SYNC")}, + {FilePath: os.Getenv("BATON_INCREMENTAL_INC_PATH"), SyncID: os.Getenv("BATON_INCREMENTAL_INC_SYNC")}, + } + compactor, _, err := NewCompactor(ctx, os.Getenv("BATON_INCREMENTAL_DEST"), entries, + WithTmpDir(os.Getenv("BATON_INCREMENTAL_TMP")), + WithEngine(c1zstore.EnginePebble), + WithIncrementalExpansion()) + require.NoError(t, err) + compactor.incrementalTestHook = func(stage string) error { + if stage == os.Getenv("BATON_INCREMENTAL_KILL_STAGE") { + process, findErr := os.FindProcess(os.Getpid()) + if findErr == nil { + _ = process.Kill() + } + os.Exit(91) + } + return nil + } + _, _ = compactor.Compact(ctx) + os.Exit(92) +} + +// TestIncrementalExpansionCrashRetry injects failures at the feature's three +// commit cuts. A failed attempt must publish nothing; a fresh retry from the +// same immutable inputs must converge to the full-expansion oracle. +func TestIncrementalExpansionCrashRetry(t *testing.T) { + for _, stage := range allIncrementalFaultStages { + stage := stage + t.Run(stage, func(t *testing.T) { + ctx := context.Background() + entries := buildIncrementalFixtures(t, ctx, t.TempDir()) + failedDest := t.TempDir() + failed, failedCleanup, err := NewCompactor(ctx, failedDest, entries, + WithTmpDir(t.TempDir()), + WithEngine(c1zstore.EnginePebble), + WithIncrementalExpansion(), + ) + require.NoError(t, err) + failed.incrementalTestHook = func(at string) error { + if at == "after_end_sync" { + run, runErr := failed.compactedC1z.SyncMeta().LatestFinishedSyncOfAnyType(ctx) + require.NoError(t, runErr) + require.Empty(t, run.Generation, "verification must be absent immediately after seal") + } + if at == "after_marker" { + run, runErr := failed.compactedC1z.SyncMeta().LatestFinishedSyncOfAnyType(ctx) + require.NoError(t, runErr) + require.True(t, run.IsVerified(), "marker must be present only after seal") + } + if at == stage { + return errors.New("injected crash") + } + return nil + } + _, err = failed.Compact(ctx) + require.Error(t, err) + require.NoError(t, failedCleanup()) + published, err := os.ReadDir(failedDest) + require.NoError(t, err) + if stage == "after_publish" { + require.Len(t, published, 1, "post-publish failure must leave one complete artifact") + path := filepath.Join(failedDest, published[0].Name()) + store, openErr := dotc1z.NewStore(ctx, path, dotc1z.WithReadOnly(true), dotc1z.WithTmpDir(t.TempDir())) + require.NoError(t, openErr) + run, runErr := store.SyncMeta().LatestFinishedSyncOfAnyType(ctx) + require.NoError(t, runErr) + require.NoError(t, store.Close(ctx)) + assertSealedCompactionArtifact(t, ctx, &CompactableSync{FilePath: path, SyncID: run.ID}, true) + } else { + require.Empty(t, published, "failed attempt must not publish an artifact") + } + + retry, retryCleanup, err := NewCompactor(ctx, t.TempDir(), entries, + WithTmpDir(t.TempDir()), + WithEngine(c1zstore.EnginePebble), + WithIncrementalExpansion(), + ) + require.NoError(t, err) + defer func() { require.NoError(t, retryCleanup()) }() + retryOut, err := retry.Compact(ctx) + require.NoError(t, err) + require.True(t, retry.incrementalExpansionRan) + + fullEntries := buildIncrementalFixtures(t, ctx, t.TempDir()) + full, fullCleanup, err := NewCompactor(ctx, t.TempDir(), fullEntries, + WithTmpDir(t.TempDir()), WithEngine(c1zstore.EnginePebble)) + require.NoError(t, err) + defer func() { require.NoError(t, fullCleanup()) }() + fullOut, err := full.Compact(ctx) + require.NoError(t, err) + + require.Equal(t, + grantOutcome(t, ctx, fullOut.FilePath, fullOut.SyncID), + grantOutcome(t, ctx, retryOut.FilePath, retryOut.SyncID)) + assertSealedCompactionArtifact(t, ctx, retryOut, true) + }) + } +} From 6ea68c07b28dee9ab02c5cc6258303a8c5e09631 Mon Sep 17 00:00:00 2001 From: manojacs Date: Mon, 10 Aug 2026 22:00:10 +0000 Subject: [PATCH 08/15] Harden incremental graph reuse Co-authored-by: c1-squire-dev[bot] --- .../pebble/entitlement_graph_sidecar.go | 22 +++++++++ .../pebble/entitlement_graph_sidecar_test.go | 47 +++++++++++++++++++ pkg/sync/expand/graph.go | 14 ++++++ pkg/sync/expand/graph_blob.go | 5 +- pkg/sync/expand/graph_blob_test.go | 38 ++++++++++++++- pkg/synccompactor/compactor.go | 5 ++ pkg/synccompactor/compactor_pebble.go | 14 +++--- .../incremental_hardening_test.go | 15 ++++++ pkg/synccompactor/pebble/merge.go | 20 ++++---- pkg/synccompactor/pebble/merge_test.go | 35 ++++++++++---- .../pebble/winner_equivalence_test.go | 2 +- 11 files changed, 188 insertions(+), 29 deletions(-) create mode 100644 pkg/dotc1z/engine/pebble/entitlement_graph_sidecar_test.go diff --git a/pkg/dotc1z/engine/pebble/entitlement_graph_sidecar.go b/pkg/dotc1z/engine/pebble/entitlement_graph_sidecar.go index a5c8f0f1d..75ad75b70 100644 --- a/pkg/dotc1z/engine/pebble/entitlement_graph_sidecar.go +++ b/pkg/dotc1z/engine/pebble/entitlement_graph_sidecar.go @@ -37,7 +37,15 @@ func EntitlementGraphSidecarUpperBound() []byte { // PutEntitlementGraphSidecar stores the opaque graph blob. Same write // barrier as the stats sidecar: callers span EndSync's sealed window. func (e *Engine) PutEntitlementGraphSidecar(ctx context.Context, data []byte) error { + if err := ctx.Err(); err != nil { + return err + } return e.withWriteAllowSealed(func() error { + // Re-check after waiting for the engine's write lock. The context may + // have been canceled while another writer held the lock. + if err := ctx.Err(); err != nil { + return err + } return e.db.MetaSet(encodeEntitlementGraphKey(), data, pebble.Sync) }) } @@ -45,6 +53,9 @@ func (e *Engine) PutEntitlementGraphSidecar(ctx context.Context, data []byte) er // GetEntitlementGraphSidecar returns the stored blob, or (nil, nil) if // none exists. func (e *Engine) GetEntitlementGraphSidecar(ctx context.Context) ([]byte, error) { + if err := ctx.Err(); err != nil { + return nil, err + } val, closer, err := e.db.Get(encodeEntitlementGraphKey()) if err != nil { if errors.Is(err, pebble.ErrNotFound) { @@ -55,12 +66,23 @@ func (e *Engine) GetEntitlementGraphSidecar(ctx context.Context) ([]byte, error) defer closer.Close() out := make([]byte, len(val)) copy(out, val) + if err := ctx.Err(); err != nil { + return nil, err + } return out, nil } // DeleteEntitlementGraphSidecar removes the blob (no-op when absent). func (e *Engine) DeleteEntitlementGraphSidecar(ctx context.Context) error { + if err := ctx.Err(); err != nil { + return err + } return e.withWriteAllowSealed(func() error { + // Re-check after waiting for the engine's write lock. The context may + // have been canceled while another writer held the lock. + if err := ctx.Err(); err != nil { + return err + } return e.db.MetaDelete(encodeEntitlementGraphKey(), pebble.Sync) }) } diff --git a/pkg/dotc1z/engine/pebble/entitlement_graph_sidecar_test.go b/pkg/dotc1z/engine/pebble/entitlement_graph_sidecar_test.go new file mode 100644 index 000000000..cd78019ff --- /dev/null +++ b/pkg/dotc1z/engine/pebble/entitlement_graph_sidecar_test.go @@ -0,0 +1,47 @@ +package pebble + +import ( + "context" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestEntitlementGraphSidecarHonorsContext(t *testing.T) { + ctx := context.Background() + e, _ := newTestEngine(t) + + require.NoError(t, e.PutEntitlementGraphSidecar(ctx, []byte("original"))) + + canceledCtx, cancel := context.WithCancel(ctx) + cancel() + + t.Run("put", func(t *testing.T) { + err := e.PutEntitlementGraphSidecar(canceledCtx, []byte("replacement")) + require.ErrorIs(t, err, context.Canceled) + + got, err := e.GetEntitlementGraphSidecar(ctx) + require.NoError(t, err) + require.Equal(t, []byte("original"), got) + }) + + t.Run("get", func(t *testing.T) { + got, err := e.GetEntitlementGraphSidecar(canceledCtx) + require.ErrorIs(t, err, context.Canceled) + require.Nil(t, got) + }) + + t.Run("delete", func(t *testing.T) { + err := e.DeleteEntitlementGraphSidecar(canceledCtx) + require.ErrorIs(t, err, context.Canceled) + + got, err := e.GetEntitlementGraphSidecar(ctx) + require.NoError(t, err) + require.Equal(t, []byte("original"), got) + }) + + require.NoError(t, e.DeleteEntitlementGraphSidecar(ctx)) + got, err := e.GetEntitlementGraphSidecar(ctx) + require.NoError(t, err) + require.Nil(t, got) +} diff --git a/pkg/sync/expand/graph.go b/pkg/sync/expand/graph.go index c34a4c402..cf741dcdf 100644 --- a/pkg/sync/expand/graph.go +++ b/pkg/sync/expand/graph.go @@ -161,7 +161,11 @@ func (g *EntitlementGraph) ValidateCompleted() error { if !g.IsExpanded() { return fmt.Errorf("graph has unexpanded edges") } + maxNodeID := 0 for nodeID, node := range g.Nodes { + if nodeID > maxNodeID { + maxNodeID = nodeID + } if node.Id != nodeID { return fmt.Errorf("node map key %d does not match node id %d", nodeID, node.Id) } @@ -171,13 +175,20 @@ func (g *EntitlementGraph) ValidateCompleted() error { } } } + if g.NextNodeID < maxNodeID { + return fmt.Errorf("next node id %d is below existing maximum %d", g.NextNodeID, maxNodeID) + } for entitlementID, nodeID := range g.EntitlementsToNodes { node, ok := g.Nodes[nodeID] if !ok || !slices.Contains(node.EntitlementIDs, entitlementID) { return fmt.Errorf("entitlement map entry %q points to inconsistent node %d", entitlementID, nodeID) } } + maxEdgeID := 0 for edgeID, edge := range g.Edges { + if edgeID > maxEdgeID { + maxEdgeID = edgeID + } if edge.EdgeID != edgeID { return fmt.Errorf("edge map key %d does not match edge id %d", edgeID, edge.EdgeID) } @@ -194,6 +205,9 @@ func (g *EntitlementGraph) ValidateCompleted() error { return fmt.Errorf("edge %d missing from destination adjacency", edgeID) } } + if g.NextEdgeID < maxEdgeID { + return fmt.Errorf("next edge id %d is below existing maximum %d", g.NextEdgeID, maxEdgeID) + } for sourceID, destinations := range g.SourcesToDestinations { for destinationID, edgeID := range destinations { edge, ok := g.Edges[edgeID] diff --git a/pkg/sync/expand/graph_blob.go b/pkg/sync/expand/graph_blob.go index 910029ee5..6241f8eab 100644 --- a/pkg/sync/expand/graph_blob.go +++ b/pkg/sync/expand/graph_blob.go @@ -41,8 +41,9 @@ func marshalGraphBlob(syncID string, g *EntitlementGraph, digest *c1zstore.Grant if g == nil { return nil, fmt.Errorf("marshal graph blob: nil graph") } - g.ClearTransientState() - data, err := json.Marshal(graphBlobEnvelope{FormatVersion: graphBlobFormatVersion, SyncID: syncID, GrantDigest: digest, Graph: g}) + graphCopy := *g + graphCopy.ClearTransientState() + data, err := json.Marshal(graphBlobEnvelope{FormatVersion: graphBlobFormatVersion, SyncID: syncID, GrantDigest: digest, Graph: &graphCopy}) if err != nil { return nil, fmt.Errorf("marshal graph blob: %w", err) } diff --git a/pkg/sync/expand/graph_blob_test.go b/pkg/sync/expand/graph_blob_test.go index ae82c346b..3a9ecc942 100644 --- a/pkg/sync/expand/graph_blob_test.go +++ b/pkg/sync/expand/graph_blob_test.go @@ -41,6 +41,32 @@ func TestValidateCompletedRejectsInconsistentAdjacency(t *testing.T) { require.ErrorContains(t, g.ValidateCompleted(), "missing from source adjacency") } +func TestValidateCompletedRejectsStaleIDCounters(t *testing.T) { + ctx := context.Background() + build := func(t *testing.T) *EntitlementGraph { + t.Helper() + g := NewEntitlementGraph(ctx) + g.AddEntitlementID("ent-a") + g.AddEntitlementID("ent-b") + require.NoError(t, g.AddEdge(ctx, "ent-a", "ent-b", false, nil)) + g.Loaded = true + g.MarkExpansionComplete() + return g + } + + t.Run("node counter", func(t *testing.T) { + g := build(t) + g.NextNodeID = 0 + require.ErrorContains(t, g.ValidateCompleted(), "next node id") + }) + + t.Run("edge counter", func(t *testing.T) { + g := build(t) + g.NextEdgeID = 0 + require.ErrorContains(t, g.ValidateCompleted(), "next edge id") + }) +} + // TestGraphBlobRoundTrip: marshal/unmarshal preserves the graph; the sync-id // guard rejects a blob from a different sync. func TestGraphBlobRoundTrip(t *testing.T) { @@ -78,13 +104,23 @@ func TestMarshalGraphBlob_StripsTransientState(t *testing.T) { ctx := context.Background() g := NewEntitlementGraph(ctx) g.AddEntitlementID("ent-a") - g.Actions = []*EntitlementGraphAction{{}} + actions := []*EntitlementGraphAction{{}} + plan := &EntitlementGraphPlan{} + metrics := &EntitlementGraphMetrics{} + g.Actions = actions + g.ExpansionPlan = plan + g.ExpansionMetrics = metrics data, err := MarshalGraphBlob("s", g) require.NoError(t, err) + require.Equal(t, actions, g.Actions, "marshal must not change the caller's actions") + require.Same(t, plan, g.ExpansionPlan, "marshal must not change the caller's plan") + require.Same(t, metrics, g.ExpansionMetrics, "marshal must not change the caller's metrics") got, err := UnmarshalGraphBlob(data, "s") require.NoError(t, err) require.Nil(t, got.Actions, "transient state must be stripped from the blob") + require.Nil(t, got.ExpansionPlan, "transient plan must be stripped from the blob") + require.Nil(t, got.ExpansionMetrics, "transient metrics must be stripped from the blob") } // TestGraphBlobSizeAtScale measures the sidecar blob for a nested-groups graph diff --git a/pkg/synccompactor/compactor.go b/pkg/synccompactor/compactor.go index 0c3940638..7ec7ad40c 100644 --- a/pkg/synccompactor/compactor.go +++ b/pkg/synccompactor/compactor.go @@ -1287,5 +1287,10 @@ func (c *Compactor) loadIncrementalBaseGraph(ctx context.Context) (*expand.Entit if closeErr != nil { return nil, fmt.Errorf("incremental expansion: close base graph store: %w", closeErr) } + if graph != nil { + if err := graph.ValidateCompleted(); err != nil { + return nil, fmt.Errorf("incremental expansion: invalid base graph: %w", err) + } + } return graph, nil } diff --git a/pkg/synccompactor/compactor_pebble.go b/pkg/synccompactor/compactor_pebble.go index 9152f284e..8c8fbdbf1 100644 --- a/pkg/synccompactor/compactor_pebble.go +++ b/pkg/synccompactor/compactor_pebble.go @@ -549,7 +549,7 @@ func (c *Compactor) compactPebbleFold(ctx context.Context) (string, error) { partialSyncIDs = append(partialSyncIDs, srcSyncID) partialTokens = append(partialTokens, readSourceSyncToken(ctx, srcEng, srcSyncID)) - mergeStats, mergeErr := mergepkg.MergeInto(ctx, destEng, []mergepkg.SourceSync{{Engine: srcEng, SyncID: srcSyncID}}, baseSyncID) + mergeStats, mergeErr := mergepkg.MergeInto(ctx, destEng, []mergepkg.SourceSync{{Engine: srcEng, SyncID: srcSyncID}}, baseSyncID, c.incrementalExpansion) foldStats.Add(mergeStats) if cerr := w.Close(ctx); cerr != nil { l.Error("compactPebbleFold: error closing source store", zap.Error(cerr), zap.String("file", sourcePath)) @@ -559,11 +559,13 @@ func (c *Compactor) compactPebbleFold(ctx context.Context) (string, error) { } } - // Hand the fold's changed-entitlement set to incremental expansion. - // Non-nil even when empty: nil means "no fold ran" (derive fallback). - c.foldChangedEntitlementIDs = foldStats.GrantEntitlementIDs - if c.foldChangedEntitlementIDs == nil { - c.foldChangedEntitlementIDs = map[string]struct{}{} + if c.incrementalExpansion { + // Hand the fold's changed-entitlement set to incremental expansion. + // Non-nil even when empty: nil means "no fold ran" (derive fallback). + c.foldChangedEntitlementIDs = foldStats.GrantEntitlementIDs + if c.foldChangedEntitlementIDs == nil { + c.foldChangedEntitlementIDs = map[string]struct{}{} + } } // Record the bytes this fold shadowed in the base keyspace. The diff --git a/pkg/synccompactor/incremental_hardening_test.go b/pkg/synccompactor/incremental_hardening_test.go index 3435270ff..e844a2b38 100644 --- a/pkg/synccompactor/incremental_hardening_test.go +++ b/pkg/synccompactor/incremental_hardening_test.go @@ -92,6 +92,21 @@ func TestIncrementalExpansionOutcomeLogging(t *testing.T) { options: []Option{WithEngine(c1zstore.EnginePebble), WithIncrementalExpansion()}, wantOutcome: "declined", wantReason: "cycle", }, + { + name: "invalid graph counters", + build: func(t *testing.T, ctx context.Context, dir string) []*CompactableSync { + entries := buildIncrementalFixtures(t, ctx, dir) + graph := baseGraphForFixtures(t, ctx) + graph.NextNodeID = 0 + store, err := dotc1z.NewStore(ctx, entries[0].FilePath, dotc1z.WithTmpDir(t.TempDir())) + require.NoError(t, err) + persistFixtureGraph(t, ctx, store, entries[0].SyncID, graph) + require.NoError(t, store.Close(ctx)) + return entries + }, + options: []Option{WithEngine(c1zstore.EnginePebble), WithIncrementalExpansion()}, + wantOutcome: "fell_back", wantReason: "base_graph_error", + }, { name: "unsupported engine", build: func(t *testing.T, ctx context.Context, dir string) []*CompactableSync { return buildIncrementalFixturesEngine(t, ctx, dir, c1zstore.EngineSQLite) diff --git a/pkg/synccompactor/pebble/merge.go b/pkg/synccompactor/pebble/merge.go index 8ef3ae444..1da9db19a 100644 --- a/pkg/synccompactor/pebble/merge.go +++ b/pkg/synccompactor/pebble/merge.go @@ -157,7 +157,7 @@ func (s *FoldStats) bumpReplaced(bucket string, n int64) { // (Engine.BuildGrantDigests rebuilds both keyspaces atomically from // scratch, so no separate drop is needed even then — see // compactPebbleFold). -func MergeInto(ctx context.Context, dest *enginepkg.Engine, sources []SourceSync, destSyncID string) (FoldStats, error) { +func MergeInto(ctx context.Context, dest *enginepkg.Engine, sources []SourceSync, destSyncID string, collectGrantEntitlementIDs bool) (FoldStats, error) { var stats FoldStats if dest == nil { return stats, errors.New("synccompactor/pebble.MergeInto: dest engine is nil") @@ -180,7 +180,7 @@ func MergeInto(ctx context.Context, dest *enginepkg.Engine, sources []SourceSync if s.Engine == nil || s.SyncID == "" { continue } - srcStats, err := mergeOneSource(ctx, dest, s, destSyncID) + srcStats, err := mergeOneSource(ctx, dest, s, destSyncID, collectGrantEntitlementIDs) stats.Add(srcStats) if err != nil { return stats, fmt.Errorf("merge source %s: %w", s.SyncID, err) @@ -211,10 +211,10 @@ const mergeRawFlushRecords = 32768 // incumbent, mirroring the engine's Put*RecordsIfNewer rule — // missing discovered_at scans as 0, reproducing its nil-timestamp // ordering ("never overwrite an incumbent, always fill a hole"). -func mergeOneSource(ctx context.Context, dest *enginepkg.Engine, s SourceSync, destSyncID string) (FoldStats, error) { +func mergeOneSource(ctx context.Context, dest *enginepkg.Engine, s SourceSync, destSyncID string, collectGrantEntitlementIDs bool) (FoldStats, error) { var stats FoldStats for _, bucket := range allBuckets() { - bucketStats, err := mergeBucketRawIfNewer(ctx, dest, s.Engine, bucket) + bucketStats, err := mergeBucketRawIfNewer(ctx, dest, s.Engine, bucket, collectGrantEntitlementIDs) stats.Add(bucketStats) if err != nil { return stats, fmt.Errorf("merge %s: %w", bucket.name, err) @@ -223,7 +223,7 @@ func mergeOneSource(ctx context.Context, dest *enginepkg.Engine, s SourceSync, d return stats, nil } -func mergeBucketRawIfNewer(ctx context.Context, dest *enginepkg.Engine, src *enginepkg.Engine, bucket bucketSpec) (FoldStats, error) { +func mergeBucketRawIfNewer(ctx context.Context, dest *enginepkg.Engine, src *enginepkg.Engine, bucket bucketSpec, collectGrantEntitlementIDs bool) (FoldStats, error) { var stats FoldStats lower, upper := bucket.syncRange() iter, err := src.NewIter(&pebble.IterOptions{LowerBound: lower, UpperBound: upper}) @@ -319,11 +319,13 @@ func mergeBucketRawIfNewer(ctx context.Context, dest *enginepkg.Engine, src *eng } stats.TouchedGrantPartitions[partition] = struct{}{} } - _, _, entID, _, _, _, scanErr := scanGrantIndexFieldsBytes(value) - if scanErr != nil { - return stats, scanErr + if collectGrantEntitlementIDs { + _, _, entID, _, _, _, scanErr := scanGrantIndexFieldsBytes(value) + if scanErr != nil { + return stats, scanErr + } + stats.noteGrantEntitlementID(entID) } - stats.noteGrantEntitlementID(entID) } if err := forEachIndexKeyFromRaw(bucket, key, lower, value, &scratch, nil, setIndexKey); err != nil { return stats, err diff --git a/pkg/synccompactor/pebble/merge_test.go b/pkg/synccompactor/pebble/merge_test.go index 13fb28b48..16b806d3c 100644 --- a/pkg/synccompactor/pebble/merge_test.go +++ b/pkg/synccompactor/pebble/merge_test.go @@ -55,7 +55,7 @@ func TestMergeIntoUnionNewerWins(t *testing.T) { // No SetCurrentSync here: MergeInto must bind the dest engine to // destSyncID itself (record values carry no sync_id, so a stale // binding would silently write into the wrong sync's keyspace). - stats, err := MergeInto(ctx, dst, []SourceSync{{Engine: src1, SyncID: syncA}, {Engine: src2, SyncID: syncB}}, destSync) + stats, err := MergeInto(ctx, dst, []SourceSync{{Engine: src1, SyncID: syncA}, {Engine: src2, SyncID: syncB}}, destSync, false) require.NoError(t, err, "MergeInto") // src2's g-shared@newer overrode src1's incumbent — exactly one // override, and its dead bytes must cover at least the incumbent's @@ -107,7 +107,7 @@ func TestMergeIntoGrantWritesTracksActualChanges(t *testing.T) { require.NoError(t, src.PutGrantRecords(ctx, grantAt(syncID, "g1", at))) // First merge: a genuinely new grant is admitted. - stats, err := MergeInto(ctx, dst, []SourceSync{{Engine: src, SyncID: syncID}}, destSync) + stats, err := MergeInto(ctx, dst, []SourceSync{{Engine: src, SyncID: syncID}}, destSync, false) require.NoError(t, err) require.EqualValues(t, 1, stats.GrantWrites, "first merge admits one new grant") @@ -115,7 +115,7 @@ func TestMergeIntoGrantWritesTracksActualChanges(t *testing.T) { // record is now byte-identical to the incumbent, which // mergeBucketRawIfNewer treats as a true no-op — zero grant writes, // even though a grant record was iterated and compared. - stats2, err := MergeInto(ctx, dst, []SourceSync{{Engine: src, SyncID: syncID}}, destSync) + stats2, err := MergeInto(ctx, dst, []SourceSync{{Engine: src, SyncID: syncID}}, destSync, false) require.NoError(t, err) require.Zero(t, stats2.GrantWrites, "resubmitting an identical grant must not count as a write") @@ -123,7 +123,7 @@ func TestMergeIntoGrantWritesTracksActualChanges(t *testing.T) { empty, _ := newEngine(t, "gw-empty-src") emptySyncID := ksuid.New().String() require.NoError(t, empty.SetCurrentSync(ctx, emptySyncID)) - stats3, err := MergeInto(ctx, dst, []SourceSync{{Engine: empty, SyncID: emptySyncID}}, destSync) + stats3, err := MergeInto(ctx, dst, []SourceSync{{Engine: empty, SyncID: emptySyncID}}, destSync, false) require.NoError(t, err) require.Zero(t, stats3.GrantWrites, "a source with no grants contributes zero grant writes") @@ -132,7 +132,7 @@ func TestMergeIntoGrantWritesTracksActualChanges(t *testing.T) { src2, _ := newEngine(t, "gw-src2") require.NoError(t, src2.SetCurrentSync(ctx, syncID)) require.NoError(t, src2.PutGrantRecords(ctx, grantAt(syncID, "g1", newer))) - stats4, err := MergeInto(ctx, dst, []SourceSync{{Engine: src2, SyncID: syncID}}, destSync) + stats4, err := MergeInto(ctx, dst, []SourceSync{{Engine: src2, SyncID: syncID}}, destSync, false) require.NoError(t, err) require.EqualValues(t, 1, stats4.GrantWrites, "a strictly-newer override counts as a grant write") } @@ -163,7 +163,7 @@ func TestMergeIntoTieKeepsIncumbent(t *testing.T) { require.NoError(t, src2.PutGrantRecords(ctx, g2)) // src1 applied first → incumbent wins the tie. MergeInto binds the // dest sync itself. - stats, err := MergeInto(ctx, dst, []SourceSync{{Engine: src1, SyncID: syncA}, {Engine: src2, SyncID: syncB}}, destSync) + stats, err := MergeInto(ctx, dst, []SourceSync{{Engine: src1, SyncID: syncA}, {Engine: src2, SyncID: syncB}}, destSync, false) require.NoError(t, err) // A tie keeps the incumbent — nothing is overridden, no dead bytes. require.Zero(t, stats.OverriddenRecords, "FoldStats.OverriddenRecords (tie keeps incumbent)") @@ -247,7 +247,7 @@ func TestMergeIntoBaseNewerKeepsBaseAllTypes(t *testing.T) { dest, destSync := seedBase(t, ctx, "base-newer-dest", baseSet) src := buildEngineSource(t, ctx, "older-src", olderSet) - stats, err := MergeInto(ctx, dest, []SourceSync{src}, destSync) + stats, err := MergeInto(ctx, dest, []SourceSync{src}, destSync, false) require.NoError(t, err) // Nothing regressed: the base (newer) version survives for every type. @@ -273,7 +273,7 @@ func TestMergeIntoNewerPartialOverridesBaseAllTypes(t *testing.T) { dest, destSync := seedBase(t, ctx, "base-older-dest", baseSet) src := buildEngineSource(t, ctx, "newer-src", newerSet) - stats, err := MergeInto(ctx, dest, []SourceSync{src}, destSync) + stats, err := MergeInto(ctx, dest, []SourceSync{src}, destSync, false) require.NoError(t, err) // The newer partial wins for every type. @@ -340,7 +340,7 @@ func TestMergeIntoDeadBytesExactCount(t *testing.T) { }.Build() src := buildEngineSource(t, ctx, "deadbytes-src", recordSet{gs: []*v3.GrantRecord{override}}) - stats, err := MergeInto(ctx, dest, []SourceSync{src}, destSync) + stats, err := MergeInto(ctx, dest, []SourceSync{src}, destSync, false) require.NoError(t, err) require.Equal(t, int64(1), stats.OverriddenRecords) require.Equal(t, wantDead, stats.DeadBytes, @@ -386,7 +386,7 @@ func TestMergeIntoCollectsGrantEntitlementIDs(t *testing.T) { }, }) - stats, err := MergeInto(ctx, dest, []SourceSync{src}, destSync) + stats, err := MergeInto(ctx, dest, []SourceSync{src}, destSync, true) require.NoError(t, err) got := make([]string, 0, len(stats.GrantEntitlementIDs)) @@ -397,6 +397,21 @@ func TestMergeIntoCollectsGrantEntitlementIDs(t *testing.T) { "only applied records contribute; identical/older no-ops must not") } +func TestMergeIntoSkipsGrantEntitlementIDsWhenDisabled(t *testing.T) { + ctx := context.Background() + now := time.Unix(2000, 0).UTC() + dest, destSync := seedBase(t, ctx, "entids-disabled-dest", recordSet{}) + src := buildEngineSource(t, ctx, "entids-disabled-src", recordSet{ + gs: []*v3.GrantRecord{grantEnt("g-new", "ent-added", "dave", now)}, + }) + + stats, err := MergeInto(ctx, dest, []SourceSync{src}, destSync, false) + require.NoError(t, err) + require.Equal(t, int64(1), stats.GrantWrites) + require.Nil(t, stats.GrantEntitlementIDs, + "flag-off merge must not allocate the changed-entitlement hashmap") +} + // TestFoldStatsAddUnionsGrantEntitlementIDs: Add unions the sets. func TestFoldStatsAddUnionsGrantEntitlementIDs(t *testing.T) { var total FoldStats diff --git a/pkg/synccompactor/pebble/winner_equivalence_test.go b/pkg/synccompactor/pebble/winner_equivalence_test.go index 5f41968f0..d2ad34d65 100644 --- a/pkg/synccompactor/pebble/winner_equivalence_test.go +++ b/pkg/synccompactor/pebble/winner_equivalence_test.go @@ -263,7 +263,7 @@ func TestPebbleStrategiesAgreeOnWinners(t *testing.T) { for i, rs := range srcSets { sources[i] = buildEngineSource(t, ctx, fmt.Sprintf("fold-src%d", i), rs) } - _, err := MergeInto(ctx, dest, sources, destSyncID) + _, err := MergeInto(ctx, dest, sources, destSyncID, false) return err }) require.Equal(t, want, got) From 93923eb8441a90ccaf2dda9406076530dc4739ea Mon Sep 17 00:00:00 2001 From: manojacs Date: Mon, 10 Aug 2026 22:20:51 +0000 Subject: [PATCH 09/15] test(sync): adapt graph compatibility to checkpoint changes Co-authored-by: c1-squire-dev[bot] --- pkg/sync/graph_compatibility_matrix_test.go | 4 +++- pkg/sync/graph_from_token_test.go | 8 ++++++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/pkg/sync/graph_compatibility_matrix_test.go b/pkg/sync/graph_compatibility_matrix_test.go index 30a6b2ebd..99bbbb0ee 100644 --- a/pkg/sync/graph_compatibility_matrix_test.go +++ b/pkg/sync/graph_compatibility_matrix_test.go @@ -12,7 +12,9 @@ import ( // TestGraphFromStore and synccompactor's TestCompactorGraphCompatibilityHealing. func TestEntitlementGraphTokenCompatibilityMatrix(t *testing.T) { ctx := context.Background() - stateWithGraph := newState() + // Opt in to the legacy inline-graph token shape. Current checkpoints omit + // graphs by default, while readers remain compatible with older tokens. + stateWithGraph := newState(withCheckpointEntitlementGraph(true)) graph := stateWithGraph.EntitlementGraph(ctx) graph.AddEntitlementID("a") graph.Loaded = true diff --git a/pkg/sync/graph_from_token_test.go b/pkg/sync/graph_from_token_test.go index f8846aee3..d5db4752c 100644 --- a/pkg/sync/graph_from_token_test.go +++ b/pkg/sync/graph_from_token_test.go @@ -18,7 +18,9 @@ func TestGraphFromToken(t *testing.T) { g.AddEntitlementID("eng:member") require.NoError(t, g.AddEdge(ctx, "eng:manager", "eng:member", false, nil)) - st := newState() + // Opt in to the legacy inline-graph token shape. New checkpoints omit the + // graph by default, but readers must remain compatible with older tokens. + st := newState(withCheckpointEntitlementGraph(true)) st.entitlementGraph = g token, err := st.Marshal() require.NoError(t, err) @@ -57,7 +59,9 @@ func TestPrepareExpansionReplayToken_ClearsPreservedGraph(t *testing.T) { g.MarkEdgeExpanded("eng:manager", "eng:member") g.Loaded = true - st := newState() + // Build the legacy inline-graph token shape so replay compatibility is + // exercised even though new checkpoints omit graphs by default. + st := newState(withCheckpointEntitlementGraph(true)) st.entitlementGraph = g token, err := st.Marshal() require.NoError(t, err) From 1b3362023a0b8ce1e97adb1ae1742d2a0aa301c1 Mon Sep 17 00:00:00 2001 From: manojacs Date: Tue, 11 Aug 2026 16:29:41 +0000 Subject: [PATCH 10/15] fix(compactor): merge split edge specs before classification Co-authored-by: c1-squire-dev[bot] --- pkg/synccompactor/compactor.go | 108 +++++++++++++----- .../incremental_expansion_test.go | 36 ++++++ 2 files changed, 116 insertions(+), 28 deletions(-) diff --git a/pkg/synccompactor/compactor.go b/pkg/synccompactor/compactor.go index 7ec7ad40c..456984bd9 100644 --- a/pkg/synccompactor/compactor.go +++ b/pkg/synccompactor/compactor.go @@ -678,10 +678,11 @@ func (c *Compactor) expandGrantsIncremental(ctx context.Context, newSyncId strin } // Every rule grant currently in the compacted c1z (base + merged - // increments) yields one or more edges. New edges are those the base graph - // didn't already have expanded. - var newEdges []expand.NewEdge - currentBaseNodeEdges := make(map[[2]int]struct{}) + // increments) contributes to one or more edges. Multiple grants may describe + // different pieces of the SAME edge, so merge their specs before comparing + // them with the base graph. Comparing each piece independently turns an + // unchanged split filter (for example users + groups) into false narrowing. + currentEdges := make(map[[2]string]expand.NewEdge) for pe, err := range c.compactedC1z.Grants().PendingExpansion(walkCtx) { if err != nil { if endErr := c.restoreEndedSync(ctx); endErr != nil { @@ -694,38 +695,62 @@ func (c *Compactor) expandGrantsIncremental(ctx context.Context, newSyncId strin continue } for _, src := range anno.GetEntitlementIds() { - srcNode := base.GetNode(src) - dstNode := base.GetNode(pe.TargetEntitlementID) - if srcNode != nil && dstNode != nil && srcNode.Id != dstNode.Id { - currentBaseNodeEdges[[2]int{srcNode.Id, dstNode.Id}] = struct{}{} - } - baseEdge, inBase := baseGraphEdge(base, src, pe.TargetEntitlementID) curEdge := expand.NewEdge{ SourceEntitlementID: src, DestEntitlementID: pe.TargetEntitlementID, Shallow: anno.GetShallow(), ResourceTypeIDs: anno.GetResourceTypeIds(), } - if !inBase { - newEdges = append(newEdges, curEdge) // brand-new edge - continue + key := [2]string{src, pe.TargetEntitlementID} + if existing, ok := currentEdges[key]; ok { + currentEdges[key] = mergeCurrentEdgeSpecs(existing, curEdge) + } else { + curEdge.ResourceTypeIDs = append([]string(nil), curEdge.ResourceTypeIDs...) + currentEdges[key] = curEdge } - // Existing edge: compare specs, not just endpoints (C3). - switch classifyEdgeSpecChange(baseEdge, curEdge) { - case edgeSpecNarrowed: - // Revocation-shaped (shallow-ified / filter tightened): can't - // remove grants incrementally — decline via the named hook (#6). - if endErr := c.restoreEndedSync(ctx); endErr != nil { - return false, endErr - } - return false, expand.ErrIncrementalRevocationDecline - case edgeSpecWidened: - // More members now qualify: re-expand (AddEdge folds the wider - // spec into the graph, deep-wins/unfiltered-wins). - newEdges = append(newEdges, curEdge) - case edgeSpecUnchanged: - // nothing to do + } + } + + var newEdges []expand.NewEdge + currentBaseNodeEdges := make(map[[2]int]struct{}) + currentEdgeKeys := make([][2]string, 0, len(currentEdges)) + for key := range currentEdges { + currentEdgeKeys = append(currentEdgeKeys, key) + } + sort.Slice(currentEdgeKeys, func(i, j int) bool { + if currentEdgeKeys[i][0] != currentEdgeKeys[j][0] { + return currentEdgeKeys[i][0] < currentEdgeKeys[j][0] + } + return currentEdgeKeys[i][1] < currentEdgeKeys[j][1] + }) + for _, key := range currentEdgeKeys { + curEdge := currentEdges[key] + srcNode := base.GetNode(curEdge.SourceEntitlementID) + dstNode := base.GetNode(curEdge.DestEntitlementID) + if srcNode != nil && dstNode != nil && srcNode.Id != dstNode.Id { + currentBaseNodeEdges[[2]int{srcNode.Id, dstNode.Id}] = struct{}{} + } + baseEdge, inBase := baseGraphEdge(base, curEdge.SourceEntitlementID, curEdge.DestEntitlementID) + if !inBase { + newEdges = append(newEdges, curEdge) // brand-new edge + continue + } + // Existing edge: compare its combined current spec with the combined + // spec persisted in the base graph. + switch classifyEdgeSpecChange(baseEdge, curEdge) { + case edgeSpecNarrowed: + // Revocation-shaped (shallow-ified / filter tightened): can't + // remove grants incrementally — decline via the named hook (#6). + if endErr := c.restoreEndedSync(ctx); endErr != nil { + return false, endErr } + return false, expand.ErrIncrementalRevocationDecline + case edgeSpecWidened: + // More members now qualify: re-expand (AddEdge folds the wider + // spec into the graph, deep-wins/unfiltered-wins). + newEdges = append(newEdges, curEdge) + case edgeSpecUnchanged: + // nothing to do } } // PendingExpansion describes the complete current edge set. Check the @@ -1054,6 +1079,33 @@ func baseGraphEdge(g *expand.EntitlementGraph, src, dst string) (*expand.Edge, b return &e, true } +// mergeCurrentEdgeSpecs folds parallel connector rules for the same endpoints +// into the one effective graph edge AddEdge would build: deep wins over +// shallow, an unfiltered rule wins over filtered rules, and otherwise filters +// are unioned. +func mergeCurrentEdgeSpecs(left, right expand.NewEdge) expand.NewEdge { + out := left + out.Shallow = left.Shallow && right.Shallow + if len(left.ResourceTypeIDs) == 0 || len(right.ResourceTypeIDs) == 0 { + out.ResourceTypeIDs = nil + return out + } + + resourceTypeIDs := make(map[string]struct{}, len(left.ResourceTypeIDs)+len(right.ResourceTypeIDs)) + for _, id := range left.ResourceTypeIDs { + resourceTypeIDs[id] = struct{}{} + } + for _, id := range right.ResourceTypeIDs { + resourceTypeIDs[id] = struct{}{} + } + out.ResourceTypeIDs = make([]string, 0, len(resourceTypeIDs)) + for id := range resourceTypeIDs { + out.ResourceTypeIDs = append(out.ResourceTypeIDs, id) + } + sort.Strings(out.ResourceTypeIDs) + return out +} + type edgeSpecChange int const ( diff --git a/pkg/synccompactor/incremental_expansion_test.go b/pkg/synccompactor/incremental_expansion_test.go index fba338d7b..c97610194 100644 --- a/pkg/synccompactor/incremental_expansion_test.go +++ b/pkg/synccompactor/incremental_expansion_test.go @@ -631,6 +631,42 @@ func TestCompactor_IncrementalNarrowedEdgeDeclines(t *testing.T) { require.Equal(t, fullGrants, incGrants, "declined incremental must equal full expansion") } +func TestSplitEdgeSpecsAreMergedBeforeClassification(t *testing.T) { + base := &expand.Edge{ + IsShallow: false, + ResourceTypeIDs: []string{"group", "user"}, + } + users := expand.NewEdge{ + SourceEntitlementID: "ent-a", + DestEntitlementID: "ent-b", + Shallow: true, + ResourceTypeIDs: []string{"user"}, + } + groups := expand.NewEdge{ + SourceEntitlementID: "ent-a", + DestEntitlementID: "ent-b", + Shallow: false, + ResourceTypeIDs: []string{"group"}, + } + + // Comparing either fragment by itself produces the previous false + // revocation: each fragment is narrower than the effective base edge. + require.Equal(t, edgeSpecNarrowed, classifyEdgeSpecChange(base, users)) + require.Equal(t, edgeSpecNarrowed, classifyEdgeSpecChange(base, groups)) + + // Production now merges both fragments first. Deep wins and the filters + // union, exactly matching the effective base edge. + current := mergeCurrentEdgeSpecs(users, groups) + require.False(t, current.Shallow) + require.ElementsMatch(t, []string{"group", "user"}, current.ResourceTypeIDs) + require.Equal(t, edgeSpecUnchanged, classifyEdgeSpecChange(base, current)) + + unfiltered := users + unfiltered.ResourceTypeIDs = nil + current = mergeCurrentEdgeSpecs(groups, unfiltered) + require.Nil(t, current.ResourceTypeIDs, "an unfiltered fragment makes the effective edge unfiltered") +} + // TestCompactor_IncrementalDoesNotMutateBaseGraph (U1): running incremental // expansion must not mutate the graph persisted in the caller's base artifact. func TestCompactor_IncrementalDoesNotMutateBaseGraph(t *testing.T) { From 9dd00f356e0e02d0bd09c92bb13e95ed81106cbe Mon Sep 17 00:00:00 2001 From: manojacs Date: Tue, 11 Aug 2026 20:12:33 +0000 Subject: [PATCH 11/15] fix: address incremental expansion review findings Co-authored-by: c1-squire-dev[bot] --- cmd/baton-compat-harness/driver_test.go | 23 +++- cmd/baton-compat-harness/graph_modes_new.go | 2 +- cmd/baton-compat-harness/main.go | 128 +++++++++++++++++- pkg/sync/expand/graph_blob.go | 18 ++- pkg/sync/expand/incremental.go | 53 +++++++- pkg/sync/expand/incremental_benchmark_test.go | 33 +++++ pkg/sync/expand/incremental_test.go | 23 ++++ pkg/sync/state.go | 11 +- pkg/synccompactor/compactor.go | 51 ++++++- pkg/synccompactor/compactor_pebble.go | 22 +-- .../incremental_expansion_test.go | 64 ++++++--- pkg/synccompactor/pebble/merge.go | 27 +++- pkg/synccompactor/pebble/merge_test.go | 22 +-- .../pebble/winner_equivalence_test.go | 2 +- 14 files changed, 409 insertions(+), 70 deletions(-) diff --git a/cmd/baton-compat-harness/driver_test.go b/cmd/baton-compat-harness/driver_test.go index 579b409df..e8728b864 100644 --- a/cmd/baton-compat-harness/driver_test.go +++ b/cmd/baton-compat-harness/driver_test.go @@ -122,6 +122,7 @@ type compatDriverResult struct { IncrementalError string `json:"incremental_error"` ArtifactPath string `json:"artifact_path"` AllocatedBytes uint64 `json:"allocated_bytes"` + LogicalDigest string `json:"logical_digest"` } func TestDefaultPathPerformanceAgainstPinnedMain(t *testing.T) { @@ -158,9 +159,10 @@ func TestDefaultPathPerformanceAgainstPinnedMain(t *testing.T) { baseResult := runHarness(t, mainBin, "resume", base) require.Empty(t, baseResult.SyncErr) - measure := func(t *testing.T, bin, label string) uint64 { + measure := func(t *testing.T, bin, label string) (uint64, string) { t.Helper() values := make([]uint64, 0, 5) + var logicalDigest string for i := 0; i < 5; i++ { input := filepath.Join(tmp, fmt.Sprintf("%s-input-%d.c1z", label, i)) copyCompatArtifact(t, base, input) @@ -170,13 +172,26 @@ func TestDefaultPathPerformanceAgainstPinnedMain(t *testing.T) { require.Equal(t, wantEnts, result.Ents) require.Equal(t, wantGrants, result.Grants) require.Positive(t, result.AllocatedBytes) + inspection := runHarness(t, candidateBin, "graph-inspect", out) + require.False(t, inspection.GraphPresent, + "default flag-off compaction must not write a graph sidecar (%s run %d)", label, i) + require.False(t, inspection.GraphReusable) + require.NotEmpty(t, inspection.LogicalDigest) + if logicalDigest == "" { + logicalDigest = inspection.LogicalDigest + } else { + require.Equal(t, logicalDigest, inspection.LogicalDigest, + "logical output changed across identical %s runs", label) + } values = append(values, result.AllocatedBytes) } sort.Slice(values, func(i, j int) bool { return values[i] < values[j] }) - return values[len(values)/2] + return values[len(values)/2], logicalDigest } - mainAlloc := measure(t, mainBin, "main") - candidateAlloc := measure(t, candidateBin, "candidate") + mainAlloc, mainDigest := measure(t, mainBin, "main") + candidateAlloc, candidateDigest := measure(t, candidateBin, "candidate") + require.Equal(t, mainDigest, candidateDigest, + "flag-off compaction must preserve exact logical resources, entitlements, and grants") require.LessOrEqual(t, candidateAlloc, mainAlloc*110/100, "default compaction allocation regression: candidate=%d main=%d", candidateAlloc, mainAlloc) } diff --git a/cmd/baton-compat-harness/graph_modes_new.go b/cmd/baton-compat-harness/graph_modes_new.go index 1f14953e0..4a14bddec 100644 --- a/cmd/baton-compat-harness/graph_modes_new.go +++ b/cmd/baton-compat-harness/graph_modes_new.go @@ -192,7 +192,7 @@ func graphCompatInspect(ctx context.Context, path string) (compatResult, error) err = nil } result.GraphReusable = graph != nil - result.Resources, result.Ents, result.Grants, err = countList(ctx, store) + result.Resources, result.Ents, result.Grants, err = summarizeRows(ctx, store, &result) if closeErr := store.Close(ctx); err == nil { err = closeErr } diff --git a/cmd/baton-compat-harness/main.go b/cmd/baton-compat-harness/main.go index bb6c32203..4a8ad98f7 100644 --- a/cmd/baton-compat-harness/main.go +++ b/cmd/baton-compat-harness/main.go @@ -34,18 +34,21 @@ package main import ( "context" + "crypto/sha256" "encoding/json" "flag" "fmt" "os" "path/filepath" "runtime" + "sort" "strings" "time" "github.com/grpc-ecosystem/go-grpc-middleware/logging/zap/ctxzap" "go.uber.org/zap" "google.golang.org/grpc" + "google.golang.org/protobuf/proto" v2 "github.com/conductorone/baton-sdk/pb/c1/connector/v2" "github.com/conductorone/baton-sdk/pkg/annotations" @@ -328,6 +331,128 @@ func countList(ctx context.Context, store c1zstore.Store) (int, int, int, error) return resources, ents, grants, nil } +// logicalContentDigest hashes the complete logical rows returned by the public +// reader surface. Stable sorting makes it independent of pagination and Pebble +// iteration order; deterministic protobuf encoding includes nested refs, +// sources, and annotations instead of comparing counts only. +func logicalContentDigest(ctx context.Context, store c1zstore.Store) (string, error) { + rows := make([]string, 0) + appendRows := func(kind byte, messages []proto.Message) error { + for _, message := range messages { + data, err := (proto.MarshalOptions{Deterministic: true}).Marshal(message) + if err != nil { + return err + } + rows = append(rows, fmt.Sprintf("%c:%x", kind, data)) + } + return nil + } + + pageToken := "" + for { + resp, err := store.ListResourceTypes(ctx, v2.ResourceTypesServiceListResourceTypesRequest_builder{ + PageToken: pageToken, + }.Build()) + if err != nil { + return "", fmt.Errorf("digest resource types: %w", err) + } + messages := make([]proto.Message, 0, len(resp.GetList())) + for _, item := range resp.GetList() { + messages = append(messages, item) + } + if err := appendRows('T', messages); err != nil { + return "", fmt.Errorf("digest resource types: %w", err) + } + pageToken = resp.GetNextPageToken() + if pageToken == "" { + break + } + } + + pageToken = "" + for { + resp, err := store.ListResources(ctx, v2.ResourcesServiceListResourcesRequest_builder{ + PageToken: pageToken, + }.Build()) + if err != nil { + return "", fmt.Errorf("digest resources: %w", err) + } + messages := make([]proto.Message, 0, len(resp.GetList())) + for _, item := range resp.GetList() { + messages = append(messages, item) + } + if err := appendRows('R', messages); err != nil { + return "", fmt.Errorf("digest resources: %w", err) + } + pageToken = resp.GetNextPageToken() + if pageToken == "" { + break + } + } + + pageToken = "" + for { + resp, err := store.ListEntitlements(ctx, v2.EntitlementsServiceListEntitlementsRequest_builder{ + PageToken: pageToken, + }.Build()) + if err != nil { + return "", fmt.Errorf("digest entitlements: %w", err) + } + messages := make([]proto.Message, 0, len(resp.GetList())) + for _, item := range resp.GetList() { + messages = append(messages, item) + } + if err := appendRows('E', messages); err != nil { + return "", fmt.Errorf("digest entitlements: %w", err) + } + pageToken = resp.GetNextPageToken() + if pageToken == "" { + break + } + } + + pageToken = "" + for { + resp, err := store.ListGrants(ctx, v2.GrantsServiceListGrantsRequest_builder{ + PageToken: pageToken, + }.Build()) + if err != nil { + return "", fmt.Errorf("digest grants: %w", err) + } + messages := make([]proto.Message, 0, len(resp.GetList())) + for _, item := range resp.GetList() { + messages = append(messages, item) + } + if err := appendRows('G', messages); err != nil { + return "", fmt.Errorf("digest grants: %w", err) + } + pageToken = resp.GetNextPageToken() + if pageToken == "" { + break + } + } + + sort.Strings(rows) + hash := sha256.New() + for _, row := range rows { + _, _ = hash.Write([]byte(row)) + _, _ = hash.Write([]byte{'\n'}) + } + return fmt.Sprintf("%x", hash.Sum(nil)), nil +} + +func summarizeRows(ctx context.Context, store c1zstore.Store, result *compatResult) (int, int, int, error) { + resources, ents, grants, err := countList(ctx, store) + if err != nil { + return 0, 0, 0, err + } + result.LogicalDigest, err = logicalContentDigest(ctx, store) + if err != nil { + return 0, 0, 0, err + } + return resources, ents, grants, nil +} + // syncRunLister is the store surface the run inspection needs; asserted // dynamically because the harness compiles against two SDK versions. type syncRunLister interface { @@ -370,7 +495,7 @@ func countRows(ctx context.Context, c1zPath, tmpDir string, result *compatResult } defer func() { _ = store.Close(ctx) }() inspectRuns(ctx, store, result) - return countList(ctx, store) + return summarizeRows(ctx, store, result) } // compatResult is the machine-readable summary the driver parses from the @@ -397,6 +522,7 @@ type compatResult struct { IncrementalError string `json:"incremental_error,omitempty"` ArtifactPath string `json:"artifact_path,omitempty"` AllocatedBytes uint64 `json:"allocated_bytes,omitempty"` + LogicalDigest string `json:"logical_digest,omitempty"` } // graphCompatHandler is installed by graph_modes_new.go in the candidate diff --git a/pkg/sync/expand/graph_blob.go b/pkg/sync/expand/graph_blob.go index 6241f8eab..7e394fc6b 100644 --- a/pkg/sync/expand/graph_blob.go +++ b/pkg/sync/expand/graph_blob.go @@ -21,8 +21,12 @@ type graphBlobEnvelope struct { const graphBlobFormatVersion uint32 = 2 -// MarshalGraphBlob serializes a graph for the c1z sidecar, stamped with the -// sync it belongs to. Transient state is stripped first (a reload rebuilds it). +// MarshalGraphBlob serializes a legacy, unbound graph blob for compatibility +// tests. Transient state is stripped first (a reload rebuilds it). +// +// The blob has no grant-generation digest, so sync.GraphFromStore deliberately +// rejects it for incremental reuse. Production persistence must use +// MarshalGraphBlobWithGrantDigest. func MarshalGraphBlob(syncID string, g *EntitlementGraph) ([]byte, error) { return marshalGraphBlob(syncID, g, nil) } @@ -50,9 +54,13 @@ func marshalGraphBlob(syncID string, g *EntitlementGraph, digest *c1zstore.Grant return data, nil } -// UnmarshalGraphBlob parses a sidecar blob. Returns (nil, nil) when the blob -// belongs to a different sync than wantSyncID (stale inherited sidecar); -// pass "" to skip the guard. +// UnmarshalGraphBlob parses a graph for compatibility tests while discarding +// its grant-generation binding. Returns (nil, nil) when the blob belongs to a +// different sync than wantSyncID (stale inherited sidecar); pass "" to skip +// the guard. +// +// The returned graph must not drive incremental reuse. Production readers +// must use UnmarshalGraphBlobWithGrantDigest and verify the returned digest. func UnmarshalGraphBlob(data []byte, wantSyncID string) (*EntitlementGraph, error) { graph, _, err := UnmarshalGraphBlobWithGrantDigest(data, wantSyncID) return graph, err diff --git a/pkg/sync/expand/incremental.go b/pkg/sync/expand/incremental.go index a555d86c1..f85531095 100644 --- a/pkg/sync/expand/incremental.go +++ b/pkg/sync/expand/incremental.go @@ -272,17 +272,15 @@ func topologicalAffectedNodeOrder(g *EntitlementGraph, affected map[int]struct{} } } } - frontier := make([]int, 0, len(inDegree)) + frontier := make(intMinHeap, 0, len(inDegree)) for nodeID, degree := range inDegree { if degree == 0 { - frontier = append(frontier, nodeID) + frontier.push(nodeID) } } - sort.Ints(frontier) order := make([]int, 0, len(inDegree)) for len(frontier) > 0 { - nodeID := frontier[0] - frontier = frontier[1:] + nodeID := frontier.pop() order = append(order, nodeID) for childID := range g.SourcesToDestinations[nodeID] { if _, ok := inDegree[childID]; !ok { @@ -290,8 +288,7 @@ func topologicalAffectedNodeOrder(g *EntitlementGraph, affected map[int]struct{} } inDegree[childID]-- if inDegree[childID] == 0 { - frontier = append(frontier, childID) - sort.Ints(frontier) + frontier.push(childID) } } } @@ -301,6 +298,48 @@ func topologicalAffectedNodeOrder(g *EntitlementGraph, affected map[int]struct{} return order, nil } +// intMinHeap keeps the smallest ready node at the front without re-sorting +// every ready node after each insertion. +type intMinHeap []int + +func (h *intMinHeap) push(nodeID int) { + *h = append(*h, nodeID) + for child := len(*h) - 1; child > 0; { + parent := (child - 1) / 2 + if (*h)[parent] <= (*h)[child] { + break + } + (*h)[parent], (*h)[child] = (*h)[child], (*h)[parent] + child = parent + } +} + +func (h *intMinHeap) pop() int { + root := (*h)[0] + last := len(*h) - 1 + (*h)[0] = (*h)[last] + *h = (*h)[:last] + + for parent := 0; ; { + left := 2*parent + 1 + if left >= len(*h) { + break + } + child := left + right := left + 1 + if right < len(*h) && (*h)[right] < (*h)[left] { + child = right + } + if (*h)[parent] <= (*h)[child] { + break + } + (*h)[parent], (*h)[child] = (*h)[child], (*h)[parent] + parent = child + } + + return root +} + func (ie *IncrementalExpander) forwardReachable(seeds map[int]struct{}) map[int]struct{} { reached := make(map[int]struct{}) queue := make([]int, 0, len(seeds)) diff --git a/pkg/sync/expand/incremental_benchmark_test.go b/pkg/sync/expand/incremental_benchmark_test.go index ff9cd8ea6..36e8aa5b8 100644 --- a/pkg/sync/expand/incremental_benchmark_test.go +++ b/pkg/sync/expand/incremental_benchmark_test.go @@ -17,6 +17,39 @@ type incrementalBenchFixture struct { changed []string } +// BenchmarkTopologicalAffectedNodeOrderWideFanout exercises the shape that +// made repeated frontier sorting quadratic: one root makes every child ready +// during the same iteration. +func BenchmarkTopologicalAffectedNodeOrderWideFanout(b *testing.B) { + ctx := context.Background() + g := NewEntitlementGraph(ctx) + g.AddEntitlementID("root") + + const children = 10_000 + affected := make(map[int]struct{}, children+1) + affected[g.GetNode("root").Id] = struct{}{} + for i := 0; i < children; i++ { + child := fmt.Sprintf("child:%05d", i) + g.AddEntitlementID(child) + if err := g.AddEdge(ctx, "root", child, false, nil); err != nil { + b.Fatal(err) + } + affected[g.GetNode(child).Id] = struct{}{} + } + + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + order, err := topologicalAffectedNodeOrder(g, affected) + if err != nil { + b.Fatal(err) + } + if len(order) != children+1 { + b.Fatalf("unexpected order length: %d", len(order)) + } + } +} + // TestIncrementalPerformanceGates is intentionally opt-in because its // 100k-entitlement/100k-principal fixtures allocate hundreds of megabytes. // It enforces allocation/work gates; wall time remains benchmark evidence. diff --git a/pkg/sync/expand/incremental_test.go b/pkg/sync/expand/incremental_test.go index dd196ab4a..33f2968b1 100644 --- a/pkg/sync/expand/incremental_test.go +++ b/pkg/sync/expand/incremental_test.go @@ -141,6 +141,29 @@ func TestIncremental_BoundedWalk(t *testing.T) { require.Contains(t, principalsOn(t, ctx, store, "new:leaf"), "u7") } +func TestTopologicalAffectedNodeOrder_WideFanoutIsStable(t *testing.T) { + ctx := context.Background() + g := NewEntitlementGraph(ctx) + g.AddEntitlementID("root") + + const children = 5000 + affected := make(map[int]struct{}, children+1) + affected[g.GetNode("root").Id] = struct{}{} + for i := 0; i < children; i++ { + child := "child:" + itoa(i) + g.AddEntitlementID(child) + require.NoError(t, g.AddEdge(ctx, "root", child, false, nil)) + affected[g.GetNode(child).Id] = struct{}{} + } + + order, err := topologicalAffectedNodeOrder(g, affected) + require.NoError(t, err) + require.Len(t, order, children+1) + for i, nodeID := range order { + require.Equal(t, i+1, nodeID) + } +} + // TestIncremental_InsertBetween: a new node spliced between two existing // nodes (A -> newMid -> C added on top of an already-expanded A -> C). The // mid node is populated from A, and C gains only what the mid contributes diff --git a/pkg/sync/state.go b/pkg/sync/state.go index dd79dd965..ecec42c46 100644 --- a/pkg/sync/state.go +++ b/pkg/sync/state.go @@ -107,11 +107,12 @@ func PrepareExpansionReplayToken(stateStr string) (string, error) { return st.Marshal() } -// GraphFromToken parses a sync token and returns its persisted entitlement -// graph, for running an incremental expansion against a prior sync's graph. -// Returns nil if the token carried no graph (e.g. a sync without -// WithPreserveEntitlementGraph). Legacy: preserve now writes the graph into -// the c1z sidecar when the store supports it — prefer GraphFromStore. +// GraphFromToken parses a legacy sync token and returns its entitlement graph +// for compatibility tests. It returns nil if the token carried no graph. +// +// Token graphs have no grant-generation binding and must not drive incremental +// reuse. Production readers must use GraphFromStore, which verifies that the +// sidecar graph describes the store's exact sealed grant generation. func GraphFromToken(stateStr string) (*expand.EntitlementGraph, error) { st := newState() if err := st.Unmarshal(stateStr); err != nil { diff --git a/pkg/synccompactor/compactor.go b/pkg/synccompactor/compactor.go index 456984bd9..67623bb14 100644 --- a/pkg/synccompactor/compactor.go +++ b/pkg/synccompactor/compactor.go @@ -24,6 +24,8 @@ import ( "github.com/grpc-ecosystem/go-grpc-middleware/logging/zap/ctxzap" "go.opentelemetry.io/otel" "go.uber.org/zap" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" "github.com/conductorone/baton-sdk/pkg/uotel" ) @@ -132,7 +134,7 @@ func (c *Compactor) inferEngineFromInputs() (c1zstore.Engine, error) { if entry == nil || entry.FilePath == "" { continue } - f, err := os.Open(entry.FilePath) // #nosec G304 - compaction inputs are caller-provided c1z paths. + f, err := os.Open(entry.FilePath) // #nosec G304,G703 -- compaction inputs are intentionally caller-provided c1z paths. if err != nil { return "", fmt.Errorf("infer compactor engine from %s: %w", entry.FilePath, err) } @@ -533,7 +535,7 @@ func (c *Compactor) Compact(ctx context.Context) (*CompactableSync, error) { } func cpFile(ctx context.Context, sourcePath string, destPath string) error { - err := os.Rename(sourcePath, destPath) + err := os.Rename(sourcePath, destPath) // #nosec G703 -- compaction source and destination paths are intentional API inputs. if err == nil { return nil } @@ -547,7 +549,7 @@ func cpFile(ctx context.Context, sourcePath string, destPath string) error { } defer source.Close() - destination, err := os.Create(destPath) + destination, err := os.Create(destPath) // #nosec G703 -- the caller intentionally selects the compacted artifact destination. if err != nil { return fmt.Errorf("failed to create destination file: %w", err) } @@ -683,6 +685,8 @@ func (c *Compactor) expandGrantsIncremental(ctx context.Context, newSyncId strin // them with the base graph. Comparing each piece independently turns an // unchanged split filter (for example users + groups) into false narrowing. currentEdges := make(map[[2]string]expand.NewEdge) + sourceEntitlements := make(map[string]*v2.Entitlement) + missingSourceEntitlements := make(map[string]struct{}) for pe, err := range c.compactedC1z.Grants().PendingExpansion(walkCtx) { if err != nil { if endErr := c.restoreEndedSync(ctx); endErr != nil { @@ -695,6 +699,47 @@ func (c *Compactor) expandGrantsIncremental(ctx context.Context, newSyncId strin continue } for _, src := range anno.GetEntitlementIds() { + if _, missing := missingSourceEntitlements[src]; missing { + continue + } + sourceEntitlement, cached := sourceEntitlements[src] + if !cached { + resp, getErr := c.compactedC1z.GetEntitlement(walkCtx, + reader_v2.EntitlementsReaderServiceGetEntitlementRequest_builder{ + EntitlementId: src, + }.Build()) + if status.Code(getErr) == codes.NotFound { + missingSourceEntitlements[src] = struct{}{} + ctxzap.Extract(ctx).Debug("incremental expansion: source entitlement not found, skipping edge", + zap.String("src_entitlement_id", src), + zap.String("dst_entitlement_id", pe.TargetEntitlementID)) + continue + } + if getErr != nil { + if endErr := c.restoreEndedSync(ctx); endErr != nil { + return false, endErr + } + return false, fmt.Errorf("incremental expansion: get source entitlement %s: %w", src, getErr) + } + sourceEntitlement = resp.GetEntitlement() + sourceEntitlements[src] = sourceEntitlement + } + + sourceResourceID := sourceEntitlement.GetResource().GetId() + if sourceResourceID == nil { + if endErr := c.restoreEndedSync(ctx); endErr != nil { + return false, endErr + } + return false, fmt.Errorf("incremental expansion: source entitlement resource id was nil") + } + if pe.PrincipalResourceTypeID != sourceResourceID.GetResourceType() || + pe.PrincipalResourceID != sourceResourceID.GetResource() { + if endErr := c.restoreEndedSync(ctx); endErr != nil { + return false, endErr + } + return false, fmt.Errorf("incremental expansion: source entitlement resource id did not match grant principal id") + } + curEdge := expand.NewEdge{ SourceEntitlementID: src, DestEntitlementID: pe.TargetEntitlementID, diff --git a/pkg/synccompactor/compactor_pebble.go b/pkg/synccompactor/compactor_pebble.go index 8c8fbdbf1..90fc5aa22 100644 --- a/pkg/synccompactor/compactor_pebble.go +++ b/pkg/synccompactor/compactor_pebble.go @@ -492,7 +492,7 @@ func (c *Compactor) compactPebbleFold(ctx context.Context) (string, error) { var convertedInputs []string defer func() { for _, path := range convertedInputs { - _ = os.Remove(path) + _ = os.Remove(path) // #nosec G703 -- paths come only from CreateTemp in the compactor temp directory. } }() for i := len(c.entries) - 1; i >= 1; i-- { @@ -549,7 +549,11 @@ func (c *Compactor) compactPebbleFold(ctx context.Context) (string, error) { partialSyncIDs = append(partialSyncIDs, srcSyncID) partialTokens = append(partialTokens, readSourceSyncToken(ctx, srcEng, srcSyncID)) - mergeStats, mergeErr := mergepkg.MergeInto(ctx, destEng, []mergepkg.SourceSync{{Engine: srcEng, SyncID: srcSyncID}}, baseSyncID, c.incrementalExpansion) + var mergeOpts []mergepkg.MergeOption + if c.incrementalExpansion { + mergeOpts = append(mergeOpts, mergepkg.WithGrantEntitlementIDs()) + } + mergeStats, mergeErr := mergepkg.MergeInto(ctx, destEng, []mergepkg.SourceSync{{Engine: srcEng, SyncID: srcSyncID}}, baseSyncID, mergeOpts...) foldStats.Add(mergeStats) if cerr := w.Close(ctx); cerr != nil { l.Error("compactPebbleFold: error closing source store", zap.Error(cerr), zap.String("file", sourcePath)) @@ -897,7 +901,7 @@ func copyFileForFold(src, dst string) error { // readCompactionInputFormat reads the c1z header of path and returns its // on-disk format, rejecting anything that is not a supported v1/v3 c1z. func readCompactionInputFormat(path string) (dotc1z.C1ZFormat, error) { - f, err := os.Open(path) // #nosec G304 - compaction inputs are caller-provided c1z paths. + f, err := os.Open(path) // #nosec G304,G703 -- compaction inputs are intentionally caller-provided c1z paths. if err != nil { return dotc1z.C1ZFormatUnknown, fmt.Errorf("compactPebble: open input header %s: %w", path, err) } @@ -1016,11 +1020,11 @@ func (c *Compactor) convertSQLiteInputToPebble(ctx context.Context, cs *Compacta } convertedPath := tmp.Name() if err := tmp.Close(); err != nil { - _ = os.Remove(convertedPath) + _ = os.Remove(convertedPath) // #nosec G703 -- convertedPath was returned by CreateTemp above. return "", fmt.Errorf("compactPebble: close conversion temp file: %w", err) } // ToPebble requires the destination path to not exist. - if err := os.Remove(convertedPath); err != nil { + if err := os.Remove(convertedPath); err != nil { // #nosec G703 -- convertedPath was returned by CreateTemp above. return "", fmt.Errorf("compactPebble: remove conversion temp placeholder: %w", err) } @@ -1036,16 +1040,16 @@ func (c *Compactor) convertSQLiteInputToPebble(ctx context.Context, cs *Compacta syncID, err := resolveSQLiteCompactionSyncID(ctx, sqliteStore, cs.SyncID) if err != nil { _ = store.Close(ctx) - _ = os.Remove(convertedPath) + _ = os.Remove(convertedPath) // #nosec G703 -- convertedPath was returned by CreateTemp above. return "", fmt.Errorf("compactPebble: select sqlite input sync %s: %w", cs.FilePath, err) } if _, err := sqliteStore.ToPebble(ctx, convertedPath, syncID, dotc1z.WithConvertTmpDir(c.tmpDir)); err != nil { _ = store.Close(ctx) - _ = os.Remove(convertedPath) + _ = os.Remove(convertedPath) // #nosec G703 -- convertedPath was returned by CreateTemp above. return "", fmt.Errorf("compactPebble: convert sqlite input %s to pebble: %w", cs.FilePath, err) } if err := store.Close(ctx); err != nil { - _ = os.Remove(convertedPath) + _ = os.Remove(convertedPath) // #nosec G703 -- convertedPath was returned by CreateTemp above. return "", fmt.Errorf("compactPebble: close sqlite input after conversion %s: %w", cs.FilePath, err) } return convertedPath, nil @@ -1104,7 +1108,7 @@ func (c *Compactor) compactPebble(ctx context.Context, newSyncId string) error { var convertedInputs []string defer func() { for _, path := range convertedInputs { - _ = os.Remove(path) + _ = os.Remove(path) // #nosec G703 -- paths come only from CreateTemp in the compactor temp directory. } }() for i := len(c.entries) - 1; i >= 0; i-- { diff --git a/pkg/synccompactor/incremental_expansion_test.go b/pkg/synccompactor/incremental_expansion_test.go index c97610194..44fd6799d 100644 --- a/pkg/synccompactor/incremental_expansion_test.go +++ b/pkg/synccompactor/incremental_expansion_test.go @@ -695,15 +695,14 @@ func TestCompactor_IncrementalDoesNotMutateBaseGraph(t *testing.T) { require.Nil(t, after.GetNode("ent-a"), "new edge's node must not leak into the base artifact") } -// buildDanglingRefFixtures builds a base (ent-b -> ent-c, mandy) and a single -// increment whose only change is a rule grant whose SOURCE entitlement -// ("ent-ghost") is absent from the merged set — a dangling ref, which both -// paths must skip. -func buildDanglingRefFixtures(t *testing.T, ctx context.Context, dir string) []*CompactableSync { +// buildEdgeValidationFixtures builds a valid base plus one new rule targeting +// ent-d. sourceEntitlementID selects either a missing source ("ent-ghost") or +// an existing source whose resource differs from the rule principal ("ent-b"). +func buildEdgeValidationFixtures(t *testing.T, ctx context.Context, dir, sourceEntitlementID string) []*CompactableSync { t.Helper() - grpB, grpC := grp("grpB"), grp("grpC") + grpB, grpC, grpD := grp("grpB"), grp("grpC"), grp("grpD") mandy := usr("mandy") - entB, entC := ent("ent-b", grpB), ent("ent-c", grpC) + entB, entC, entD := ent("ent-b", grpB), ent("ent-c", grpC), ent("ent-d", grpD) userRT := v2.ResourceType_builder{Id: "user", DisplayName: "User"}.Build() groupRT := v2.ResourceType_builder{Id: "group", DisplayName: "Group"}.Build() @@ -713,15 +712,18 @@ func buildDanglingRefFixtures(t *testing.T, ctx context.Context, dir string) []* baseSyncID, err := base.StartNewSync(ctx, connectorstore.SyncTypeFull, "") require.NoError(t, err) require.NoError(t, base.PutResourceTypes(ctx, userRT, groupRT)) - require.NoError(t, base.PutResources(ctx, grpB, grpC, mandy)) - require.NoError(t, base.PutEntitlements(ctx, entB, entC)) + require.NoError(t, base.PutResources(ctx, grpB, grpC, grpD, mandy)) + require.NoError(t, base.PutEntitlements(ctx, entB, entC, entD)) require.NoError(t, base.PutGrants(ctx, memberGrant(entB, mandy), expandedGrant(entC, mandy, "ent-b"), ruleGrant(entC, grpB, "ent-b"), )) require.NoError(t, base.EndSync(ctx)) - persistFixtureGraph(t, ctx, base, baseSyncID, baseGraphForFixtures(t, ctx)) + baseGraph := baseGraphForFixtures(t, ctx) + baseGraph.AddEntitlementID("ent-d") + baseGraph.HasNoCycles = true + persistFixtureGraph(t, ctx, base, baseSyncID, baseGraph) require.NoError(t, base.Close(ctx)) incPath := filepath.Join(dir, "inc.c1z") @@ -730,10 +732,9 @@ func buildDanglingRefFixtures(t *testing.T, ctx context.Context, dir string) []* incSyncID, err := inc.StartNewSync(ctx, connectorstore.SyncTypePartial, "") require.NoError(t, err) require.NoError(t, inc.PutResourceTypes(ctx, userRT, groupRT)) - require.NoError(t, inc.PutResources(ctx, grpB, grpC)) - require.NoError(t, inc.PutEntitlements(ctx, entC)) - // rule grant: members of ent-ghost (which does not exist) get ent-c. - require.NoError(t, inc.PutGrants(ctx, ruleGrant(entC, grpB, "ent-ghost"))) + require.NoError(t, inc.PutResources(ctx, grpD)) + require.NoError(t, inc.PutEntitlements(ctx, entD)) + require.NoError(t, inc.PutGrants(ctx, ruleGrant(entD, grpD, sourceEntitlementID))) require.NoError(t, inc.EndSync(ctx)) require.NoError(t, inc.Close(ctx)) @@ -743,14 +744,12 @@ func buildDanglingRefFixtures(t *testing.T, ctx context.Context, dir string) []* } } -// TestCompactor_IncrementalDanglingRefMatchesFull (#11a): an increment with a -// grant referencing an entitlement absent from the merged set is skipped by -// both paths. This fixture also replaces the base rule, dropping ent-b -> -// ent-c, so the incremental path must safely decline and equal full expansion. +// TestCompactor_IncrementalDanglingRefMatchesFull (#11a): a missing source is +// skipped before it can create a phantom graph node or edge. func TestCompactor_IncrementalDanglingRefMatchesFull(t *testing.T) { ctx := context.Background() - incEntries := buildDanglingRefFixtures(t, ctx, t.TempDir()) + incEntries := buildEdgeValidationFixtures(t, ctx, t.TempDir(), "ent-ghost") cInc, cleanupInc, err := NewCompactor(ctx, t.TempDir(), incEntries, WithTmpDir(t.TempDir()), WithEngine(c1zstore.EnginePebble), @@ -760,9 +759,9 @@ func TestCompactor_IncrementalDanglingRefMatchesFull(t *testing.T) { defer func() { _ = cleanupInc() }() incOut, err := cInc.Compact(ctx) require.NoError(t, err) - require.False(t, cInc.incrementalExpansionRan, "the dropped base edge must decline to full expansion") + require.True(t, cInc.incrementalExpansionRan, "a dangling new edge must be skipped without forcing fallback") - fullEntries := buildDanglingRefFixtures(t, ctx, t.TempDir()) + fullEntries := buildEdgeValidationFixtures(t, ctx, t.TempDir(), "ent-ghost") cFull, cleanupFull, err := NewCompactor(ctx, t.TempDir(), fullEntries, WithTmpDir(t.TempDir()), WithEngine(c1zstore.EnginePebble)) require.NoError(t, err) @@ -773,6 +772,29 @@ func TestCompactor_IncrementalDanglingRefMatchesFull(t *testing.T) { incGrants := grantOutcome(t, ctx, incOut.FilePath, incOut.SyncID) fullGrants := grantOutcome(t, ctx, fullOut.FilePath, fullOut.SyncID) require.Equal(t, fullGrants, incGrants, "dangling-ref incremental must equal full") + graph := artifactGraph(t, ctx, incOut.FilePath, incOut.SyncID) + require.NotNil(t, graph) + require.Nil(t, graph.GetNode("ent-ghost"), "missing source must not persist as a phantom graph node") +} + +func TestCompactor_IncrementalPrincipalMismatchMatchesFullError(t *testing.T) { + ctx := context.Background() + + incEntries := buildEdgeValidationFixtures(t, ctx, t.TempDir(), "ent-b") + cInc, cleanupInc, err := NewCompactor(ctx, t.TempDir(), incEntries, + WithTmpDir(t.TempDir()), WithEngine(c1zstore.EnginePebble), WithIncrementalExpansion()) + require.NoError(t, err) + defer func() { _ = cleanupInc() }() + _, incErr := cInc.Compact(ctx) + require.ErrorContains(t, incErr, "source entitlement resource id did not match grant principal id") + + fullEntries := buildEdgeValidationFixtures(t, ctx, t.TempDir(), "ent-b") + cFull, cleanupFull, err := NewCompactor(ctx, t.TempDir(), fullEntries, + WithTmpDir(t.TempDir()), WithEngine(c1zstore.EnginePebble)) + require.NoError(t, err) + defer func() { _ = cleanupFull() }() + _, fullErr := cFull.Compact(ctx) + require.ErrorContains(t, fullErr, "source entitlement resource id did not match grant principal id") } // TestCompactor_IncrementalSealedArtifactLifecycle (#11b): after the diff --git a/pkg/synccompactor/pebble/merge.go b/pkg/synccompactor/pebble/merge.go index 1da9db19a..a13893ea2 100644 --- a/pkg/synccompactor/pebble/merge.go +++ b/pkg/synccompactor/pebble/merge.go @@ -18,6 +18,23 @@ type SourceSync struct { SyncID string } +type mergeOptions struct { + collectGrantEntitlementIDs bool +} + +// MergeOption enables optional MergeInto behavior without breaking callers +// that use the original four-argument API. +type MergeOption func(*mergeOptions) + +// WithGrantEntitlementIDs records the entitlement IDs of grant records that +// MergeInto actually writes. The incremental expander uses these IDs as its +// changed-node seeds. +func WithGrantEntitlementIDs() MergeOption { + return func(opts *mergeOptions) { + opts.collectGrantEntitlementIDs = true + } +} + // FoldStats reports what a MergeInto call overrode in the destination // keyspace. DeadBytes is the exact raw size (keys + values) of the // incumbent records — and their derived index keys — that the fold @@ -157,8 +174,14 @@ func (s *FoldStats) bumpReplaced(bucket string, n int64) { // (Engine.BuildGrantDigests rebuilds both keyspaces atomically from // scratch, so no separate drop is needed even then — see // compactPebbleFold). -func MergeInto(ctx context.Context, dest *enginepkg.Engine, sources []SourceSync, destSyncID string, collectGrantEntitlementIDs bool) (FoldStats, error) { +func MergeInto(ctx context.Context, dest *enginepkg.Engine, sources []SourceSync, destSyncID string, options ...MergeOption) (FoldStats, error) { var stats FoldStats + opts := mergeOptions{} + for _, option := range options { + if option != nil { + option(&opts) + } + } if dest == nil { return stats, errors.New("synccompactor/pebble.MergeInto: dest engine is nil") } @@ -180,7 +203,7 @@ func MergeInto(ctx context.Context, dest *enginepkg.Engine, sources []SourceSync if s.Engine == nil || s.SyncID == "" { continue } - srcStats, err := mergeOneSource(ctx, dest, s, destSyncID, collectGrantEntitlementIDs) + srcStats, err := mergeOneSource(ctx, dest, s, destSyncID, opts.collectGrantEntitlementIDs) stats.Add(srcStats) if err != nil { return stats, fmt.Errorf("merge source %s: %w", s.SyncID, err) diff --git a/pkg/synccompactor/pebble/merge_test.go b/pkg/synccompactor/pebble/merge_test.go index 16b806d3c..aeca9de3e 100644 --- a/pkg/synccompactor/pebble/merge_test.go +++ b/pkg/synccompactor/pebble/merge_test.go @@ -55,7 +55,7 @@ func TestMergeIntoUnionNewerWins(t *testing.T) { // No SetCurrentSync here: MergeInto must bind the dest engine to // destSyncID itself (record values carry no sync_id, so a stale // binding would silently write into the wrong sync's keyspace). - stats, err := MergeInto(ctx, dst, []SourceSync{{Engine: src1, SyncID: syncA}, {Engine: src2, SyncID: syncB}}, destSync, false) + stats, err := MergeInto(ctx, dst, []SourceSync{{Engine: src1, SyncID: syncA}, {Engine: src2, SyncID: syncB}}, destSync) require.NoError(t, err, "MergeInto") // src2's g-shared@newer overrode src1's incumbent — exactly one // override, and its dead bytes must cover at least the incumbent's @@ -107,7 +107,7 @@ func TestMergeIntoGrantWritesTracksActualChanges(t *testing.T) { require.NoError(t, src.PutGrantRecords(ctx, grantAt(syncID, "g1", at))) // First merge: a genuinely new grant is admitted. - stats, err := MergeInto(ctx, dst, []SourceSync{{Engine: src, SyncID: syncID}}, destSync, false) + stats, err := MergeInto(ctx, dst, []SourceSync{{Engine: src, SyncID: syncID}}, destSync) require.NoError(t, err) require.EqualValues(t, 1, stats.GrantWrites, "first merge admits one new grant") @@ -115,7 +115,7 @@ func TestMergeIntoGrantWritesTracksActualChanges(t *testing.T) { // record is now byte-identical to the incumbent, which // mergeBucketRawIfNewer treats as a true no-op — zero grant writes, // even though a grant record was iterated and compared. - stats2, err := MergeInto(ctx, dst, []SourceSync{{Engine: src, SyncID: syncID}}, destSync, false) + stats2, err := MergeInto(ctx, dst, []SourceSync{{Engine: src, SyncID: syncID}}, destSync) require.NoError(t, err) require.Zero(t, stats2.GrantWrites, "resubmitting an identical grant must not count as a write") @@ -123,7 +123,7 @@ func TestMergeIntoGrantWritesTracksActualChanges(t *testing.T) { empty, _ := newEngine(t, "gw-empty-src") emptySyncID := ksuid.New().String() require.NoError(t, empty.SetCurrentSync(ctx, emptySyncID)) - stats3, err := MergeInto(ctx, dst, []SourceSync{{Engine: empty, SyncID: emptySyncID}}, destSync, false) + stats3, err := MergeInto(ctx, dst, []SourceSync{{Engine: empty, SyncID: emptySyncID}}, destSync) require.NoError(t, err) require.Zero(t, stats3.GrantWrites, "a source with no grants contributes zero grant writes") @@ -132,7 +132,7 @@ func TestMergeIntoGrantWritesTracksActualChanges(t *testing.T) { src2, _ := newEngine(t, "gw-src2") require.NoError(t, src2.SetCurrentSync(ctx, syncID)) require.NoError(t, src2.PutGrantRecords(ctx, grantAt(syncID, "g1", newer))) - stats4, err := MergeInto(ctx, dst, []SourceSync{{Engine: src2, SyncID: syncID}}, destSync, false) + stats4, err := MergeInto(ctx, dst, []SourceSync{{Engine: src2, SyncID: syncID}}, destSync) require.NoError(t, err) require.EqualValues(t, 1, stats4.GrantWrites, "a strictly-newer override counts as a grant write") } @@ -163,7 +163,7 @@ func TestMergeIntoTieKeepsIncumbent(t *testing.T) { require.NoError(t, src2.PutGrantRecords(ctx, g2)) // src1 applied first → incumbent wins the tie. MergeInto binds the // dest sync itself. - stats, err := MergeInto(ctx, dst, []SourceSync{{Engine: src1, SyncID: syncA}, {Engine: src2, SyncID: syncB}}, destSync, false) + stats, err := MergeInto(ctx, dst, []SourceSync{{Engine: src1, SyncID: syncA}, {Engine: src2, SyncID: syncB}}, destSync) require.NoError(t, err) // A tie keeps the incumbent — nothing is overridden, no dead bytes. require.Zero(t, stats.OverriddenRecords, "FoldStats.OverriddenRecords (tie keeps incumbent)") @@ -247,7 +247,7 @@ func TestMergeIntoBaseNewerKeepsBaseAllTypes(t *testing.T) { dest, destSync := seedBase(t, ctx, "base-newer-dest", baseSet) src := buildEngineSource(t, ctx, "older-src", olderSet) - stats, err := MergeInto(ctx, dest, []SourceSync{src}, destSync, false) + stats, err := MergeInto(ctx, dest, []SourceSync{src}, destSync) require.NoError(t, err) // Nothing regressed: the base (newer) version survives for every type. @@ -273,7 +273,7 @@ func TestMergeIntoNewerPartialOverridesBaseAllTypes(t *testing.T) { dest, destSync := seedBase(t, ctx, "base-older-dest", baseSet) src := buildEngineSource(t, ctx, "newer-src", newerSet) - stats, err := MergeInto(ctx, dest, []SourceSync{src}, destSync, false) + stats, err := MergeInto(ctx, dest, []SourceSync{src}, destSync) require.NoError(t, err) // The newer partial wins for every type. @@ -340,7 +340,7 @@ func TestMergeIntoDeadBytesExactCount(t *testing.T) { }.Build() src := buildEngineSource(t, ctx, "deadbytes-src", recordSet{gs: []*v3.GrantRecord{override}}) - stats, err := MergeInto(ctx, dest, []SourceSync{src}, destSync, false) + stats, err := MergeInto(ctx, dest, []SourceSync{src}, destSync) require.NoError(t, err) require.Equal(t, int64(1), stats.OverriddenRecords) require.Equal(t, wantDead, stats.DeadBytes, @@ -386,7 +386,7 @@ func TestMergeIntoCollectsGrantEntitlementIDs(t *testing.T) { }, }) - stats, err := MergeInto(ctx, dest, []SourceSync{src}, destSync, true) + stats, err := MergeInto(ctx, dest, []SourceSync{src}, destSync, WithGrantEntitlementIDs()) require.NoError(t, err) got := make([]string, 0, len(stats.GrantEntitlementIDs)) @@ -405,7 +405,7 @@ func TestMergeIntoSkipsGrantEntitlementIDsWhenDisabled(t *testing.T) { gs: []*v3.GrantRecord{grantEnt("g-new", "ent-added", "dave", now)}, }) - stats, err := MergeInto(ctx, dest, []SourceSync{src}, destSync, false) + stats, err := MergeInto(ctx, dest, []SourceSync{src}, destSync) require.NoError(t, err) require.Equal(t, int64(1), stats.GrantWrites) require.Nil(t, stats.GrantEntitlementIDs, diff --git a/pkg/synccompactor/pebble/winner_equivalence_test.go b/pkg/synccompactor/pebble/winner_equivalence_test.go index d2ad34d65..5f41968f0 100644 --- a/pkg/synccompactor/pebble/winner_equivalence_test.go +++ b/pkg/synccompactor/pebble/winner_equivalence_test.go @@ -263,7 +263,7 @@ func TestPebbleStrategiesAgreeOnWinners(t *testing.T) { for i, rs := range srcSets { sources[i] = buildEngineSource(t, ctx, fmt.Sprintf("fold-src%d", i), rs) } - _, err := MergeInto(ctx, dest, sources, destSyncID, false) + _, err := MergeInto(ctx, dest, sources, destSyncID) return err }) require.Equal(t, want, got) From 273f89fbabca3b20a53c0d58b843f38b4fd1505d Mon Sep 17 00:00:00 2001 From: manojacs Date: Wed, 12 Aug 2026 06:33:52 +0000 Subject: [PATCH 12/15] fix: share the full expander's contribution guard; gate graph preservation to Pebble MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Incremental expansion now routes every candidate source grant through grantContributesOverEdge — the same predicate the full expander uses — so a malformed grant with a nil principal (or nil principal id) is skipped instead of collapsing onto a shared contribution key and aborting the attempt into a full-expansion fallback. WithPreserveEntitlementGraph is now applied only on Pebble outputs: other engines always decline incremental expansion and have no graph sidecar, so preserving the graph there is pure wasted work. Co-authored-by: c1-squire-dev[bot] --- pkg/sync/expand/incremental.go | 9 ++-- pkg/sync/expand/incremental_test.go | 46 +++++++++++++++++++ pkg/synccompactor/compactor.go | 6 ++- .../incremental_expansion_test.go | 16 +++++++ 4 files changed, 72 insertions(+), 5 deletions(-) diff --git a/pkg/sync/expand/incremental.go b/pkg/sync/expand/incremental.go index f85531095..8295e2799 100644 --- a/pkg/sync/expand/incremental.go +++ b/pkg/sync/expand/incremental.go @@ -411,13 +411,16 @@ func (ie *IncrementalExpander) recomputeDestination(ctx context.Context, nodeID continue } perGrantErr := ie.forEachGrant(ctx, sourceEnt, edge.ResourceTypeIDs, func(sourceGrant *v2.Grant) error { + // Shared definition of "contributes" with the full expander: + // rejects nil-principal grants, off-type principals, and + // non-direct grants over shallow edges. + if !grantContributesOverEdge(sourceGrant, sourceEntitlementID, edge) { + return nil + } // Directness is relative to the source entitlement (matches the // full expander): a plain direct grant or one whose sources map // records this entitlement counts as direct. isSourceDirect := isGrantDirectOnEntitlement(sourceGrant, sourceEntitlementID) - if edge.IsShallow && !isSourceDirect { - return nil - } principal := sourceGrant.GetPrincipal() pid := principal.GetId() key := pid.GetResourceType() + "\x00" + pid.GetResource() diff --git a/pkg/sync/expand/incremental_test.go b/pkg/sync/expand/incremental_test.go index 33f2968b1..23932987d 100644 --- a/pkg/sync/expand/incremental_test.go +++ b/pkg/sync/expand/incremental_test.go @@ -407,6 +407,52 @@ func TestIncremental_DenseAffectedGraphDeclinesBeforeWrites(t *testing.T) { require.ErrorIs(t, err, ErrIncrementalDenseChangeDecline) } +// TestIncremental_MalformedNilPrincipalGrantSkipped: a stored grant with no +// principal, or a principal without a resource id, must be skipped by the +// incremental walk exactly as grantContributesOverEdge skips it on the full +// path — not collapse onto a shared "\x00" contribution key (which could merge +// into an unrelated malformed destination grant) or abort the attempt in +// newExpandedGrantWithSources. +func TestIncremental_MalformedNilPrincipalGrantSkipped(t *testing.T) { + ctx := context.Background() + + seedStore := func() *MockExpanderStore { + store := NewMockExpanderStore() + store.AddEntitlement(makeEntitlement("eng:member", makeResource("group", "eng:member"))) + store.AddEntitlement(makeEntitlement("github:access", makeResource("app", "github"))) + store.AddGrant(directGrant("eng:member", makeResource("user", "alice"))) + // Malformed rows on the source entitlement: one grant with no + // principal at all, one whose principal has no resource id. + srcEnt := makeEntitlement("eng:member", makeResource("group", "eng:member")) + store.AddGrant(makeGrant("broken:nil-principal", srcEnt, nil)) + store.AddGrant(makeGrant("broken:nil-principal-id", srcEnt, v2.Resource_builder{}.Build())) + return store + } + + // Full-expansion oracle over the final rule set. + fullStore := seedStore() + fullGraph := NewEntitlementGraph(ctx) + fullGraph.AddEntitlementID("eng:member") + fullGraph.AddEntitlementID("github:access") + require.NoError(t, fullGraph.AddEdge(ctx, "eng:member", "github:access", false, nil)) + require.NoError(t, NewExpander(fullStore, fullGraph).Run(ctx)) + + // Incremental: expanded base without the edge, then the edge arrives. + incStore := seedStore() + incGraph := NewEntitlementGraph(ctx) + incGraph.AddEntitlementID("eng:member") + require.NoError(t, NewExpander(incStore, incGraph).Run(ctx)) + + res, err := NewIncrementalExpander(incStore, incGraph). + ExpandChanges(ctx, []NewEdge{{SourceEntitlementID: "eng:member", DestEntitlementID: "github:access"}}, nil) + require.NoError(t, err, "malformed grants must be skipped, not abort the incremental attempt") + require.Equal(t, 1, res.GrantsWritten, "only alice contributes over the new edge") + + require.Contains(t, principalsOn(t, ctx, incStore, "github:access"), "alice") + require.Equal(t, snapshotStoreGrants(fullStore), snapshotStoreGrants(incStore), + "incremental must match the full-expansion oracle in the presence of malformed grants") +} + func itoa(i int) string { if i == 0 { return "0" diff --git a/pkg/synccompactor/compactor.go b/pkg/synccompactor/compactor.go index 67623bb14..a5f703843 100644 --- a/pkg/synccompactor/compactor.go +++ b/pkg/synccompactor/compactor.go @@ -1307,8 +1307,10 @@ func (c *Compactor) expandGrants(ctx context.Context, newSyncId string, compacti // Keep the artifact's graph sidecar coherent with this full expansion: // opted-in compactions preserve a fresh graph (so the incremental chain // heals after a fallback); otherwise drop any sidecar inherited from a - // fold-copied base. - if c.incrementalExpansion { + // fold-copied base. Pebble-only: incremental expansion declines on other + // engines, and without a sidecar the preserved graph would only bloat + // the final sync token. + if c.incrementalExpansion && c.resolvedEngine() == c1zstore.EnginePebble { syncOpts = append(syncOpts, sync.WithPreserveEntitlementGraph()) } else if gs, ok := c.compactedC1z.(sync.EntitlementGraphStore); ok { if err := gs.DeleteEntitlementGraphBlob(ctx); err != nil { diff --git a/pkg/synccompactor/incremental_expansion_test.go b/pkg/synccompactor/incremental_expansion_test.go index 44fd6799d..cfc7944a8 100644 --- a/pkg/synccompactor/incremental_expansion_test.go +++ b/pkg/synccompactor/incremental_expansion_test.go @@ -896,6 +896,22 @@ func TestCompactor_IncrementalDegradesGracefullyOnSQLite(t *testing.T) { hasGrant(t, grants, "ent-b|user|sam") hasGrant(t, grants, "ent-c|user|sam") hasGrant(t, grants, "ent-c|user|mandy") + + // SQLite has no graph sidecar, so a preserved graph could never be read + // back — the only place it could land is the final sync token, as + // unreadable bloat. Pin that the final token carries no graph (enforced + // twice over: graph preservation is Pebble-gated in expandGrants, and + // state.Marshal drops the graph from tokens by default). + store, err := dotc1z.NewStore(ctx, out.FilePath, dotc1z.WithReadOnly(true), dotc1z.WithTmpDir(t.TempDir())) + require.NoError(t, err) + defer store.Close(ctx) + run, err := store.SyncMeta().LatestFinishedSyncOfAnyType(ctx) + require.NoError(t, err) + if run.SyncToken != "" { + tokenGraph, err := sdksync.GraphFromToken(run.SyncToken) + require.NoError(t, err) + require.Nil(t, tokenGraph, "SQLite final sync token must not carry an entitlement graph") + } } // TestCompactor_IncrementalNewMemberFoldCollectsChangedEnts: fold mode From 0cd1d05a053ab59d6b831bd4dfdd799d639bbf41 Mon Sep 17 00:00:00 2001 From: manojacs Date: Wed, 12 Aug 2026 07:37:02 +0000 Subject: [PATCH 13/15] fix: address second-round incremental expansion review findings - loadIncrementalBaseGraph: guard the nil run LatestFinishedSyncOfAnyType returns for an artifact with no finished sync (both engines return nil, nil), declining to full expansion instead of panicking on run.ID. - recomputeDestination: merge a principal's contribution into every existing grant row sharing the principal key, matching the full expander's per-key group merge; previously the first row consumed the contribution and later duplicate rows kept stale sources. Co-authored-by: c1-squire-dev[bot] --- pkg/sync/expand/incremental.go | 13 +++++- pkg/sync/expand/incremental_test.go | 44 +++++++++++++++++++ pkg/synccompactor/compactor.go | 6 +++ .../incremental_expansion_test.go | 25 +++++++++++ 4 files changed, 86 insertions(+), 2 deletions(-) diff --git a/pkg/sync/expand/incremental.go b/pkg/sync/expand/incremental.go index 8295e2799..f5db117c7 100644 --- a/pkg/sync/expand/incremental.go +++ b/pkg/sync/expand/incremental.go @@ -457,7 +457,13 @@ func (ie *IncrementalExpander) recomputeDestination(ctx context.Context, nodeID // 2. Merge contributions into the destination's existing grants (union the // sources map, upgrade direct-ness), streaming one page at a time. Only a - // grant that actually changed is rewritten; matched principals leave contrib. + // grant that actually changed is rewritten. A principal can hold several + // grant rows on one entitlement (connector-authored IDs are arbitrary), and + // the full expander merges the contribution into every row sharing the + // principal key — so record matches in a side set instead of consuming the + // contribution on the first row, and drop them from contrib only after the + // whole destination has streamed. + matched := make(map[string]struct{}) mergeErr := ie.forEachGrant(ctx, destEnt, nil, func(g *v2.Grant) error { pid := g.GetPrincipal().GetId() key := pid.GetResourceType() + "\x00" + pid.GetResource() @@ -465,7 +471,7 @@ func (ie *IncrementalExpander) recomputeDestination(ctx context.Context, nodeID if pc == nil { return nil } - delete(contrib, key) + matched[key] = struct{}{} updated := mergeContributionIntoExistingGrant(g, destEntitlementID, pc.sources) if updated == nil { return nil // already had these sources — no write @@ -479,6 +485,9 @@ func (ie *IncrementalExpander) recomputeDestination(ctx context.Context, nodeID if mergeErr != nil { return 0, mergeErr } + for key := range matched { + delete(contrib, key) + } // 3. Whatever is left in contrib are brand-new principals. Sort for // deterministic (byte-stable) output. diff --git a/pkg/sync/expand/incremental_test.go b/pkg/sync/expand/incremental_test.go index 23932987d..b4ed2e8c3 100644 --- a/pkg/sync/expand/incremental_test.go +++ b/pkg/sync/expand/incremental_test.go @@ -453,6 +453,50 @@ func TestIncremental_MalformedNilPrincipalGrantSkipped(t *testing.T) { "incremental must match the full-expansion oracle in the presence of malformed grants") } +// TestIncremental_DuplicatePrincipalRowsAllMerged: a destination entitlement +// can hold several grant rows for the same principal (connector-authored grant +// IDs are arbitrary). The full expander merges a contribution into every row +// sharing the principal key (topological_merge_streaming.go); the incremental +// merge must do the same, not consume the contribution on the first row and +// leave later rows with stale sources. +func TestIncremental_DuplicatePrincipalRowsAllMerged(t *testing.T) { + ctx := context.Background() + + alice := makeResource("user", "alice") + seedStore := func() *MockExpanderStore { + store := NewMockExpanderStore() + store.AddEntitlement(makeEntitlement("eng:member", makeResource("group", "eng:member"))) + store.AddEntitlement(makeEntitlement("github:access", makeResource("app", "github"))) + store.AddGrant(directGrant("eng:member", alice)) + // Two pre-existing rows for alice on the destination, distinct IDs. + destEnt := makeEntitlement("github:access", makeResource("app", "github")) + store.AddGrant(makeGrant("dup-row-1", destEnt, alice)) + store.AddGrant(makeGrant("dup-row-2", destEnt, alice)) + return store + } + + // Full-expansion oracle over the final rule set. + fullStore := seedStore() + fullGraph := NewEntitlementGraph(ctx) + fullGraph.AddEntitlementID("eng:member") + fullGraph.AddEntitlementID("github:access") + require.NoError(t, fullGraph.AddEdge(ctx, "eng:member", "github:access", false, nil)) + require.NoError(t, NewExpander(fullStore, fullGraph).Run(ctx)) + + // Incremental: expanded base without the edge, then the edge arrives. + incStore := seedStore() + incGraph := NewEntitlementGraph(ctx) + incGraph.AddEntitlementID("eng:member") + require.NoError(t, NewExpander(incStore, incGraph).Run(ctx)) + + _, err := NewIncrementalExpander(incStore, incGraph). + ExpandChanges(ctx, []NewEdge{{SourceEntitlementID: "eng:member", DestEntitlementID: "github:access"}}, nil) + require.NoError(t, err) + + require.Equal(t, snapshotStoreGrants(fullStore), snapshotStoreGrants(incStore), + "every duplicate-principal row must carry the merged sources, matching full expansion") +} + func itoa(i int) string { if i == 0 { return "0" diff --git a/pkg/synccompactor/compactor.go b/pkg/synccompactor/compactor.go index a5f703843..14803a629 100644 --- a/pkg/synccompactor/compactor.go +++ b/pkg/synccompactor/compactor.go @@ -1372,6 +1372,12 @@ func (c *Compactor) loadIncrementalBaseGraph(ctx context.Context) (*expand.Entit _ = store.Close(ctx) return nil, fmt.Errorf("incremental expansion: load base verification: %w", runErr) } + // Both engines return (nil, nil) when the artifact holds no finished sync + // (e.g. an interrupted collection): decline to full expansion, don't panic. + if run == nil { + _ = store.Close(ctx) + return nil, fmt.Errorf("incremental expansion: base has no finished sync") + } if run.ID != c.entries[0].SyncID || !run.IsVerified() || run.Generation != sync.IngestInvariantGeneration { diff --git a/pkg/synccompactor/incremental_expansion_test.go b/pkg/synccompactor/incremental_expansion_test.go index cfc7944a8..7829fc347 100644 --- a/pkg/synccompactor/incremental_expansion_test.go +++ b/pkg/synccompactor/incremental_expansion_test.go @@ -914,6 +914,31 @@ func TestCompactor_IncrementalDegradesGracefullyOnSQLite(t *testing.T) { } } +// TestCompactor_IncrementalBaseWithNoFinishedSyncDeclines: a base c1z whose +// sync never ended (interrupted collection) makes LatestFinishedSyncOfAnyType +// return (nil, nil) on both engines. The base-graph loader must return an +// error — declining to full expansion — not panic on run.ID. +func TestCompactor_IncrementalBaseWithNoFinishedSyncDeclines(t *testing.T) { + ctx := context.Background() + + basePath := filepath.Join(t.TempDir(), "unfinished.c1z") + store, err := dotc1z.NewStore(ctx, basePath, dotc1z.WithEngine(c1zstore.EnginePebble)) + require.NoError(t, err) + syncID, err := store.StartNewSync(ctx, connectorstore.SyncTypeFull, "") + require.NoError(t, err) + // No EndSync: the artifact holds no finished sync run. + require.NoError(t, store.Close(ctx)) + + c := &Compactor{ + entries: []*CompactableSync{{FilePath: basePath, SyncID: syncID}}, + tmpDir: t.TempDir(), + } + graph, err := c.loadIncrementalBaseGraph(ctx) + require.Error(t, err, "an unfinished base must decline, not panic") + require.ErrorContains(t, err, "no finished sync") + require.Nil(t, graph) +} + // TestCompactor_IncrementalNewMemberFoldCollectsChangedEnts: fold mode // collects the changed-entitlement set during the merge (no re-read) and // still matches full expansion. From 8d52ffc91349cd3808b107d45dccb6d311c29c16 Mon Sep 17 00:00:00 2001 From: manojacs Date: Wed, 12 Aug 2026 15:25:52 +0000 Subject: [PATCH 14/15] fix(compat-harness): guard nil run from LatestFinishedSyncOfAnyType Same nil-return class as loadIncrementalBaseGraph: an artifact with no finished sync returns (nil, nil), and the three graph-mode helpers then panicked on run.ID. Return a named error instead so a broken harness step stays diagnosable. Co-authored-by: c1-squire-dev[bot] --- cmd/baton-compat-harness/graph_modes_new.go | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/cmd/baton-compat-harness/graph_modes_new.go b/cmd/baton-compat-harness/graph_modes_new.go index 4a14bddec..c899e2a46 100644 --- a/cmd/baton-compat-harness/graph_modes_new.go +++ b/cmd/baton-compat-harness/graph_modes_new.go @@ -130,6 +130,10 @@ func graphCompatSeed(ctx context.Context, path string) (compatResult, error) { _ = store.Close(ctx) return compatResult{}, err } + if run == nil { + _ = store.Close(ctx) + return compatResult{}, fmt.Errorf("candidate artifact has no finished sync") + } graph := expand.NewEntitlementGraph(ctx) for _, entitlement := range connector.entsByRes { graph.AddEntitlementID(entitlement.GetId()) @@ -178,6 +182,10 @@ func graphCompatInspect(ctx context.Context, path string) (compatResult, error) _ = store.Close(ctx) return result, err } + if run == nil { + _ = store.Close(ctx) + return result, fmt.Errorf("inspected artifact has no finished sync") + } if graphStore, ok := store.(sdksync.EntitlementGraphStore); ok { data, graphErr := graphStore.GetEntitlementGraphBlob(ctx) if graphErr != nil { @@ -215,6 +223,9 @@ func graphCompatCompact(ctx context.Context, inputPath, outPath string, incremen if err != nil { return compatResult{}, err } + if run == nil { + return compatResult{}, fmt.Errorf("compaction input has no finished sync") + } var logBytes bytes.Buffer core := zapcore.NewCore(zapcore.NewJSONEncoder(zap.NewProductionEncoderConfig()), From 3da22557ac19c15919f07cbe8dc3b23e6215f733 Mon Sep 17 00:00:00 2001 From: manojacs Date: Wed, 12 Aug 2026 18:41:35 +0000 Subject: [PATCH 15/15] fix(compat-harness): guard the fourth nil-run site in graphCompatFullCompact Same LatestFinishedSyncOfAnyType (nil, nil) class as the three sites guarded in graph_modes_new.go; this was the remaining unguarded dereference. Co-authored-by: c1-squire-dev[bot] --- cmd/baton-compat-harness/main.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/cmd/baton-compat-harness/main.go b/cmd/baton-compat-harness/main.go index 4a8ad98f7..8f4deaf8d 100644 --- a/cmd/baton-compat-harness/main.go +++ b/cmd/baton-compat-harness/main.go @@ -565,6 +565,9 @@ func graphCompatFullCompact(ctx context.Context, inputPath, outPath string) (com if err != nil { return compatResult{}, err } + if run == nil { + return compatResult{}, fmt.Errorf("input artifact has no finished sync") + } empty, err := graphCompatEmptyPartial(ctx, filepath.Dir(outPath)) if err != nil { return compatResult{}, err