From aef6dd79f33a44cdbc5836a31655bf91a5d71e65 Mon Sep 17 00:00:00 2001 From: strantalis Date: Fri, 4 Sep 2026 13:18:12 -0400 Subject: [PATCH 01/10] test(authz): add multi-resource scale regression coverage Signed-off-by: strantalis --- tests-bdd/cukes/steps_authorization.go | 38 ++++++ tests-bdd/cukes/steps_subjectmappings.go | 121 +++++++++++++++++- ...ion-v2-subject-mapping-performance.feature | 35 +++++ 3 files changed, 193 insertions(+), 1 deletion(-) create mode 100644 tests-bdd/features/authorization-v2-subject-mapping-performance.feature diff --git a/tests-bdd/cukes/steps_authorization.go b/tests-bdd/cukes/steps_authorization.go index ed8541d443..59062ccb8a 100644 --- a/tests-bdd/cukes/steps_authorization.go +++ b/tests-bdd/cukes/steps_authorization.go @@ -4,9 +4,11 @@ import ( "context" "errors" "fmt" + "log/slog" "net" "strconv" "strings" + "time" "github.com/cucumber/godog" authzV2 "github.com/opentdf/platform/protocol/go/authorization/v2" @@ -214,6 +216,41 @@ func (s *AuthorizationServiceStepDefinitions) iSendAMultiResourceDecisionRequest return ctx, nil } +func (s *AuthorizationServiceStepDefinitions) iSendAMultiResourceDecisionRequestWithin(ctx context.Context, entityChainID, action, maximumDuration string, tbl *godog.Table) (context.Context, error) { + limit, err := time.ParseDuration(maximumDuration) + if err != nil { + return ctx, fmt.Errorf("parse maximum decision duration %q: %w", maximumDuration, err) + } + if limit <= 0 { + return ctx, fmt.Errorf("maximum decision duration must be positive, got %s", limit) + } + + requestCtx, cancel := context.WithTimeout(ctx, limit) + defer cancel() + started := time.Now() + _, err = s.iSendAMultiResourceDecisionRequestForEntityChainForActionOnResources(requestCtx, entityChainID, action, tbl) + elapsed := time.Since(started) + + scenarioContext := GetPlatformScenarioContext(ctx) + scenarioContext.TestSuiteContext.Logger.Info( + "authorization v2 multi-resource performance", + slog.Duration("duration", elapsed), + slog.Duration("maximum_duration", limit), + slog.Int("resource_count", len(tbl.Rows)-1), + ) + if err != nil { + return ctx, err + } + if requestErr := scenarioContext.GetError(); requestErr != nil { + return ctx, fmt.Errorf("multi-resource decision failed after %s: %w", elapsed, requestErr) + } + if elapsed > limit { + return ctx, fmt.Errorf("multi-resource decision took %s, exceeding the %s limit", elapsed, limit) + } + + return ctx, nil +} + func (s *AuthorizationServiceStepDefinitions) iSendAMultiResourceDecisionRequestForEntityChainForActionOnResourcesWithNoFulfillableObligations(ctx context.Context, entityChainID string, action string, tbl *godog.Table) (context.Context, error) { return s.iSendAMultiResourceDecisionRequestForEntityChainForActionOnResourcesWithFulfillableObligations(ctx, entityChainID, action, "[]", tbl) } @@ -556,6 +593,7 @@ func RegisterAuthorizationStepDefinitions(ctx *godog.ScenarioContext) { ctx.Step(`^I send a decision request for entity chain "([^"]*)" for "([^"]*)" action on resource "([^"]*)" with fulfillable obligations "([^"]*)"$`, stepDefinitions.iSendADecisionRequestForEntityChainForActionOnResourceWithFulfillableObligations) ctx.Step(`^I send a decision request for entity chain "([^"]*)" for "([^"]*)" action on resource "([^"]*)" with no fulfillable obligations$`, stepDefinitions.iSendADecisionRequestForEntityChainForActionOnResourceWithNoFulfillableObligations) ctx.Step(`^I send a multi-resource decision request for entity chain "([^"]*)" for "([^"]*)" action on resources:$`, stepDefinitions.iSendAMultiResourceDecisionRequestForEntityChainForActionOnResources) + ctx.Step(`^I send a multi-resource decision request for entity chain "([^"]*)" for "([^"]*)" action on resources within "([^"]*)":$`, stepDefinitions.iSendAMultiResourceDecisionRequestWithin) ctx.Step(`^I send a multi-resource decision request for entity chain "([^"]*)" for "([^"]*)" action on resources with no fulfillable obligations:$`, stepDefinitions.iSendAMultiResourceDecisionRequestForEntityChainForActionOnResourcesWithNoFulfillableObligations) ctx.Step(`^I send a multi-resource decision request for entity chain "([^"]*)" for "([^"]*)" action on resources with fulfillable obligations "([^"]*)":$`, stepDefinitions.iSendAMultiResourceDecisionRequestForEntityChainForActionOnResourcesWithFulfillableObligations) ctx.Step(`^I should get a "([^"]*)" decision response$`, stepDefinitions.iShouldGetADecisionResponse) diff --git a/tests-bdd/cukes/steps_subjectmappings.go b/tests-bdd/cukes/steps_subjectmappings.go index 76d7c7a108..c98b804bd5 100644 --- a/tests-bdd/cukes/steps_subjectmappings.go +++ b/tests-bdd/cukes/steps_subjectmappings.go @@ -4,15 +4,21 @@ import ( "context" "errors" "fmt" + "log/slog" "strings" "github.com/cucumber/godog" + "github.com/google/uuid" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgxpool" "github.com/opentdf/platform/protocol/go/policy" "github.com/opentdf/platform/protocol/go/policy/subjectmapping" ) type SubjectMappingsStepDefinitions struct{} +const policyDatabaseSchema = "otdf" + func (s *SubjectMappingsStepDefinitions) iSendARequestToCreateSubjectMapping(ctx context.Context, tbl *godog.Table) (context.Context, error) { scenarioContext := GetPlatformScenarioContext(ctx) scenarioContext.ClearError() @@ -29,7 +35,7 @@ func (s *SubjectMappingsStepDefinitions) iSendARequestToCreateSubjectMapping(ctx cellIndexMap[ci] = c.Value } else { switch cellIndexMap[ci] { - case "namespace_id": + case namespaceIDKey: nsID, ok := scenarioContext.GetObject(strings.TrimSpace(c.Value)).(string) if !ok { return ctx, fmt.Errorf("unable to get namespace id for %s", c.Value) @@ -220,6 +226,118 @@ func (s *SubjectMappingsStepDefinitions) iSendARequestToCreateSubjectMappingForE return ctx, nil } +// seedSubjectAndResourceMappingsAtScale uses the scenario database for fixture setup so the +// end-to-end test spends its measured time in authorization, not in thousands of setup requests. +// The decision itself still goes through the public v2 API and the running platform container. +func (s *SubjectMappingsStepDefinitions) seedSubjectAndResourceMappingsAtScale(ctx context.Context, expectedSubjectMappings int, attributeRef, conditionSetRef, namespaceRef, action string, resourceMappingCount int) (context.Context, error) { + scenarioContext := GetPlatformScenarioContext(ctx) + scenarioContext.ClearError() + + attr, ok := scenarioContext.GetObject(strings.TrimSpace(attributeRef)).(*policy.Attribute) + if !ok { + return ctx, fmt.Errorf("unable to get attribute for %s", attributeRef) + } + if len(attr.GetValues()) != expectedSubjectMappings { + return ctx, fmt.Errorf("attribute %s has %d values, expected %d", attributeRef, len(attr.GetValues()), expectedSubjectMappings) + } + if resourceMappingCount < 0 || resourceMappingCount > len(attr.GetValues()) { + return ctx, fmt.Errorf("resource mapping count %d must be between 0 and %d", resourceMappingCount, len(attr.GetValues())) + } + scs, ok := scenarioContext.GetObject(strings.TrimSpace(conditionSetRef)).(*policy.SubjectConditionSet) + if !ok { + return ctx, fmt.Errorf("unable to get condition set for %s", conditionSetRef) + } + namespaceID, ok := scenarioContext.GetObject(strings.TrimSpace(namespaceRef)).(string) + if !ok { + return ctx, fmt.Errorf("unable to get namespace id for %s", namespaceRef) + } + localPlatformGlue, ok := (*scenarioContext.TestSuiteContext.PlatformGlue).(*LocalDevPlatformGlue) + if !ok { + return ctx, errors.New("failed to load local platform glue") + } + + pgConfig, err := pgxpool.ParseConfig(fmt.Sprintf( + "postgres://postgres:changeme@localhost:%d/%s?sslmode=prefer", + localPlatformGlue.Options.postgresPort, + scenarioContext.ScenarioOptions.DatabaseName, + )) + if err != nil { + return ctx, fmt.Errorf("parse scenario database config: %w", err) + } + pool, err := pgxpool.NewWithConfig(ctx, pgConfig) + if err != nil { + return ctx, fmt.Errorf("connect to scenario database: %w", err) + } + defer pool.Close() + + tx, err := pool.Begin(ctx) + if err != nil { + return ctx, fmt.Errorf("begin scale fixture transaction: %w", err) + } + defer func() { _ = tx.Rollback(ctx) }() + + var actionID string + err = tx.QueryRow(ctx, ` + SELECT id + FROM otdf.actions + WHERE name = $1 AND (namespace_id = $2 OR namespace_id IS NULL) + ORDER BY (namespace_id = $2) DESC + LIMIT 1 + `, strings.ToLower(strings.TrimSpace(action)), namespaceID).Scan(&actionID) + if err != nil { + return ctx, fmt.Errorf("resolve action %q: %w", action, err) + } + + subjectMappingRows := make([][]any, 0, expectedSubjectMappings) + actionRows := make([][]any, 0, expectedSubjectMappings) + resourceMappingRows := make([][]any, 0, resourceMappingCount) + for i, value := range attr.GetValues() { + mappingID := uuid.NewString() + subjectMappingRows = append(subjectMappingRows, []any{mappingID, value.GetId(), scs.GetId(), namespaceID}) + actionRows = append(actionRows, []any{mappingID, actionID}) + if i < resourceMappingCount { + resourceMappingRows = append(resourceMappingRows, []any{ + uuid.NewString(), + value.GetId(), + []string{fmt.Sprintf("resource-%04d", i)}, + namespaceID, + }) + } + } + + inserted, err := tx.CopyFrom(ctx, pgx.Identifier{policyDatabaseSchema, "subject_mappings"}, []string{"id", "attribute_value_id", "subject_condition_set_id", namespaceIDKey}, pgx.CopyFromRows(subjectMappingRows)) + if err != nil { + return ctx, fmt.Errorf("seed subject mappings: %w", err) + } + if inserted != int64(expectedSubjectMappings) { + return ctx, fmt.Errorf("seeded %d subject mappings, expected %d", inserted, expectedSubjectMappings) + } + inserted, err = tx.CopyFrom(ctx, pgx.Identifier{policyDatabaseSchema, "subject_mapping_actions"}, []string{"subject_mapping_id", "action_id"}, pgx.CopyFromRows(actionRows)) + if err != nil { + return ctx, fmt.Errorf("seed subject mapping actions: %w", err) + } + if inserted != int64(expectedSubjectMappings) { + return ctx, fmt.Errorf("seeded %d subject mapping actions, expected %d", inserted, expectedSubjectMappings) + } + inserted, err = tx.CopyFrom(ctx, pgx.Identifier{policyDatabaseSchema, "resource_mappings"}, []string{"id", "attribute_value_id", "terms", namespaceIDKey}, pgx.CopyFromRows(resourceMappingRows)) + if err != nil { + return ctx, fmt.Errorf("seed resource mappings: %w", err) + } + if inserted != int64(resourceMappingCount) { + return ctx, fmt.Errorf("seeded %d resource mappings, expected %d", inserted, resourceMappingCount) + } + if err := tx.Commit(ctx); err != nil { + return ctx, fmt.Errorf("commit scale fixtures: %w", err) + } + + scenarioContext.TestSuiteContext.Logger.Info( + "seeded authorization scale fixture", + slog.Int("subject_mapping_count", expectedSubjectMappings), + slog.Int("resource_mapping_count", resourceMappingCount), + ) + return ctx, nil +} + func RegisterSubjectMappingsStepsDefinitions(ctx *godog.ScenarioContext) { subjectMappingStepDefinitions := &SubjectMappingsStepDefinitions{} ctx.Step(`a condition group referenced as "([^"]*)" with an "([^"]*)" operator with conditions:$`, subjectMappingStepDefinitions.aConditionGroup) @@ -228,4 +346,5 @@ func RegisterSubjectMappingsStepsDefinitions(ctx *godog.ScenarioContext) { ctx.Step(`^I send a request to create a subject condition set referenced as "([^"]*)" in namespace "([^"]*)" containing subject sets "([^"]*)"$`, subjectMappingStepDefinitions.iSendARequestToCreateSubjectConditionSetInNamespace) ctx.Step(`^I send a request to create a subject mapping with:$`, subjectMappingStepDefinitions.iSendARequestToCreateSubjectMapping) ctx.Step(`^I send a request to create a subject mapping for every value of attribute "([^"]*)" using condition set "([^"]*)" with actions "([^"]*)"$`, subjectMappingStepDefinitions.iSendARequestToCreateSubjectMappingForEveryAttributeValue) + ctx.Step(`^the policy database contains (\d+) subject mappings for attribute "([^"]*)" using condition set "([^"]*)" in namespace "([^"]*)" with action "([^"]*)" and (\d+) resource mappings$`, subjectMappingStepDefinitions.seedSubjectAndResourceMappingsAtScale) } diff --git a/tests-bdd/features/authorization-v2-subject-mapping-performance.feature b/tests-bdd/features/authorization-v2-subject-mapping-performance.feature new file mode 100644 index 0000000000..5ef8301a2d --- /dev/null +++ b/tests-bdd/features/authorization-v2-subject-mapping-performance.feature @@ -0,0 +1,35 @@ +@authorization @authz-v2 @performance @scale +Feature: v2 multi-resource decisions at large policy scale + GetDecisionMultiResource must remain responsive when the policy database contains the + subject-mapping and resource-mapping cardinality from the reported regression. Fixture setup is + not timed. The measured operation is one request to the public v2 authorization endpoint. + + Background: + Given a user exists with username "scale-user" and email "scale-user@example.com" and the following attributes: + | name | value | + | department | ["engineering"] | + And an empty local platform + And I submit a request to create a namespace with name "scale.example" and reference id "scale_ns" + And I send a request to create an attribute referenced as "scale_attr" in namespace "scale_ns" named "access-level" with rule "anyOf" and 6011 generated values + Then the response should be successful + And a condition group referenced as "scale_condition" with an "or" operator with conditions: + | selector_value | operator | values | + | .attributes.department[] | in | engineering | + And a subject set referenced as "scale_subject_set" containing the condition groups "scale_condition" + And I send a request to create a subject condition set referenced as "scale_condition_set" in namespace "scale_ns" containing subject sets "scale_subject_set" + Then the response should be successful + And the policy database contains 6011 subject mappings for attribute "scale_attr" using condition set "scale_condition_set" in namespace "scale_ns" with action "read" and 6000 resource mappings + And there is a "user_name" subject entity with value "scale-user" and referenced as "scale-user" + + Scenario: Multi-resource decision completes within five seconds with 6011 subject mappings + When I send a multi-resource decision request for entity chain "scale-user" for "read" action on resources within "5s": + | resource | + | https://scale.example/attr/access-level/value/v0000 | + | https://scale.example/attr/access-level/value/v3005 | + | https://scale.example/attr/access-level/value/v6010 | + Then the response should be successful + And I should get 3 decision responses + And the multi-resource decision should be "PERMIT" + And the decision response for resource "https://scale.example/attr/access-level/value/v0000" should be "PERMIT" + And the decision response for resource "https://scale.example/attr/access-level/value/v3005" should be "PERMIT" + And the decision response for resource "https://scale.example/attr/access-level/value/v6010" should be "PERMIT" From 29d0a161eb182baf02af49375a4b851fd3588a1b Mon Sep 17 00:00:00 2001 From: strantalis Date: Fri, 4 Sep 2026 14:00:49 -0400 Subject: [PATCH 02/10] fix(authz): batch scale fixture value creation Signed-off-by: strantalis --- tests-bdd/cukes/steps_attributes.go | 110 ++++++++++++++++-- ...ion-v2-subject-mapping-performance.feature | 2 +- 2 files changed, 101 insertions(+), 11 deletions(-) diff --git a/tests-bdd/cukes/steps_attributes.go b/tests-bdd/cukes/steps_attributes.go index ba8dae68c0..5585edd228 100644 --- a/tests-bdd/cukes/steps_attributes.go +++ b/tests-bdd/cukes/steps_attributes.go @@ -4,7 +4,10 @@ import ( "context" "errors" "fmt" + "log/slog" "strings" + "sync" + "time" "github.com/cucumber/godog" "github.com/opentdf/platform/protocol/go/policy" @@ -19,6 +22,19 @@ type AttributesStepDefinitions struct { PlatformCukesContext *PlatformTestSuiteContext } +func parseAttributeRule(rule string) (policy.AttributeRuleTypeEnum, error) { + switch strings.TrimSpace(rule) { + case "anyOf": + return policy.AttributeRuleTypeEnum_ATTRIBUTE_RULE_TYPE_ENUM_ANY_OF, nil + case "allOf": + return policy.AttributeRuleTypeEnum_ATTRIBUTE_RULE_TYPE_ENUM_ALL_OF, nil + case "hierarchy": + return policy.AttributeRuleTypeEnum_ATTRIBUTE_RULE_TYPE_ENUM_HIERARCHY, nil + default: + return policy.AttributeRuleTypeEnum_ATTRIBUTE_RULE_TYPE_ENUM_UNSPECIFIED, fmt.Errorf("unknown attribute rule type %s", rule) + } +} + func (s *AttributesStepDefinitions) aAttributeDef(ctx context.Context, _ string, _ string) (context.Context, error) { return ctx, nil } @@ -98,16 +114,9 @@ func (s *AttributesStepDefinitions) iSendARequestToCreateAnAttributeWithGenerate return ctx, fmt.Errorf("unable to get namespace id for %s", namespaceRef) } - var ruleType policy.AttributeRuleTypeEnum - switch strings.TrimSpace(rule) { - case "anyOf": - ruleType = policy.AttributeRuleTypeEnum_ATTRIBUTE_RULE_TYPE_ENUM_ANY_OF - case "allOf": - ruleType = policy.AttributeRuleTypeEnum_ATTRIBUTE_RULE_TYPE_ENUM_ALL_OF - case "hierarchy": - ruleType = policy.AttributeRuleTypeEnum_ATTRIBUTE_RULE_TYPE_ENUM_HIERARCHY - default: - return ctx, fmt.Errorf("unknown attribute rule type %s", rule) + ruleType, err := parseAttributeRule(rule) + if err != nil { + return ctx, err } values := make([]string, 0, valueCount) @@ -128,6 +137,86 @@ func (s *AttributesStepDefinitions) iSendARequestToCreateAnAttributeWithGenerate return ctx, nil } +func (s *AttributesStepDefinitions) iSendARequestToCreateAnAttributeWithBatchedGeneratedValues(ctx context.Context, referenceID, namespaceRef, name, rule string, valueCount, batchSize int) (context.Context, error) { + scenarioContext := GetPlatformScenarioContext(ctx) + scenarioContext.ClearError() + if valueCount < 1 { + return ctx, errors.New("generated value count must be positive") + } + if batchSize < 1 { + return ctx, errors.New("generated value batch size must be positive") + } + + namespaceID, ok := scenarioContext.GetObject(strings.TrimSpace(namespaceRef)).(string) + if !ok { + return ctx, fmt.Errorf("unable to get namespace id for %s", namespaceRef) + } + ruleType, err := parseAttributeRule(rule) + if err != nil { + return ctx, err + } + + created, err := scenarioContext.SDK.Attributes.CreateAttribute(ctx, &attributes.CreateAttributeRequest{ + NamespaceId: namespaceID, + Name: strings.TrimSpace(name), + Rule: ruleType, + }) + if err != nil { + scenarioContext.SetError(err) + return ctx, nil + } + if created.GetAttribute() == nil { + return ctx, errors.New("create attribute returned no attribute") + } + + started := time.Now() + values := make([]*policy.Value, valueCount) + for batchStart := 0; batchStart < valueCount; batchStart += batchSize { + batchEnd := min(batchStart+batchSize, valueCount) + batchCtx, cancel := context.WithCancel(ctx) + errCh := make(chan error, batchEnd-batchStart) + var wg sync.WaitGroup + for i := batchStart; i < batchEnd; i++ { + wg.Add(1) + go func(index int) { + defer wg.Done() + resp, createErr := scenarioContext.SDK.Attributes.CreateAttributeValue(batchCtx, &attributes.CreateAttributeValueRequest{ + AttributeId: created.GetAttribute().GetId(), + Value: fmt.Sprintf("v%04d", index), + }) + if createErr != nil { + errCh <- fmt.Errorf("create generated attribute value v%04d: %w", index, createErr) + cancel() + return + } + if resp.GetValue() == nil { + errCh <- fmt.Errorf("create generated attribute value v%04d returned no value", index) + cancel() + return + } + values[index] = resp.GetValue() + }(i) + } + wg.Wait() + cancel() + close(errCh) + if batchErr, hasBatchErr := <-errCh; hasBatchErr { + scenarioContext.SetError(batchErr) + return ctx, nil + } + } + + created.GetAttribute().Values = values + scenarioContext.RecordObject(strings.TrimSpace(referenceID), created.GetAttribute()) + scenarioContext.TestSuiteContext.Logger.Info( + "created generated attribute values in batches", + slog.Int("value_count", valueCount), + slog.Int("batch_size", batchSize), + slog.Duration("duration", time.Since(started)), + ) + return ctx, nil +} + func RegisterAttributeStepDefinitions(ctx *godog.ScenarioContext, x *PlatformTestSuiteContext) { stepDefinitions := AttributesStepDefinitions{ PlatformCukesContext: x, @@ -135,4 +224,5 @@ func RegisterAttributeStepDefinitions(ctx *godog.ScenarioContext, x *PlatformTes ctx.Step(`^a (anyOf|allOf|hierarchy) attribute definition with values: "([^"]*)"$`, stepDefinitions.aAttributeDef) ctx.Step(`^I send a request to create an attribute with:$`, stepDefinitions.iSendARequestToCreateAnAttributeWith) ctx.Step(`^I send a request to create an attribute referenced as "([^"]*)" in namespace "([^"]*)" named "([^"]*)" with rule "([^"]*)" and (\d+) generated values$`, stepDefinitions.iSendARequestToCreateAnAttributeWithGeneratedValues) + ctx.Step(`^I send a request to create an attribute referenced as "([^"]*)" in namespace "([^"]*)" named "([^"]*)" with rule "([^"]*)" and (\d+) generated values in batches of (\d+)$`, stepDefinitions.iSendARequestToCreateAnAttributeWithBatchedGeneratedValues) } diff --git a/tests-bdd/features/authorization-v2-subject-mapping-performance.feature b/tests-bdd/features/authorization-v2-subject-mapping-performance.feature index 5ef8301a2d..50314fcb82 100644 --- a/tests-bdd/features/authorization-v2-subject-mapping-performance.feature +++ b/tests-bdd/features/authorization-v2-subject-mapping-performance.feature @@ -10,7 +10,7 @@ Feature: v2 multi-resource decisions at large policy scale | department | ["engineering"] | And an empty local platform And I submit a request to create a namespace with name "scale.example" and reference id "scale_ns" - And I send a request to create an attribute referenced as "scale_attr" in namespace "scale_ns" named "access-level" with rule "anyOf" and 6011 generated values + And I send a request to create an attribute referenced as "scale_attr" in namespace "scale_ns" named "access-level" with rule "anyOf" and 6011 generated values in batches of 25 Then the response should be successful And a condition group referenced as "scale_condition" with an "or" operator with conditions: | selector_value | operator | values | From 73ddbca7f7258605c1b02497cc3303a0f04a57a4 Mon Sep 17 00:00:00 2001 From: strantalis Date: Fri, 4 Sep 2026 14:15:32 -0400 Subject: [PATCH 03/10] fix(tests): use policy database schema for scale fixtures Signed-off-by: strantalis --- tests-bdd/cukes/steps_subjectmappings.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests-bdd/cukes/steps_subjectmappings.go b/tests-bdd/cukes/steps_subjectmappings.go index c98b804bd5..1040754376 100644 --- a/tests-bdd/cukes/steps_subjectmappings.go +++ b/tests-bdd/cukes/steps_subjectmappings.go @@ -17,7 +17,7 @@ import ( type SubjectMappingsStepDefinitions struct{} -const policyDatabaseSchema = "otdf" +const policyDatabaseSchema = "otdf_policy" func (s *SubjectMappingsStepDefinitions) iSendARequestToCreateSubjectMapping(ctx context.Context, tbl *godog.Table) (context.Context, error) { scenarioContext := GetPlatformScenarioContext(ctx) From 191986e4eb9bf8383515315923755ed70b6ab5d5 Mon Sep 17 00:00:00 2001 From: strantalis Date: Fri, 4 Sep 2026 16:24:03 -0400 Subject: [PATCH 04/10] test(authz): cover policy page limits at scale Signed-off-by: strantalis --- tests-bdd/cukes/steps_subjectmappings.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests-bdd/cukes/steps_subjectmappings.go b/tests-bdd/cukes/steps_subjectmappings.go index 1040754376..5c03f83d04 100644 --- a/tests-bdd/cukes/steps_subjectmappings.go +++ b/tests-bdd/cukes/steps_subjectmappings.go @@ -279,7 +279,7 @@ func (s *SubjectMappingsStepDefinitions) seedSubjectAndResourceMappingsAtScale(c var actionID string err = tx.QueryRow(ctx, ` SELECT id - FROM otdf.actions + FROM `+policyDatabaseSchema+`.actions WHERE name = $1 AND (namespace_id = $2 OR namespace_id IS NULL) ORDER BY (namespace_id = $2) DESC LIMIT 1 From c9c49bd911b88f9581e11ea0d544c4145a1435e0 Mon Sep 17 00:00:00 2001 From: strantalis Date: Fri, 4 Sep 2026 16:24:36 -0400 Subject: [PATCH 05/10] test(authz): exercise policy page limits at scale Signed-off-by: strantalis --- tests-bdd/cukes/glue_platform.go | 2 ++ tests-bdd/cukes/resources/platform.template | 7 +++++ tests-bdd/cukes/steps_authorization.go | 1 + tests-bdd/cukes/steps_localplatform.go | 30 +++++++++++++------ ...ion-v2-subject-mapping-performance.feature | 26 +++++++++++++--- 5 files changed, 53 insertions(+), 13 deletions(-) diff --git a/tests-bdd/cukes/glue_platform.go b/tests-bdd/cukes/glue_platform.go index e851841348..69011b27d2 100644 --- a/tests-bdd/cukes/glue_platform.go +++ b/tests-bdd/cukes/glue_platform.go @@ -71,6 +71,7 @@ type LocalDevScenarioOptions struct { DatabaseName string PlatformPort int LDAPPort int + PolicyListRequestLimit int } func (d *DockerComposeLogger) Printf(format string, v ...interface{}) { @@ -137,6 +138,7 @@ func (c *PlatformTestSuiteContext) InitializeScenario(scenarioContext *godog.Sce PlatformEndpoint: trackedScenarioContext.ScenarioOptions.PlatformEndpoint, InsecureSkipVerifyConn: trackedScenarioContext.ScenarioOptions.InsecureSkipVerifyConn, LDAPPort: trackedScenarioContext.ScenarioOptions.LDAPPort, + PolicyListRequestLimit: trackedScenarioContext.ScenarioOptions.PolicyListRequestLimit, }, TestSuiteContext: c, } diff --git a/tests-bdd/cukes/resources/platform.template b/tests-bdd/cukes/resources/platform.template index 207eca2711..a4685c1107 100644 --- a/tests-bdd/cukes/resources/platform.template +++ b/tests-bdd/cukes/resources/platform.template @@ -46,11 +46,18 @@ services: email: true username: true # cache_expiration: 30s # disabled unless present and > 0 +{{ if .policyListRequestLimit }} + policy: + enabled: true + list_request_limit_default: {{ .policyListRequestLimit }} + list_request_limit_max: 10000 +{{ else }} # policy is enabled by default in mode 'all' # policy: # enabled: true # list_request_limit_default: 1000 # list_request_limit_max: 2500 +{{ end }} # authorization: # entitlement_policy_cache: # enabled: false diff --git a/tests-bdd/cukes/steps_authorization.go b/tests-bdd/cukes/steps_authorization.go index 59062ccb8a..05b9aac23d 100644 --- a/tests-bdd/cukes/steps_authorization.go +++ b/tests-bdd/cukes/steps_authorization.go @@ -237,6 +237,7 @@ func (s *AuthorizationServiceStepDefinitions) iSendAMultiResourceDecisionRequest slog.Duration("duration", elapsed), slog.Duration("maximum_duration", limit), slog.Int("resource_count", len(tbl.Rows)-1), + slog.Int("policy_list_request_limit", scenarioContext.ScenarioOptions.PolicyListRequestLimit), ) if err != nil { return ctx, err diff --git a/tests-bdd/cukes/steps_localplatform.go b/tests-bdd/cukes/steps_localplatform.go index 042133d1ef..594fdc55cf 100644 --- a/tests-bdd/cukes/steps_localplatform.go +++ b/tests-bdd/cukes/steps_localplatform.go @@ -288,6 +288,16 @@ func (s *LocalPlatformStepDefinitions) aEmptyLocalPlatform(ctx context.Context) return s.commonLocalPlatform(ctx, &platformStartOptions{kcProvisionPath: kt}) } +func (s *LocalPlatformStepDefinitions) anEmptyLocalPlatformWithPolicyListRequestPageLimit(ctx context.Context, limit int) (context.Context, error) { + if limit <= 0 || limit >= 10000 { + return ctx, fmt.Errorf("policy list request page limit must be between 1 and 9999, got %d", limit) + } + + scenarioContext := GetPlatformScenarioContext(ctx) + scenarioContext.ScenarioOptions.PolicyListRequestLimit = limit + return s.aEmptyLocalPlatform(ctx) +} + func (s *LocalPlatformStepDefinitions) aDefaultLocalPlatform(ctx context.Context) (context.Context, error) { kt := template.Must(template.New("kc").Parse(keycloakBaseTemplate)) return s.commonLocalPlatform(ctx, &platformStartOptions{ @@ -550,15 +560,16 @@ func createPlatformConfiguration(options *LocalDevOptions, scenarioOptions *Loca t := template.Must(template.New("platform").Parse(templateSource)) var strBuffer bytes.Buffer if err := t.Execute(&strBuffer, map[string]any{ - "hostname": options.Hostname, - "kcPort": options.keycloakPort, - "platformPort": scenarioOptions.PlatformPort, - "pgPort": options.postgresPort, - "pgDatabase": scenarioOptions.DatabaseName, - "pgHost": pgHost, - "platformKeysDir": platformKeysDir, - "authRealm": scenarioOptions.KeycloakRealm, - "ldapPort": scenarioOptions.LDAPPort, + "hostname": options.Hostname, + "kcPort": options.keycloakPort, + "platformPort": scenarioOptions.PlatformPort, + "pgPort": options.postgresPort, + "pgDatabase": scenarioOptions.DatabaseName, + "pgHost": pgHost, + "platformKeysDir": platformKeysDir, + "authRealm": scenarioOptions.KeycloakRealm, + "ldapPort": scenarioOptions.LDAPPort, + "policyListRequestLimit": scenarioOptions.PolicyListRequestLimit, }); err != nil { return tempFileName, err } @@ -597,6 +608,7 @@ func RegisterLocalPlatformStepDefinitions(ctx *godog.ScenarioContext, x *Platfor PlatformCukesContext: x, } ctx.Step(`^an empty local platform$`, platformStepDefinitions.aEmptyLocalPlatform) + ctx.Step(`^an empty local platform with policy list request page limit (\d+)$`, platformStepDefinitions.anEmptyLocalPlatformWithPolicyListRequestPageLimit) ctx.Step(`^a default local platform$`, platformStepDefinitions.aDefaultLocalPlatform) ctx.Step(`^I use the platform as "([^"]*)"$`, platformStepDefinitions.iUseThePlatformAs) ctx.Step(`^a user exists with username "([^"]*)" and email "([^"]*)" and the following attributes:$`, platformStepDefinitions.aUser) diff --git a/tests-bdd/features/authorization-v2-subject-mapping-performance.feature b/tests-bdd/features/authorization-v2-subject-mapping-performance.feature index 50314fcb82..e78cf2c99d 100644 --- a/tests-bdd/features/authorization-v2-subject-mapping-performance.feature +++ b/tests-bdd/features/authorization-v2-subject-mapping-performance.feature @@ -4,11 +4,11 @@ Feature: v2 multi-resource decisions at large policy scale subject-mapping and resource-mapping cardinality from the reported regression. Fixture setup is not timed. The measured operation is one request to the public v2 authorization endpoint. - Background: + Scenario Outline: Multi-resource decision completes within five seconds with 6011 subject mappings and page limit Given a user exists with username "scale-user" and email "scale-user@example.com" and the following attributes: | name | value | | department | ["engineering"] | - And an empty local platform + And an empty local platform with policy list request page limit And I submit a request to create a namespace with name "scale.example" and reference id "scale_ns" And I send a request to create an attribute referenced as "scale_attr" in namespace "scale_ns" named "access-level" with rule "anyOf" and 6011 generated values in batches of 25 Then the response should be successful @@ -20,8 +20,6 @@ Feature: v2 multi-resource decisions at large policy scale Then the response should be successful And the policy database contains 6011 subject mappings for attribute "scale_attr" using condition set "scale_condition_set" in namespace "scale_ns" with action "read" and 6000 resource mappings And there is a "user_name" subject entity with value "scale-user" and referenced as "scale-user" - - Scenario: Multi-resource decision completes within five seconds with 6011 subject mappings When I send a multi-resource decision request for entity chain "scale-user" for "read" action on resources within "5s": | resource | | https://scale.example/attr/access-level/value/v0000 | @@ -33,3 +31,23 @@ Feature: v2 multi-resource decisions at large policy scale And the decision response for resource "https://scale.example/attr/access-level/value/v0000" should be "PERMIT" And the decision response for resource "https://scale.example/attr/access-level/value/v3005" should be "PERMIT" And the decision response for resource "https://scale.example/attr/access-level/value/v6010" should be "PERMIT" + + @page-limit-100 + Examples: 100 mappings per page + | page_limit | + | 100 | + + @page-limit-500 + Examples: 500 mappings per page + | page_limit | + | 500 | + + @page-limit-1000 + Examples: 1000 mappings per page + | page_limit | + | 1000 | + + @page-limit-2500 + Examples: 2500 mappings per page + | page_limit | + | 2500 | From a5ae1c570beb7dd80bee2070742972d47b243228 Mon Sep 17 00:00:00 2001 From: strantalis Date: Sat, 5 Sep 2026 07:08:18 -0400 Subject: [PATCH 06/10] test(authz): exercise concurrent decisions at scale Signed-off-by: strantalis --- .github/workflows/checks.yaml | 36 +++++- tests-bdd/cukes/glue_platform.go | 2 - tests-bdd/cukes/resources/platform.template | 7 - tests-bdd/cukes/steps_authorization.go | 121 +++++++++++++++++- tests-bdd/cukes/steps_localplatform.go | 30 ++--- ...ion-v2-subject-mapping-performance.feature | 41 +++--- 6 files changed, 185 insertions(+), 52 deletions(-) diff --git a/.github/workflows/checks.yaml b/.github/workflows/checks.yaml index e55262e42e..4b66197d05 100644 --- a/.github/workflows/checks.yaml +++ b/.github/workflows/checks.yaml @@ -532,7 +532,41 @@ jobs: - name: Run BDD Tests run: | - CUKES_LOG_HANDLER=console go test ./tests-bdd -v --tags=cukes --godog.random --godog.format="cucumber:$(pwd)/cukes_platform_report.json,pretty:$(pwd)/cukes_platform_report.log,pretty" ./features + CUKES_LOG_HANDLER=console go test ./tests-bdd -v --tags=cukes --godog.random --godog.format="cucumber:$(pwd)/cukes_platform_report.json,pretty:$(pwd)/cukes_platform_report.log,pretty" ./features 2>&1 | tee cukes_test_output.log + + - name: Summarize authorization performance + if: ${{ !cancelled() }} + run: | + PERFORMANCE_ROWS=$(awk ' + /authorization v2 concurrent multi-resource performance/ { + delete values + for (i = 1; i <= NF; i++) { + split($i, pair, "=") + values[pair[1]] = pair[2] + } + printf "%d\t%s\t%s\t%s\t%s\t%d\n", + values["concurrency"], + values["wall_duration"], + values["median_request_duration"], + values["p95_request_duration"], + values["maximum_request_duration"], + values["failed_request_count"] + } + ' cukes_test_output.log | sort -n | awk -F '\t' '{ + printf "| %s | `%s` | `%s` | `%s` | `%s` | %s |\n", $1, $2, $3, $4, $5, $6 + }') + + { + echo "### Authorization v2 concurrency performance" + echo + if [[ -z "$PERFORMANCE_ROWS" ]]; then + echo "No authorization performance scenarios completed." + else + echo "| Concurrency | Wall time | Median request | p95 request | Maximum request | Failures |" + echo "| ---: | ---: | ---: | ---: | ---: | ---: |" + echo "$PERFORMANCE_ROWS" + fi + } >> "$GITHUB_STEP_SUMMARY" - name: Check for undefined steps run: | diff --git a/tests-bdd/cukes/glue_platform.go b/tests-bdd/cukes/glue_platform.go index 69011b27d2..e851841348 100644 --- a/tests-bdd/cukes/glue_platform.go +++ b/tests-bdd/cukes/glue_platform.go @@ -71,7 +71,6 @@ type LocalDevScenarioOptions struct { DatabaseName string PlatformPort int LDAPPort int - PolicyListRequestLimit int } func (d *DockerComposeLogger) Printf(format string, v ...interface{}) { @@ -138,7 +137,6 @@ func (c *PlatformTestSuiteContext) InitializeScenario(scenarioContext *godog.Sce PlatformEndpoint: trackedScenarioContext.ScenarioOptions.PlatformEndpoint, InsecureSkipVerifyConn: trackedScenarioContext.ScenarioOptions.InsecureSkipVerifyConn, LDAPPort: trackedScenarioContext.ScenarioOptions.LDAPPort, - PolicyListRequestLimit: trackedScenarioContext.ScenarioOptions.PolicyListRequestLimit, }, TestSuiteContext: c, } diff --git a/tests-bdd/cukes/resources/platform.template b/tests-bdd/cukes/resources/platform.template index a4685c1107..207eca2711 100644 --- a/tests-bdd/cukes/resources/platform.template +++ b/tests-bdd/cukes/resources/platform.template @@ -46,18 +46,11 @@ services: email: true username: true # cache_expiration: 30s # disabled unless present and > 0 -{{ if .policyListRequestLimit }} - policy: - enabled: true - list_request_limit_default: {{ .policyListRequestLimit }} - list_request_limit_max: 10000 -{{ else }} # policy is enabled by default in mode 'all' # policy: # enabled: true # list_request_limit_default: 1000 # list_request_limit_max: 2500 -{{ end }} # authorization: # entitlement_policy_cache: # enabled: false diff --git a/tests-bdd/cukes/steps_authorization.go b/tests-bdd/cukes/steps_authorization.go index 05b9aac23d..f4ff739781 100644 --- a/tests-bdd/cukes/steps_authorization.go +++ b/tests-bdd/cukes/steps_authorization.go @@ -6,8 +6,10 @@ import ( "fmt" "log/slog" "net" + "sort" "strconv" "strings" + "sync" "time" "github.com/cucumber/godog" @@ -15,6 +17,7 @@ import ( "github.com/opentdf/platform/protocol/go/entity" "github.com/opentdf/platform/protocol/go/policy" "google.golang.org/protobuf/encoding/protojson" + "google.golang.org/protobuf/proto" "google.golang.org/protobuf/types/known/anypb" "google.golang.org/protobuf/types/known/structpb" ) @@ -237,7 +240,6 @@ func (s *AuthorizationServiceStepDefinitions) iSendAMultiResourceDecisionRequest slog.Duration("duration", elapsed), slog.Duration("maximum_duration", limit), slog.Int("resource_count", len(tbl.Rows)-1), - slog.Int("policy_list_request_limit", scenarioContext.ScenarioOptions.PolicyListRequestLimit), ) if err != nil { return ctx, err @@ -252,6 +254,122 @@ func (s *AuthorizationServiceStepDefinitions) iSendAMultiResourceDecisionRequest return ctx, nil } +func (s *AuthorizationServiceStepDefinitions) iSendConcurrentMultiResourceDecisionRequestsWithin(ctx context.Context, concurrency int, entityChainID, action, maximumDuration string, tbl *godog.Table) (context.Context, error) { + if concurrency <= 0 { + return ctx, fmt.Errorf("concurrency must be positive, got %d", concurrency) + } + + limit, err := time.ParseDuration(maximumDuration) + if err != nil { + return ctx, fmt.Errorf("parse maximum decision duration %q: %w", maximumDuration, err) + } + if limit <= 0 { + return ctx, fmt.Errorf("maximum decision duration must be positive, got %s", limit) + } + + scenarioContext := GetPlatformScenarioContext(ctx) + scenarioContext.ClearError() + entityChain, err := buildEntityChainFromIDs(scenarioContext, entityChainID) + if err != nil { + return ctx, err + } + resources, resourceFQNMap, err := buildResourcesFromTable(tbl) + if err != nil { + return ctx, err + } + + req := &authzV2.GetDecisionMultiResourceRequest{ + EntityIdentifier: &authzV2.EntityIdentifier{ + Identifier: &authzV2.EntityIdentifier_EntityChain{EntityChain: entityChain}, + }, + Action: &policy.Action{Name: strings.ToLower(action)}, + Resources: resources, + FulfillableObligationFqns: getAllObligationsFromScenario(scenarioContext), + } + + requestCtx, cancel := context.WithTimeout(ctx, limit) + defer cancel() + start := make(chan struct{}) + responses := make([]*authzV2.GetDecisionMultiResourceResponse, concurrency) + requestErrors := make([]error, concurrency) + durations := make([]time.Duration, concurrency) + + var wg sync.WaitGroup + wg.Add(concurrency) + for i := range concurrency { + go func() { + defer wg.Done() + <-start + started := time.Now() + clonedReq, ok := proto.Clone(req).(*authzV2.GetDecisionMultiResourceRequest) + if !ok { + requestErrors[i] = errors.New("clone multi-resource decision request") + return + } + responses[i], requestErrors[i] = scenarioContext.SDK.AuthorizationV2.GetDecisionMultiResource(requestCtx, clonedReq) + durations[i] = time.Since(started) + }() + } + + wallStarted := time.Now() + close(start) + wg.Wait() + wallElapsed := time.Since(wallStarted) + sortedDurations := append([]time.Duration(nil), durations...) + sort.Slice(sortedDurations, func(i, j int) bool { return sortedDurations[i] < sortedDurations[j] }) + maximumRequestDuration := sortedDurations[len(sortedDurations)-1] + medianRequestDuration := sortedDurations[(len(sortedDurations)-1)/2] + p95RequestDuration := sortedDurations[(95*len(sortedDurations)-1)/100] + failedRequestCount := 0 + var firstRequestError error + for i, requestErr := range requestErrors { + if requestErr != nil { + failedRequestCount++ + if firstRequestError == nil { + firstRequestError = fmt.Errorf("concurrent multi-resource decision request %d failed after %s: %w", i+1, durations[i], requestErr) + } + continue + } + if responses[i] == nil { + failedRequestCount++ + if firstRequestError == nil { + firstRequestError = fmt.Errorf("concurrent multi-resource decision request %d returned no response", i+1) + } + continue + } + if i > 0 && !proto.Equal(responses[0], responses[i]) { + failedRequestCount++ + if firstRequestError == nil { + firstRequestError = fmt.Errorf("concurrent multi-resource decision request %d returned a different response", i+1) + } + } + } + + scenarioContext.TestSuiteContext.Logger.Info( + "authorization v2 concurrent multi-resource performance", + slog.Int("concurrency", concurrency), + slog.Duration("wall_duration", wallElapsed), + slog.Duration("median_request_duration", medianRequestDuration), + slog.Duration("p95_request_duration", p95RequestDuration), + slog.Duration("maximum_request_duration", maximumRequestDuration), + slog.Duration("maximum_duration", limit), + slog.Int("resource_count", len(resources)), + slog.Int("failed_request_count", failedRequestCount), + ) + + if firstRequestError != nil { + return ctx, firstRequestError + } + if maximumRequestDuration > limit { + return ctx, fmt.Errorf("slowest concurrent multi-resource decision took %s, exceeding the %s limit", maximumRequestDuration, limit) + } + + scenarioContext.RecordObject(multiDecisionResponseKey, responses[0]) + scenarioContext.RecordObject(decisionResponse, responses[0]) + scenarioContext.RecordObject("resourceFQNMap", resourceFQNMap) + return ctx, nil +} + func (s *AuthorizationServiceStepDefinitions) iSendAMultiResourceDecisionRequestForEntityChainForActionOnResourcesWithNoFulfillableObligations(ctx context.Context, entityChainID string, action string, tbl *godog.Table) (context.Context, error) { return s.iSendAMultiResourceDecisionRequestForEntityChainForActionOnResourcesWithFulfillableObligations(ctx, entityChainID, action, "[]", tbl) } @@ -595,6 +713,7 @@ func RegisterAuthorizationStepDefinitions(ctx *godog.ScenarioContext) { ctx.Step(`^I send a decision request for entity chain "([^"]*)" for "([^"]*)" action on resource "([^"]*)" with no fulfillable obligations$`, stepDefinitions.iSendADecisionRequestForEntityChainForActionOnResourceWithNoFulfillableObligations) ctx.Step(`^I send a multi-resource decision request for entity chain "([^"]*)" for "([^"]*)" action on resources:$`, stepDefinitions.iSendAMultiResourceDecisionRequestForEntityChainForActionOnResources) ctx.Step(`^I send a multi-resource decision request for entity chain "([^"]*)" for "([^"]*)" action on resources within "([^"]*)":$`, stepDefinitions.iSendAMultiResourceDecisionRequestWithin) + ctx.Step(`^I send (\d+) concurrent multi-resource decision requests for entity chain "([^"]*)" for "([^"]*)" action on resources each within "([^"]*)":$`, stepDefinitions.iSendConcurrentMultiResourceDecisionRequestsWithin) ctx.Step(`^I send a multi-resource decision request for entity chain "([^"]*)" for "([^"]*)" action on resources with no fulfillable obligations:$`, stepDefinitions.iSendAMultiResourceDecisionRequestForEntityChainForActionOnResourcesWithNoFulfillableObligations) ctx.Step(`^I send a multi-resource decision request for entity chain "([^"]*)" for "([^"]*)" action on resources with fulfillable obligations "([^"]*)":$`, stepDefinitions.iSendAMultiResourceDecisionRequestForEntityChainForActionOnResourcesWithFulfillableObligations) ctx.Step(`^I should get a "([^"]*)" decision response$`, stepDefinitions.iShouldGetADecisionResponse) diff --git a/tests-bdd/cukes/steps_localplatform.go b/tests-bdd/cukes/steps_localplatform.go index 594fdc55cf..042133d1ef 100644 --- a/tests-bdd/cukes/steps_localplatform.go +++ b/tests-bdd/cukes/steps_localplatform.go @@ -288,16 +288,6 @@ func (s *LocalPlatformStepDefinitions) aEmptyLocalPlatform(ctx context.Context) return s.commonLocalPlatform(ctx, &platformStartOptions{kcProvisionPath: kt}) } -func (s *LocalPlatformStepDefinitions) anEmptyLocalPlatformWithPolicyListRequestPageLimit(ctx context.Context, limit int) (context.Context, error) { - if limit <= 0 || limit >= 10000 { - return ctx, fmt.Errorf("policy list request page limit must be between 1 and 9999, got %d", limit) - } - - scenarioContext := GetPlatformScenarioContext(ctx) - scenarioContext.ScenarioOptions.PolicyListRequestLimit = limit - return s.aEmptyLocalPlatform(ctx) -} - func (s *LocalPlatformStepDefinitions) aDefaultLocalPlatform(ctx context.Context) (context.Context, error) { kt := template.Must(template.New("kc").Parse(keycloakBaseTemplate)) return s.commonLocalPlatform(ctx, &platformStartOptions{ @@ -560,16 +550,15 @@ func createPlatformConfiguration(options *LocalDevOptions, scenarioOptions *Loca t := template.Must(template.New("platform").Parse(templateSource)) var strBuffer bytes.Buffer if err := t.Execute(&strBuffer, map[string]any{ - "hostname": options.Hostname, - "kcPort": options.keycloakPort, - "platformPort": scenarioOptions.PlatformPort, - "pgPort": options.postgresPort, - "pgDatabase": scenarioOptions.DatabaseName, - "pgHost": pgHost, - "platformKeysDir": platformKeysDir, - "authRealm": scenarioOptions.KeycloakRealm, - "ldapPort": scenarioOptions.LDAPPort, - "policyListRequestLimit": scenarioOptions.PolicyListRequestLimit, + "hostname": options.Hostname, + "kcPort": options.keycloakPort, + "platformPort": scenarioOptions.PlatformPort, + "pgPort": options.postgresPort, + "pgDatabase": scenarioOptions.DatabaseName, + "pgHost": pgHost, + "platformKeysDir": platformKeysDir, + "authRealm": scenarioOptions.KeycloakRealm, + "ldapPort": scenarioOptions.LDAPPort, }); err != nil { return tempFileName, err } @@ -608,7 +597,6 @@ func RegisterLocalPlatformStepDefinitions(ctx *godog.ScenarioContext, x *Platfor PlatformCukesContext: x, } ctx.Step(`^an empty local platform$`, platformStepDefinitions.aEmptyLocalPlatform) - ctx.Step(`^an empty local platform with policy list request page limit (\d+)$`, platformStepDefinitions.anEmptyLocalPlatformWithPolicyListRequestPageLimit) ctx.Step(`^a default local platform$`, platformStepDefinitions.aDefaultLocalPlatform) ctx.Step(`^I use the platform as "([^"]*)"$`, platformStepDefinitions.iUseThePlatformAs) ctx.Step(`^a user exists with username "([^"]*)" and email "([^"]*)" and the following attributes:$`, platformStepDefinitions.aUser) diff --git a/tests-bdd/features/authorization-v2-subject-mapping-performance.feature b/tests-bdd/features/authorization-v2-subject-mapping-performance.feature index e78cf2c99d..12a128c2ac 100644 --- a/tests-bdd/features/authorization-v2-subject-mapping-performance.feature +++ b/tests-bdd/features/authorization-v2-subject-mapping-performance.feature @@ -2,13 +2,14 @@ Feature: v2 multi-resource decisions at large policy scale GetDecisionMultiResource must remain responsive when the policy database contains the subject-mapping and resource-mapping cardinality from the reported regression. Fixture setup is - not timed. The measured operation is one request to the public v2 authorization endpoint. + not timed. The measured operation is a synchronized group of requests to the public v2 + authorization endpoint. - Scenario Outline: Multi-resource decision completes within five seconds with 6011 subject mappings and page limit + Scenario Outline: Concurrent multi-resource decisions complete within five seconds with 6011 subject mappings at concurrency Given a user exists with username "scale-user" and email "scale-user@example.com" and the following attributes: | name | value | | department | ["engineering"] | - And an empty local platform with policy list request page limit + And an empty local platform And I submit a request to create a namespace with name "scale.example" and reference id "scale_ns" And I send a request to create an attribute referenced as "scale_attr" in namespace "scale_ns" named "access-level" with rule "anyOf" and 6011 generated values in batches of 25 Then the response should be successful @@ -20,7 +21,7 @@ Feature: v2 multi-resource decisions at large policy scale Then the response should be successful And the policy database contains 6011 subject mappings for attribute "scale_attr" using condition set "scale_condition_set" in namespace "scale_ns" with action "read" and 6000 resource mappings And there is a "user_name" subject entity with value "scale-user" and referenced as "scale-user" - When I send a multi-resource decision request for entity chain "scale-user" for "read" action on resources within "5s": + When I send concurrent multi-resource decision requests for entity chain "scale-user" for "read" action on resources each within "5s": | resource | | https://scale.example/attr/access-level/value/v0000 | | https://scale.example/attr/access-level/value/v3005 | @@ -32,22 +33,22 @@ Feature: v2 multi-resource decisions at large policy scale And the decision response for resource "https://scale.example/attr/access-level/value/v3005" should be "PERMIT" And the decision response for resource "https://scale.example/attr/access-level/value/v6010" should be "PERMIT" - @page-limit-100 - Examples: 100 mappings per page - | page_limit | - | 100 | + @concurrency-1 + Examples: One request + | concurrency | + | 1 | - @page-limit-500 - Examples: 500 mappings per page - | page_limit | - | 500 | + @concurrency-10 + Examples: Ten concurrent requests + | concurrency | + | 10 | - @page-limit-1000 - Examples: 1000 mappings per page - | page_limit | - | 1000 | + @concurrency-25 + Examples: Twenty-five concurrent requests + | concurrency | + | 25 | - @page-limit-2500 - Examples: 2500 mappings per page - | page_limit | - | 2500 | + @concurrency-50 + Examples: Fifty concurrent requests + | concurrency | + | 50 | From 1f3a952ee3b7f9f3e4581e4aa6d1181092190084 Mon Sep 17 00:00:00 2001 From: strantalis Date: Tue, 8 Sep 2026 18:09:32 -0400 Subject: [PATCH 07/10] test(authz): address scale coverage review and report CI latency Signed-off-by: strantalis --- .../scripts/summarize-authz-performance.py | 83 +++++++ .../test_summarize_authz_performance.py | 61 +++++ .github/workflows/checks.yaml | 39 +--- tests-bdd/cukes/scale_setup.go | 42 ++++ tests-bdd/cukes/steps_authorization.go | 159 +------------ tests-bdd/cukes/steps_authorization_scale.go | 210 ++++++++++++++++++ .../cukes/steps_authorization_scale_test.go | 49 ++++ .../cukes/steps_resourcemappings_scale.go | 34 +++ tests-bdd/cukes/steps_subjectmappings.go | 120 +--------- .../cukes/steps_subjectmappings_scale.go | 35 +++ ...ion-v2-subject-mapping-performance.feature | 37 +-- tests-bdd/platform_test.go | 1 + 12 files changed, 546 insertions(+), 324 deletions(-) create mode 100644 .github/scripts/summarize-authz-performance.py create mode 100644 .github/scripts/test_summarize_authz_performance.py create mode 100644 tests-bdd/cukes/scale_setup.go create mode 100644 tests-bdd/cukes/steps_authorization_scale.go create mode 100644 tests-bdd/cukes/steps_authorization_scale_test.go create mode 100644 tests-bdd/cukes/steps_resourcemappings_scale.go create mode 100644 tests-bdd/cukes/steps_subjectmappings_scale.go diff --git a/.github/scripts/summarize-authz-performance.py b/.github/scripts/summarize-authz-performance.py new file mode 100644 index 0000000000..a79b9919b7 --- /dev/null +++ b/.github/scripts/summarize-authz-performance.py @@ -0,0 +1,83 @@ +#!/usr/bin/env python3 +"""Render structured authorization measurements, including failed BDD runs.""" +import argparse +import json +from pathlib import Path + +MARKER = "AUTHZ_PERFORMANCE " +NUMBERS = ( + "seed", "concurrency", "resources", "wall_ns", "median_ns", "p95_ns", + "maximum_ns", "timeout_ns", "failures", +) + + +def read_results(text): + results, malformed = [], 0 + for line in text.splitlines(): + # Support both console logs and go test -json artifacts. + if line.startswith("{"): + try: + output = json.loads(line).get("Output") + if isinstance(output, str): + line = output + except (ValueError, AttributeError): + pass + if MARKER not in line: + continue + try: + result = json.loads(line.split(MARKER, 1)[1]) + if not isinstance(result.get("case"), str) or not result["case"]: + raise ValueError("missing case") + if any(type(result.get(key)) is not int or result[key] < 0 for key in NUMBERS): + raise ValueError("invalid numeric field") + if not result["concurrency"] or not result["resources"] or not result["timeout_ns"]: + raise ValueError("invalid dimensions") + if result["failures"] > result["concurrency"]: + raise ValueError("invalid failure count") + results.append(result) + except (ValueError, TypeError, AttributeError): + malformed += 1 + return sorted(results, key=lambda row: (row["concurrency"], row["case"], row["seed"])), malformed + + +def milliseconds(nanoseconds): + return f"{nanoseconds / 1_000_000:.2f} ms" + + +def render(text, outcome): + results, malformed = read_results(text) + lines = ["### Authorization v2 concurrency performance", "", f"BDD step outcome: **{outcome}**.", ""] + if results: + lines += [ + "| Case | Concurrency | Resources | Seed | Wall | Median | p95 | Maximum | Timeout | Failures | Requests |", + "| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | --- |", + ] + for row in results: + case = row["case"].replace("|", "\\|").replace("\n", " ").replace("\r", " ") + status = "FAIL" if row["failures"] else "PASS" + cells = [case, str(row["concurrency"]), str(row["resources"]), str(row["seed"])] + cells += [milliseconds(row[key]) for key in ("wall_ns", "median_ns", "p95_ns", "maximum_ns", "timeout_ns")] + cells += [str(row["failures"]), status] + lines.append("| " + " | ".join(cells) + " |") + lines += ["", f"Reported {len(results)} completed case batches. Missing measurements are not passes."] + else: + lines.append("No authorization measurements were produced. Check BDD setup and logs; this is not a passing performance result.") + if malformed: + lines += ["", f"**Summary error: {malformed} malformed performance record(s).**"] + lines += ["", "Fixture setup is excluded. PASS means requests completed with the expected decisions. Failures include request errors, timeouts, and incorrect decisions. Latency is report-only until a baseline is established; the request timeout is not a latency target.", ""] + return "\n".join(lines), malformed + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("log", type=Path) + parser.add_argument("--bdd-outcome", default="unknown", choices=["success", "failure", "cancelled", "skipped", "unknown"]) + args = parser.parse_args() + text = args.log.read_text() if args.log.exists() else "" + summary, malformed = render(text, args.bdd_outcome) + print(summary) + return bool(malformed) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/.github/scripts/test_summarize_authz_performance.py b/.github/scripts/test_summarize_authz_performance.py new file mode 100644 index 0000000000..7e39142e62 --- /dev/null +++ b/.github/scripts/test_summarize_authz_performance.py @@ -0,0 +1,61 @@ +import importlib.util +import json +import subprocess +import sys +import tempfile +from pathlib import Path +import unittest + +spec = importlib.util.spec_from_file_location("summary", Path(__file__).with_name("summarize-authz-performance.py")) +summary = importlib.util.module_from_spec(spec) +spec.loader.exec_module(summary) + + +class SummaryTests(unittest.TestCase): + def record(self, **changes): + row = dict(case="allowed_read", concurrency=50, resources=3, seed=4625, + wall_ns=150000000, median_ns=100000000, p95_ns=130000000, + maximum_ns=140000000, timeout_ns=30000000000, failures=0) + row.update(changes) + return summary.MARKER + json.dumps(row) + + def test_console_and_json_records_have_identical_rendering(self): + record = self.record() + console, errors = summary.render(record, "success") + encoded, _ = summary.render(json.dumps({"Action": "output", "Output": record + "\n"}), "success") + self.assertEqual(console, encoded) + self.assertEqual(errors, 0) + self.assertIn("130.00 ms | 140.00 ms | 30000.00 ms | 0 | PASS", console) + + def test_partial_failure_keeps_rows_without_gating_slow_requests(self): + text = self.record(case="denied_user", failures=2) + "\n" + self.record(case="allowed_read", maximum_ns=8000000000) + rendered, errors = summary.render(text, "failure") + self.assertEqual(errors, 0) + self.assertIn("BDD step outcome: **failure**", rendered) + self.assertIn("8000.00 ms | 30000.00 ms | 0 | PASS |", rendered) + self.assertIn("| 2 | FAIL |", rendered) + self.assertIn("Latency is report-only", rendered) + self.assertLess(rendered.index("allowed_read"), rendered.index("denied_user")) + + def test_cli_handles_missing_log_after_setup_failure(self): + with tempfile.TemporaryDirectory() as directory: + result = subprocess.run( + [sys.executable, str(Path(__file__).with_name("summarize-authz-performance.py")), + str(Path(directory) / "missing.log"), "--bdd-outcome", "failure"], + capture_output=True, text=True, check=True, + ) + self.assertIn("No authorization measurements", result.stdout) + self.assertIn("BDD step outcome: **failure**", result.stdout) + + def test_missing_and_malformed_measurements_are_not_passes(self): + rendered, errors = summary.render("setup failed", "failure") + self.assertIn("No authorization measurements", rendered) + self.assertEqual(errors, 0) + rendered, errors = summary.render(self.record() + "\n" + summary.MARKER + "{}", "failure") + self.assertEqual(errors, 1) + self.assertIn("malformed performance record", rendered) + self.assertIn("allowed_read", rendered) + + +if __name__ == "__main__": + unittest.main() diff --git a/.github/workflows/checks.yaml b/.github/workflows/checks.yaml index 4b66197d05..f4c8333cc8 100644 --- a/.github/workflows/checks.yaml +++ b/.github/workflows/checks.yaml @@ -527,46 +527,22 @@ jobs: go-version-file: go.work cache: false + - name: Test authorization summary renderer + run: python3 -m unittest discover -s .github/scripts -p 'test_summarize_authz_performance.py' + - name: Build local platform-cukes image for testing run: docker build -t platform-cukes . - name: Run BDD Tests + id: bdd + shell: bash run: | - CUKES_LOG_HANDLER=console go test ./tests-bdd -v --tags=cukes --godog.random --godog.format="cucumber:$(pwd)/cukes_platform_report.json,pretty:$(pwd)/cukes_platform_report.log,pretty" ./features 2>&1 | tee cukes_test_output.log + CUKES_LOG_HANDLER=console go test ./tests-bdd -v -timeout=20m --tags=cukes --godog.random --godog.format="cucumber:$(pwd)/cukes_platform_report.json,pretty:$(pwd)/cukes_platform_report.log,pretty" ./features 2>&1 | tee cukes_test_output.log - name: Summarize authorization performance if: ${{ !cancelled() }} run: | - PERFORMANCE_ROWS=$(awk ' - /authorization v2 concurrent multi-resource performance/ { - delete values - for (i = 1; i <= NF; i++) { - split($i, pair, "=") - values[pair[1]] = pair[2] - } - printf "%d\t%s\t%s\t%s\t%s\t%d\n", - values["concurrency"], - values["wall_duration"], - values["median_request_duration"], - values["p95_request_duration"], - values["maximum_request_duration"], - values["failed_request_count"] - } - ' cukes_test_output.log | sort -n | awk -F '\t' '{ - printf "| %s | `%s` | `%s` | `%s` | `%s` | %s |\n", $1, $2, $3, $4, $5, $6 - }') - - { - echo "### Authorization v2 concurrency performance" - echo - if [[ -z "$PERFORMANCE_ROWS" ]]; then - echo "No authorization performance scenarios completed." - else - echo "| Concurrency | Wall time | Median request | p95 request | Maximum request | Failures |" - echo "| ---: | ---: | ---: | ---: | ---: | ---: |" - echo "$PERFORMANCE_ROWS" - fi - } >> "$GITHUB_STEP_SUMMARY" + python3 .github/scripts/summarize-authz-performance.py cukes_test_output.log --bdd-outcome "${{ steps.bdd.outcome || 'skipped' }}" >> "$GITHUB_STEP_SUMMARY" - name: Check for undefined steps run: | @@ -591,6 +567,7 @@ jobs: path: | cukes_platform_report.json cukes_platform_report.log + cukes_test_output.log retention-days: 1 # test otdfctl CLI e2e against platform PR branch diff --git a/tests-bdd/cukes/scale_setup.go b/tests-bdd/cukes/scale_setup.go new file mode 100644 index 0000000000..3eaa49a98f --- /dev/null +++ b/tests-bdd/cukes/scale_setup.go @@ -0,0 +1,42 @@ +package cukes + +import ( + "context" + "errors" + "fmt" + "sync" + + "github.com/opentdf/platform/protocol/go/policy" +) + +// Setup uses bounded API calls and finishes before authorization is measured. +func createScaleMappings(ctx context.Context, count int, create func(context.Context, int) error) error { + // Match the default HTTP idle pool so thousands of setup calls reuse connections. + const batchSize = 2 + for start := 0; start < count; start += batchSize { + end := min(start+batchSize, count) + failures := make([]error, end-start) + var workers sync.WaitGroup + for index := start; index < end; index++ { + workers.Go(func() { failures[index-start] = create(ctx, index) }) + } + workers.Wait() + if err := errors.Join(failures...); err != nil { + return err + } + } + return nil +} + +func scaleMappingInputs(ctx context.Context, count int, attributeRef, namespaceRef string) (*PlatformScenarioContext, *policy.Attribute, string, error) { + scenario := GetPlatformScenarioContext(ctx) + attribute, ok := scenario.GetObject(attributeRef).(*policy.Attribute) + if !ok || count <= 0 || count > len(attribute.GetValues()) { + return nil, nil, "", fmt.Errorf("attribute %q must contain at least %d values for mapping setup", attributeRef, count) + } + namespace, ok := scenario.GetObject(namespaceRef).(string) + if !ok { + return nil, nil, "", fmt.Errorf("missing namespace %q", namespaceRef) + } + return scenario, attribute, namespace, nil +} diff --git a/tests-bdd/cukes/steps_authorization.go b/tests-bdd/cukes/steps_authorization.go index f4ff739781..ff813a4442 100644 --- a/tests-bdd/cukes/steps_authorization.go +++ b/tests-bdd/cukes/steps_authorization.go @@ -4,20 +4,15 @@ import ( "context" "errors" "fmt" - "log/slog" "net" - "sort" "strconv" "strings" - "sync" - "time" "github.com/cucumber/godog" authzV2 "github.com/opentdf/platform/protocol/go/authorization/v2" "github.com/opentdf/platform/protocol/go/entity" "github.com/opentdf/platform/protocol/go/policy" "google.golang.org/protobuf/encoding/protojson" - "google.golang.org/protobuf/proto" "google.golang.org/protobuf/types/known/anypb" "google.golang.org/protobuf/types/known/structpb" ) @@ -219,157 +214,6 @@ func (s *AuthorizationServiceStepDefinitions) iSendAMultiResourceDecisionRequest return ctx, nil } -func (s *AuthorizationServiceStepDefinitions) iSendAMultiResourceDecisionRequestWithin(ctx context.Context, entityChainID, action, maximumDuration string, tbl *godog.Table) (context.Context, error) { - limit, err := time.ParseDuration(maximumDuration) - if err != nil { - return ctx, fmt.Errorf("parse maximum decision duration %q: %w", maximumDuration, err) - } - if limit <= 0 { - return ctx, fmt.Errorf("maximum decision duration must be positive, got %s", limit) - } - - requestCtx, cancel := context.WithTimeout(ctx, limit) - defer cancel() - started := time.Now() - _, err = s.iSendAMultiResourceDecisionRequestForEntityChainForActionOnResources(requestCtx, entityChainID, action, tbl) - elapsed := time.Since(started) - - scenarioContext := GetPlatformScenarioContext(ctx) - scenarioContext.TestSuiteContext.Logger.Info( - "authorization v2 multi-resource performance", - slog.Duration("duration", elapsed), - slog.Duration("maximum_duration", limit), - slog.Int("resource_count", len(tbl.Rows)-1), - ) - if err != nil { - return ctx, err - } - if requestErr := scenarioContext.GetError(); requestErr != nil { - return ctx, fmt.Errorf("multi-resource decision failed after %s: %w", elapsed, requestErr) - } - if elapsed > limit { - return ctx, fmt.Errorf("multi-resource decision took %s, exceeding the %s limit", elapsed, limit) - } - - return ctx, nil -} - -func (s *AuthorizationServiceStepDefinitions) iSendConcurrentMultiResourceDecisionRequestsWithin(ctx context.Context, concurrency int, entityChainID, action, maximumDuration string, tbl *godog.Table) (context.Context, error) { - if concurrency <= 0 { - return ctx, fmt.Errorf("concurrency must be positive, got %d", concurrency) - } - - limit, err := time.ParseDuration(maximumDuration) - if err != nil { - return ctx, fmt.Errorf("parse maximum decision duration %q: %w", maximumDuration, err) - } - if limit <= 0 { - return ctx, fmt.Errorf("maximum decision duration must be positive, got %s", limit) - } - - scenarioContext := GetPlatformScenarioContext(ctx) - scenarioContext.ClearError() - entityChain, err := buildEntityChainFromIDs(scenarioContext, entityChainID) - if err != nil { - return ctx, err - } - resources, resourceFQNMap, err := buildResourcesFromTable(tbl) - if err != nil { - return ctx, err - } - - req := &authzV2.GetDecisionMultiResourceRequest{ - EntityIdentifier: &authzV2.EntityIdentifier{ - Identifier: &authzV2.EntityIdentifier_EntityChain{EntityChain: entityChain}, - }, - Action: &policy.Action{Name: strings.ToLower(action)}, - Resources: resources, - FulfillableObligationFqns: getAllObligationsFromScenario(scenarioContext), - } - - requestCtx, cancel := context.WithTimeout(ctx, limit) - defer cancel() - start := make(chan struct{}) - responses := make([]*authzV2.GetDecisionMultiResourceResponse, concurrency) - requestErrors := make([]error, concurrency) - durations := make([]time.Duration, concurrency) - - var wg sync.WaitGroup - wg.Add(concurrency) - for i := range concurrency { - go func() { - defer wg.Done() - <-start - started := time.Now() - clonedReq, ok := proto.Clone(req).(*authzV2.GetDecisionMultiResourceRequest) - if !ok { - requestErrors[i] = errors.New("clone multi-resource decision request") - return - } - responses[i], requestErrors[i] = scenarioContext.SDK.AuthorizationV2.GetDecisionMultiResource(requestCtx, clonedReq) - durations[i] = time.Since(started) - }() - } - - wallStarted := time.Now() - close(start) - wg.Wait() - wallElapsed := time.Since(wallStarted) - sortedDurations := append([]time.Duration(nil), durations...) - sort.Slice(sortedDurations, func(i, j int) bool { return sortedDurations[i] < sortedDurations[j] }) - maximumRequestDuration := sortedDurations[len(sortedDurations)-1] - medianRequestDuration := sortedDurations[(len(sortedDurations)-1)/2] - p95RequestDuration := sortedDurations[(95*len(sortedDurations)-1)/100] - failedRequestCount := 0 - var firstRequestError error - for i, requestErr := range requestErrors { - if requestErr != nil { - failedRequestCount++ - if firstRequestError == nil { - firstRequestError = fmt.Errorf("concurrent multi-resource decision request %d failed after %s: %w", i+1, durations[i], requestErr) - } - continue - } - if responses[i] == nil { - failedRequestCount++ - if firstRequestError == nil { - firstRequestError = fmt.Errorf("concurrent multi-resource decision request %d returned no response", i+1) - } - continue - } - if i > 0 && !proto.Equal(responses[0], responses[i]) { - failedRequestCount++ - if firstRequestError == nil { - firstRequestError = fmt.Errorf("concurrent multi-resource decision request %d returned a different response", i+1) - } - } - } - - scenarioContext.TestSuiteContext.Logger.Info( - "authorization v2 concurrent multi-resource performance", - slog.Int("concurrency", concurrency), - slog.Duration("wall_duration", wallElapsed), - slog.Duration("median_request_duration", medianRequestDuration), - slog.Duration("p95_request_duration", p95RequestDuration), - slog.Duration("maximum_request_duration", maximumRequestDuration), - slog.Duration("maximum_duration", limit), - slog.Int("resource_count", len(resources)), - slog.Int("failed_request_count", failedRequestCount), - ) - - if firstRequestError != nil { - return ctx, firstRequestError - } - if maximumRequestDuration > limit { - return ctx, fmt.Errorf("slowest concurrent multi-resource decision took %s, exceeding the %s limit", maximumRequestDuration, limit) - } - - scenarioContext.RecordObject(multiDecisionResponseKey, responses[0]) - scenarioContext.RecordObject(decisionResponse, responses[0]) - scenarioContext.RecordObject("resourceFQNMap", resourceFQNMap) - return ctx, nil -} - func (s *AuthorizationServiceStepDefinitions) iSendAMultiResourceDecisionRequestForEntityChainForActionOnResourcesWithNoFulfillableObligations(ctx context.Context, entityChainID string, action string, tbl *godog.Table) (context.Context, error) { return s.iSendAMultiResourceDecisionRequestForEntityChainForActionOnResourcesWithFulfillableObligations(ctx, entityChainID, action, "[]", tbl) } @@ -702,6 +546,7 @@ func (s *AuthorizationServiceStepDefinitions) theDecisionResponseForResourceShou } func RegisterAuthorizationStepDefinitions(ctx *godog.ScenarioContext) { + ctx.Step(`^I exercise the authorization cases with (\d+) concurrent requests each, seed (\d+), and request timeout "([^"]*)" for attribute "([^"]*)":$`, exerciseAuthorizationCases) stepDefinitions := AuthorizationServiceStepDefinitions{} ctx.Step(`^there is a "([^"]*)" subject entity with value "([^"]*)" and referenced as "([^"]*)"$`, stepDefinitions.thereIsASubjectEntityWithValueAndReferencedAs) ctx.Step(`^there is a claims subject entity referenced as "([^"]*)" with claims:$`, stepDefinitions.thereIsAClaimsSubjectEntityReferencedAsWithClaims) @@ -712,8 +557,6 @@ func RegisterAuthorizationStepDefinitions(ctx *godog.ScenarioContext) { ctx.Step(`^I send a decision request for entity chain "([^"]*)" for "([^"]*)" action on resource "([^"]*)" with fulfillable obligations "([^"]*)"$`, stepDefinitions.iSendADecisionRequestForEntityChainForActionOnResourceWithFulfillableObligations) ctx.Step(`^I send a decision request for entity chain "([^"]*)" for "([^"]*)" action on resource "([^"]*)" with no fulfillable obligations$`, stepDefinitions.iSendADecisionRequestForEntityChainForActionOnResourceWithNoFulfillableObligations) ctx.Step(`^I send a multi-resource decision request for entity chain "([^"]*)" for "([^"]*)" action on resources:$`, stepDefinitions.iSendAMultiResourceDecisionRequestForEntityChainForActionOnResources) - ctx.Step(`^I send a multi-resource decision request for entity chain "([^"]*)" for "([^"]*)" action on resources within "([^"]*)":$`, stepDefinitions.iSendAMultiResourceDecisionRequestWithin) - ctx.Step(`^I send (\d+) concurrent multi-resource decision requests for entity chain "([^"]*)" for "([^"]*)" action on resources each within "([^"]*)":$`, stepDefinitions.iSendConcurrentMultiResourceDecisionRequestsWithin) ctx.Step(`^I send a multi-resource decision request for entity chain "([^"]*)" for "([^"]*)" action on resources with no fulfillable obligations:$`, stepDefinitions.iSendAMultiResourceDecisionRequestForEntityChainForActionOnResourcesWithNoFulfillableObligations) ctx.Step(`^I send a multi-resource decision request for entity chain "([^"]*)" for "([^"]*)" action on resources with fulfillable obligations "([^"]*)":$`, stepDefinitions.iSendAMultiResourceDecisionRequestForEntityChainForActionOnResourcesWithFulfillableObligations) ctx.Step(`^I should get a "([^"]*)" decision response$`, stepDefinitions.iShouldGetADecisionResponse) diff --git a/tests-bdd/cukes/steps_authorization_scale.go b/tests-bdd/cukes/steps_authorization_scale.go new file mode 100644 index 0000000000..147d7bbcb5 --- /dev/null +++ b/tests-bdd/cukes/steps_authorization_scale.go @@ -0,0 +1,210 @@ +package cukes + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "math/rand/v2" + "slices" + "strings" + "sync" + "time" + + "github.com/cucumber/godog" + authz "github.com/opentdf/platform/protocol/go/authorization/v2" + "github.com/opentdf/platform/protocol/go/policy" + "google.golang.org/protobuf/proto" +) + +const authorizationPerformanceMarker = "AUTHZ_PERFORMANCE " + +type authorizationScaleCase struct { + name string + entity string + action string + values []string + expected map[string]authz.Decision +} + +type authorizationPerformanceResult struct { + Case string `json:"case"` + Seed int `json:"seed"` + Concurrency int `json:"concurrency"` + Resources int `json:"resources"` + Wall time.Duration `json:"wall_ns"` + Median time.Duration `json:"median_ns"` + P95 time.Duration `json:"p95_ns"` + Maximum time.Duration `json:"maximum_ns"` + Timeout time.Duration `json:"timeout_ns"` + Failures int `json:"failures"` +} + +func parseAuthorizationScaleCases(table *godog.Table) ([]authorizationScaleCase, error) { + headers := []string{"case", "entity", "action", valuesKey, "expected"} + if table == nil || len(table.Rows) < 2 || len(table.Rows[0].Cells) != len(headers) { + return nil, errors.New("authorization case table requires case, entity, action, values, expected columns") + } + for i, header := range headers { + if table.Rows[0].Cells[i].Value != header { + return nil, fmt.Errorf("expected column %q", header) + } + } + cases := make([]authorizationScaleCase, 0, len(table.Rows)-1) + names := make(map[string]bool) + for _, row := range table.Rows[1:] { + if len(row.Cells) != len(headers) { + return nil, errors.New("authorization case row has incorrect column count") + } + item := authorizationScaleCase{ + name: strings.TrimSpace(row.Cells[0].Value), entity: strings.TrimSpace(row.Cells[1].Value), + action: strings.TrimSpace(row.Cells[2].Value), values: strings.Split(row.Cells[3].Value, ","), + expected: make(map[string]authz.Decision), + } + if item.name == "" || names[item.name] || item.entity == "" || item.action == "" { + return nil, errors.New("cases require unique names, entities, and actions") + } + names[item.name] = true + expected := strings.Split(row.Cells[4].Value, ",") + if len(item.values) != len(expected) { + return nil, fmt.Errorf("case %s has mismatched values and expectations", item.name) + } + for i, value := range item.values { + item.values[i] = strings.TrimSpace(value) + if item.values[i] == "" { + return nil, fmt.Errorf("case %s has an empty value", item.name) + } + decision, ok := authz.Decision_value["DECISION_"+strings.TrimSpace(expected[i])] + if !ok || (authz.Decision(decision) != authz.Decision_DECISION_PERMIT && authz.Decision(decision) != authz.Decision_DECISION_DENY) { + return nil, fmt.Errorf("case %s requires explicit PERMIT or DENY expectations", item.name) + } + item.expected[fmt.Sprintf("resource%d", i)] = authz.Decision(decision) + } + cases = append(cases, item) + } + return cases, nil +} + +func validateScaleDecision(response *authz.GetDecisionMultiResourceResponse, expected map[string]authz.Decision) error { + if response == nil || len(response.GetResourceDecisions()) != len(expected) { + return errors.New("unexpected resource decision count") + } + seen := make(map[string]bool, len(expected)) + for _, decision := range response.GetResourceDecisions() { + id := decision.GetEphemeralResourceId() + want, ok := expected[id] + if !ok || seen[id] { + return fmt.Errorf("unexpected or duplicate resource decision %q", id) + } + seen[id] = true + if decision.GetDecision() != want { + return fmt.Errorf("resource %s: expected %s, got %s", id, want, decision.GetDecision()) + } + if len(decision.GetRequiredObligations()) != 0 { + return fmt.Errorf("resource %s returned unexpected obligations", id) + } + } + return nil +} + +func exerciseAuthorizationCases(ctx context.Context, concurrency, seed int, requestTimeout, attributeRef string, table *godog.Table) (context.Context, error) { + if concurrency < 1 || seed < 0 { + return ctx, errors.New("concurrency must be positive and seed nonnegative") + } + timeout, err := time.ParseDuration(requestTimeout) + if err != nil || timeout <= 0 { + return ctx, fmt.Errorf("invalid duration %q", requestTimeout) + } + cases, err := parseAuthorizationScaleCases(table) + if err != nil { + return ctx, err + } + scenario := GetPlatformScenarioContext(ctx) + attribute, ok := scenario.GetObject(attributeRef).(*policy.Attribute) + if !ok || attribute.GetFqn() == "" { + return ctx, fmt.Errorf("missing attribute %q", attributeRef) + } + // Shuffle all cases rather than sampling, so no expected path is omitted. + random := rand.New(rand.NewPCG(uint64(seed), uint64(concurrency))) //nolint:gosec // reproducible test order, not security randomness + random.Shuffle(len(cases), func(i, j int) { cases[i], cases[j] = cases[j], cases[i] }) + var failures []error + for _, item := range cases { + chain, err := buildEntityChainFromIDs(scenario, item.entity) + if err != nil { + return ctx, err + } + request := &authz.GetDecisionMultiResourceRequest{ + EntityIdentifier: &authz.EntityIdentifier{Identifier: &authz.EntityIdentifier_EntityChain{EntityChain: chain}}, + Action: &policy.Action{Name: item.action}, + } + for i, value := range item.values { + request.Resources = append(request.Resources, &authz.Resource{ + EphemeralId: fmt.Sprintf("resource%d", i), + Resource: &authz.Resource_AttributeValues_{AttributeValues: &authz.Resource_AttributeValues{ + Fqns: []string{attribute.GetFqn() + "/value/" + value}, + }}, + }) + } + result, err := runAuthorizationScaleCase(ctx, scenario, item, request, concurrency, seed, timeout, random) + encoded, encodeErr := json.Marshal(result) + if encodeErr != nil { + return ctx, encodeErr + } + // A structured record survives both console and Go test JSON output formats. + fmt.Println(authorizationPerformanceMarker + string(encoded)) //nolint:forbidigo // structured CI record, independent of the configured log handler + if err != nil { + failures = append(failures, fmt.Errorf("case %s: %w", item.name, err)) + } + } + return ctx, errors.Join(failures...) +} + +func runAuthorizationScaleCase(ctx context.Context, scenario *PlatformScenarioContext, item authorizationScaleCase, request *authz.GetDecisionMultiResourceRequest, concurrency, seed int, timeout time.Duration, random *rand.Rand) (authorizationPerformanceResult, error) { + result := authorizationPerformanceResult{Case: item.name, Seed: seed, Concurrency: concurrency, Resources: len(item.values), Timeout: timeout} + // Bound request completion without treating the timeout as a latency baseline. + requestCtx, cancel := context.WithTimeout(ctx, timeout) + defer cancel() + requests := make([]*authz.GetDecisionMultiResourceRequest, concurrency) + for i := range requests { + requests[i] = proto.CloneOf(request) + resources := requests[i].GetResources() + random.Shuffle(len(resources), func(i, j int) { resources[i], resources[j] = resources[j], resources[i] }) + } + durations := make([]time.Duration, concurrency) + requestErrors := make([]error, concurrency) + start := make(chan struct{}) + var workers sync.WaitGroup + for i := range requests { + workers.Go(func() { + <-start + started := time.Now() + response, err := scenario.SDK.AuthorizationV2.GetDecisionMultiResource(requestCtx, requests[i]) + durations[i] = time.Since(started) + if err == nil { + err = validateScaleDecision(response, item.expected) + } + requestErrors[i] = err + }) + } + started := time.Now() + close(start) + workers.Wait() + result.Wall = time.Since(started) + for _, err := range requestErrors { + if err != nil { + result.Failures++ + } + } + slices.Sort(durations) + result.Median = durations[(concurrency-1)/2] + result.P95 = durations[(95*concurrency-1)/100] + result.Maximum = durations[concurrency-1] + var failure error + for _, err := range requestErrors { + if err != nil { + failure = err + break + } + } + return result, failure +} diff --git a/tests-bdd/cukes/steps_authorization_scale_test.go b/tests-bdd/cukes/steps_authorization_scale_test.go new file mode 100644 index 0000000000..c764705a1f --- /dev/null +++ b/tests-bdd/cukes/steps_authorization_scale_test.go @@ -0,0 +1,49 @@ +package cukes + +import ( + "testing" + + authz "github.com/opentdf/platform/protocol/go/authorization/v2" + "github.com/stretchr/testify/require" + "google.golang.org/protobuf/proto" +) + +func TestScaleDecisionValidatesEachResourceRegardlessOfOrder(t *testing.T) { + expected := map[string]authz.Decision{"resource0": authz.Decision_DECISION_PERMIT, "resource1": authz.Decision_DECISION_DENY} + valid := &authz.GetDecisionMultiResourceResponse{ResourceDecisions: []*authz.ResourceDecision{ + {EphemeralResourceId: "resource1", Decision: authz.Decision_DECISION_DENY}, + {EphemeralResourceId: "resource0", Decision: authz.Decision_DECISION_PERMIT}, + }} + require.NoError(t, validateScaleDecision(valid, expected)) + tests := []struct { + name string + change func(*authz.GetDecisionMultiResourceResponse) + }{ + {"incorrect deny", func(r *authz.GetDecisionMultiResourceResponse) { + r.GetResourceDecisions()[0].Decision = authz.Decision_DECISION_PERMIT + }}, + {"duplicate resource", func(r *authz.GetDecisionMultiResourceResponse) { + r.GetResourceDecisions()[0].EphemeralResourceId = "resource0" + }}, + {"unknown resource", func(r *authz.GetDecisionMultiResourceResponse) { + r.GetResourceDecisions()[0].EphemeralResourceId = "unknown" + }}, + {"missing resource", func(r *authz.GetDecisionMultiResourceResponse) { r.ResourceDecisions = r.GetResourceDecisions()[:1] }}, + {"unexpected obligations", func(r *authz.GetDecisionMultiResourceResponse) { + r.GetResourceDecisions()[1].RequiredObligations = []string{"unexpected"} + }}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + response := proto.CloneOf(valid) + tc.change(response) + require.Error(t, validateScaleDecision(response, expected)) + }) + } + require.Error(t, validateScaleDecision(nil, expected)) +} + +func TestScaleCasesRejectMissingTable(t *testing.T) { + _, err := parseAuthorizationScaleCases(nil) + require.Error(t, err) +} diff --git a/tests-bdd/cukes/steps_resourcemappings_scale.go b/tests-bdd/cukes/steps_resourcemappings_scale.go new file mode 100644 index 0000000000..4584075b88 --- /dev/null +++ b/tests-bdd/cukes/steps_resourcemappings_scale.go @@ -0,0 +1,34 @@ +package cukes + +import ( + "context" + "fmt" + + "github.com/cucumber/godog" + "github.com/opentdf/platform/protocol/go/policy/resourcemapping" +) + +func RegisterResourceMappingScaleSteps(ctx *godog.ScenarioContext) { + ctx.Step(`^I create (\d+) resource mappings for attribute "([^"]*)" in namespace "([^"]*)"$`, createScaleResourceMappings) +} + +func createScaleResourceMappings(ctx context.Context, count int, attributeRef, namespaceRef string) (context.Context, error) { + scenario, attribute, namespace, err := scaleMappingInputs(ctx, count, attributeRef, namespaceRef) + if err != nil { + return ctx, err + } + err = createScaleMappings(ctx, count, func(ctx context.Context, index int) error { + response, err := scenario.SDK.ResourceMapping.CreateResourceMapping(ctx, &resourcemapping.CreateResourceMappingRequest{ + AttributeValueId: attribute.GetValues()[index].GetId(), NamespaceId: namespace, + Terms: []string{fmt.Sprintf("resource-%04d", index)}, + }) + if err != nil { + return fmt.Errorf("create resource mapping %d: %w", index, err) + } + if response.GetResourceMapping().GetId() == "" { + return fmt.Errorf("resource mapping %d returned no identity", index) + } + return nil + }) + return ctx, err +} diff --git a/tests-bdd/cukes/steps_subjectmappings.go b/tests-bdd/cukes/steps_subjectmappings.go index 5c03f83d04..56066c6e84 100644 --- a/tests-bdd/cukes/steps_subjectmappings.go +++ b/tests-bdd/cukes/steps_subjectmappings.go @@ -4,21 +4,15 @@ import ( "context" "errors" "fmt" - "log/slog" "strings" "github.com/cucumber/godog" - "github.com/google/uuid" - "github.com/jackc/pgx/v5" - "github.com/jackc/pgx/v5/pgxpool" "github.com/opentdf/platform/protocol/go/policy" "github.com/opentdf/platform/protocol/go/policy/subjectmapping" ) type SubjectMappingsStepDefinitions struct{} -const policyDatabaseSchema = "otdf_policy" - func (s *SubjectMappingsStepDefinitions) iSendARequestToCreateSubjectMapping(ctx context.Context, tbl *godog.Table) (context.Context, error) { scenarioContext := GetPlatformScenarioContext(ctx) scenarioContext.ClearError() @@ -226,125 +220,13 @@ func (s *SubjectMappingsStepDefinitions) iSendARequestToCreateSubjectMappingForE return ctx, nil } -// seedSubjectAndResourceMappingsAtScale uses the scenario database for fixture setup so the -// end-to-end test spends its measured time in authorization, not in thousands of setup requests. -// The decision itself still goes through the public v2 API and the running platform container. -func (s *SubjectMappingsStepDefinitions) seedSubjectAndResourceMappingsAtScale(ctx context.Context, expectedSubjectMappings int, attributeRef, conditionSetRef, namespaceRef, action string, resourceMappingCount int) (context.Context, error) { - scenarioContext := GetPlatformScenarioContext(ctx) - scenarioContext.ClearError() - - attr, ok := scenarioContext.GetObject(strings.TrimSpace(attributeRef)).(*policy.Attribute) - if !ok { - return ctx, fmt.Errorf("unable to get attribute for %s", attributeRef) - } - if len(attr.GetValues()) != expectedSubjectMappings { - return ctx, fmt.Errorf("attribute %s has %d values, expected %d", attributeRef, len(attr.GetValues()), expectedSubjectMappings) - } - if resourceMappingCount < 0 || resourceMappingCount > len(attr.GetValues()) { - return ctx, fmt.Errorf("resource mapping count %d must be between 0 and %d", resourceMappingCount, len(attr.GetValues())) - } - scs, ok := scenarioContext.GetObject(strings.TrimSpace(conditionSetRef)).(*policy.SubjectConditionSet) - if !ok { - return ctx, fmt.Errorf("unable to get condition set for %s", conditionSetRef) - } - namespaceID, ok := scenarioContext.GetObject(strings.TrimSpace(namespaceRef)).(string) - if !ok { - return ctx, fmt.Errorf("unable to get namespace id for %s", namespaceRef) - } - localPlatformGlue, ok := (*scenarioContext.TestSuiteContext.PlatformGlue).(*LocalDevPlatformGlue) - if !ok { - return ctx, errors.New("failed to load local platform glue") - } - - pgConfig, err := pgxpool.ParseConfig(fmt.Sprintf( - "postgres://postgres:changeme@localhost:%d/%s?sslmode=prefer", - localPlatformGlue.Options.postgresPort, - scenarioContext.ScenarioOptions.DatabaseName, - )) - if err != nil { - return ctx, fmt.Errorf("parse scenario database config: %w", err) - } - pool, err := pgxpool.NewWithConfig(ctx, pgConfig) - if err != nil { - return ctx, fmt.Errorf("connect to scenario database: %w", err) - } - defer pool.Close() - - tx, err := pool.Begin(ctx) - if err != nil { - return ctx, fmt.Errorf("begin scale fixture transaction: %w", err) - } - defer func() { _ = tx.Rollback(ctx) }() - - var actionID string - err = tx.QueryRow(ctx, ` - SELECT id - FROM `+policyDatabaseSchema+`.actions - WHERE name = $1 AND (namespace_id = $2 OR namespace_id IS NULL) - ORDER BY (namespace_id = $2) DESC - LIMIT 1 - `, strings.ToLower(strings.TrimSpace(action)), namespaceID).Scan(&actionID) - if err != nil { - return ctx, fmt.Errorf("resolve action %q: %w", action, err) - } - - subjectMappingRows := make([][]any, 0, expectedSubjectMappings) - actionRows := make([][]any, 0, expectedSubjectMappings) - resourceMappingRows := make([][]any, 0, resourceMappingCount) - for i, value := range attr.GetValues() { - mappingID := uuid.NewString() - subjectMappingRows = append(subjectMappingRows, []any{mappingID, value.GetId(), scs.GetId(), namespaceID}) - actionRows = append(actionRows, []any{mappingID, actionID}) - if i < resourceMappingCount { - resourceMappingRows = append(resourceMappingRows, []any{ - uuid.NewString(), - value.GetId(), - []string{fmt.Sprintf("resource-%04d", i)}, - namespaceID, - }) - } - } - - inserted, err := tx.CopyFrom(ctx, pgx.Identifier{policyDatabaseSchema, "subject_mappings"}, []string{"id", "attribute_value_id", "subject_condition_set_id", namespaceIDKey}, pgx.CopyFromRows(subjectMappingRows)) - if err != nil { - return ctx, fmt.Errorf("seed subject mappings: %w", err) - } - if inserted != int64(expectedSubjectMappings) { - return ctx, fmt.Errorf("seeded %d subject mappings, expected %d", inserted, expectedSubjectMappings) - } - inserted, err = tx.CopyFrom(ctx, pgx.Identifier{policyDatabaseSchema, "subject_mapping_actions"}, []string{"subject_mapping_id", "action_id"}, pgx.CopyFromRows(actionRows)) - if err != nil { - return ctx, fmt.Errorf("seed subject mapping actions: %w", err) - } - if inserted != int64(expectedSubjectMappings) { - return ctx, fmt.Errorf("seeded %d subject mapping actions, expected %d", inserted, expectedSubjectMappings) - } - inserted, err = tx.CopyFrom(ctx, pgx.Identifier{policyDatabaseSchema, "resource_mappings"}, []string{"id", "attribute_value_id", "terms", namespaceIDKey}, pgx.CopyFromRows(resourceMappingRows)) - if err != nil { - return ctx, fmt.Errorf("seed resource mappings: %w", err) - } - if inserted != int64(resourceMappingCount) { - return ctx, fmt.Errorf("seeded %d resource mappings, expected %d", inserted, resourceMappingCount) - } - if err := tx.Commit(ctx); err != nil { - return ctx, fmt.Errorf("commit scale fixtures: %w", err) - } - - scenarioContext.TestSuiteContext.Logger.Info( - "seeded authorization scale fixture", - slog.Int("subject_mapping_count", expectedSubjectMappings), - slog.Int("resource_mapping_count", resourceMappingCount), - ) - return ctx, nil -} - func RegisterSubjectMappingsStepsDefinitions(ctx *godog.ScenarioContext) { subjectMappingStepDefinitions := &SubjectMappingsStepDefinitions{} + ctx.Step(`^I create (\d+) subject mappings for attribute "([^"]*)" using condition set "([^"]*)" with action "([^"]*)"$`, subjectMappingStepDefinitions.createScaleSubjectMappings) ctx.Step(`a condition group referenced as "([^"]*)" with an "([^"]*)" operator with conditions:$`, subjectMappingStepDefinitions.aConditionGroup) ctx.Step(`^a subject set referenced as "([^"]*)" containing the condition groups "([^"]*)"$`, subjectMappingStepDefinitions.aSubjectSet) ctx.Step(`^I send a request to create a subject condition set referenced as "([^"]*)" containing subject sets "([^"]*)"$`, subjectMappingStepDefinitions.iSendARequestToCreateSubjectConditionSet) ctx.Step(`^I send a request to create a subject condition set referenced as "([^"]*)" in namespace "([^"]*)" containing subject sets "([^"]*)"$`, subjectMappingStepDefinitions.iSendARequestToCreateSubjectConditionSetInNamespace) ctx.Step(`^I send a request to create a subject mapping with:$`, subjectMappingStepDefinitions.iSendARequestToCreateSubjectMapping) ctx.Step(`^I send a request to create a subject mapping for every value of attribute "([^"]*)" using condition set "([^"]*)" with actions "([^"]*)"$`, subjectMappingStepDefinitions.iSendARequestToCreateSubjectMappingForEveryAttributeValue) - ctx.Step(`^the policy database contains (\d+) subject mappings for attribute "([^"]*)" using condition set "([^"]*)" in namespace "([^"]*)" with action "([^"]*)" and (\d+) resource mappings$`, subjectMappingStepDefinitions.seedSubjectAndResourceMappingsAtScale) } diff --git a/tests-bdd/cukes/steps_subjectmappings_scale.go b/tests-bdd/cukes/steps_subjectmappings_scale.go new file mode 100644 index 0000000000..3a6ba0705e --- /dev/null +++ b/tests-bdd/cukes/steps_subjectmappings_scale.go @@ -0,0 +1,35 @@ +package cukes + +import ( + "context" + "fmt" + + "github.com/opentdf/platform/protocol/go/policy" + "github.com/opentdf/platform/protocol/go/policy/subjectmapping" +) + +func (s *SubjectMappingsStepDefinitions) createScaleSubjectMappings(ctx context.Context, count int, attributeRef, conditionSetRef, action string) (context.Context, error) { + scenario := GetPlatformScenarioContext(ctx) + attribute, ok := scenario.GetObject(attributeRef).(*policy.Attribute) + if !ok || count <= 0 || count > len(attribute.GetValues()) { + return ctx, fmt.Errorf("attribute %q must contain at least %d values", attributeRef, count) + } + conditionSet, ok := scenario.GetObject(conditionSetRef).(*policy.SubjectConditionSet) + if !ok { + return ctx, fmt.Errorf("missing condition set %q", conditionSetRef) + } + err := createScaleMappings(ctx, count, func(ctx context.Context, index int) error { + response, err := scenario.SDK.SubjectMapping.CreateSubjectMapping(ctx, &subjectmapping.CreateSubjectMappingRequest{ + AttributeValueId: attribute.GetValues()[index].GetId(), ExistingSubjectConditionSetId: conditionSet.GetId(), + Actions: GetActionsFromValues(&action, nil), + }) + if err != nil { + return fmt.Errorf("create subject mapping %d: %w", index, err) + } + if response.GetSubjectMapping().GetId() == "" { + return fmt.Errorf("subject mapping %d returned no identity", index) + } + return nil + }) + return ctx, err +} diff --git a/tests-bdd/features/authorization-v2-subject-mapping-performance.feature b/tests-bdd/features/authorization-v2-subject-mapping-performance.feature index 12a128c2ac..23e15c9be0 100644 --- a/tests-bdd/features/authorization-v2-subject-mapping-performance.feature +++ b/tests-bdd/features/authorization-v2-subject-mapping-performance.feature @@ -3,35 +3,40 @@ Feature: v2 multi-resource decisions at large policy scale GetDecisionMultiResource must remain responsive when the policy database contains the subject-mapping and resource-mapping cardinality from the reported regression. Fixture setup is not timed. The measured operation is a synchronized group of requests to the public v2 - authorization endpoint. + authorization endpoint. Every case runs at every concurrency level. A fixed seed + shuffles case and resource order reproducibly. The extra value has no subject mapping. + Subject mappings use the default unnamespaced policy path; attribute and resource mappings + retain their namespace. This avoids repeatedly validating the entire attribute during setup. + Latency is reported without a performance gate until a baseline is established. + Incorrect decisions, request errors, and request timeouts fail the scenario. - Scenario Outline: Concurrent multi-resource decisions complete within five seconds with 6011 subject mappings at concurrency + Scenario Outline: Varied multi-resource decisions at concurrency Given a user exists with username "scale-user" and email "scale-user@example.com" and the following attributes: | name | value | | department | ["engineering"] | + And a user exists with username "other-user" and email "other-user@example.com" and the following attributes: + | name | value | + | department | ["sales"] | And an empty local platform And I submit a request to create a namespace with name "scale.example" and reference id "scale_ns" - And I send a request to create an attribute referenced as "scale_attr" in namespace "scale_ns" named "access-level" with rule "anyOf" and 6011 generated values in batches of 25 + And I send a request to create an attribute referenced as "scale_attr" in namespace "scale_ns" named "access-level" with rule "anyOf" and 6012 generated values in batches of 25 Then the response should be successful And a condition group referenced as "scale_condition" with an "or" operator with conditions: | selector_value | operator | values | | .attributes.department[] | in | engineering | And a subject set referenced as "scale_subject_set" containing the condition groups "scale_condition" - And I send a request to create a subject condition set referenced as "scale_condition_set" in namespace "scale_ns" containing subject sets "scale_subject_set" + And I send a request to create a subject condition set referenced as "scale_condition_set" containing subject sets "scale_subject_set" Then the response should be successful - And the policy database contains 6011 subject mappings for attribute "scale_attr" using condition set "scale_condition_set" in namespace "scale_ns" with action "read" and 6000 resource mappings + And I create 6011 subject mappings for attribute "scale_attr" using condition set "scale_condition_set" with action "read" + And I create 6000 resource mappings for attribute "scale_attr" in namespace "scale_ns" And there is a "user_name" subject entity with value "scale-user" and referenced as "scale-user" - When I send concurrent multi-resource decision requests for entity chain "scale-user" for "read" action on resources each within "5s": - | resource | - | https://scale.example/attr/access-level/value/v0000 | - | https://scale.example/attr/access-level/value/v3005 | - | https://scale.example/attr/access-level/value/v6010 | - Then the response should be successful - And I should get 3 decision responses - And the multi-resource decision should be "PERMIT" - And the decision response for resource "https://scale.example/attr/access-level/value/v0000" should be "PERMIT" - And the decision response for resource "https://scale.example/attr/access-level/value/v3005" should be "PERMIT" - And the decision response for resource "https://scale.example/attr/access-level/value/v6010" should be "PERMIT" + And there is a "user_name" subject entity with value "other-user" and referenced as "other-user" + When I exercise the authorization cases with concurrent requests each, seed 4625, and request timeout "30s" for attribute "scale_attr": + | case | entity | action | values | expected | + | allowed_read | scale-user | read | v0000,v3005,v6010 | PERMIT,PERMIT,PERMIT | + | denied_action | scale-user | write | v0000,v3005,v6010 | DENY,DENY,DENY | + | denied_user | other-user | read | v0000,v3005,v6010 | DENY,DENY,DENY | + | mixed_values | scale-user | read | v0000,v6011,v6010 | PERMIT,DENY,PERMIT | @concurrency-1 Examples: One request diff --git a/tests-bdd/platform_test.go b/tests-bdd/platform_test.go index 9df363fd23..bd0b9cb946 100644 --- a/tests-bdd/platform_test.go +++ b/tests-bdd/platform_test.go @@ -110,6 +110,7 @@ func runTests() int { cukes.RegisterSmokeStepDefinitions(ctx, platformCukesContext) cukes.RegisterAuthorizationStepDefinitions(ctx) cukes.RegisterSubjectMappingsStepsDefinitions(ctx) + cukes.RegisterResourceMappingScaleSteps(ctx) cukes.RegisterDynamicValueMappingsStepDefinitions(ctx) cukes.RegisterDirectEntitlementsStepDefinitions(ctx) cukes.RegisterRegisteredResourcesStepDefinitions(ctx) From 74e173cda3e9ba01e42176fef7eb7ad6c6da4307 Mon Sep 17 00:00:00 2001 From: strantalis Date: Tue, 8 Sep 2026 22:51:10 -0400 Subject: [PATCH 08/10] test(authz): align scale server timeout with client deadline Signed-off-by: strantalis --- .../scripts/summarize-authz-performance.py | 9 +++-- .../test_summarize_authz_performance.py | 3 +- tests-bdd/cukes/resources/platform.template | 5 +++ tests-bdd/cukes/steps_authorization_scale.go | 2 + tests-bdd/cukes/steps_localplatform.go | 34 +++++++++++------ .../cukes/steps_localplatform_config_test.go | 37 +++++++++++++++++++ ...ion-v2-subject-mapping-performance.feature | 4 +- 7 files changed, 78 insertions(+), 16 deletions(-) create mode 100644 tests-bdd/cukes/steps_localplatform_config_test.go diff --git a/.github/scripts/summarize-authz-performance.py b/.github/scripts/summarize-authz-performance.py index a79b9919b7..944e557ce3 100644 --- a/.github/scripts/summarize-authz-performance.py +++ b/.github/scripts/summarize-authz-performance.py @@ -34,6 +34,8 @@ def read_results(text): raise ValueError("invalid dimensions") if result["failures"] > result["concurrency"]: raise ValueError("invalid failure count") + if not isinstance(result.get("first_error", ""), str): + raise ValueError("invalid error detail") results.append(result) except (ValueError, TypeError, AttributeError): malformed += 1 @@ -49,15 +51,16 @@ def render(text, outcome): lines = ["### Authorization v2 concurrency performance", "", f"BDD step outcome: **{outcome}**.", ""] if results: lines += [ - "| Case | Concurrency | Resources | Seed | Wall | Median | p95 | Maximum | Timeout | Failures | Requests |", - "| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | --- |", + "| Case | Concurrency | Resources | Seed | Wall | Median | p95 | Maximum | Timeout | Failures | Requests | First error |", + "| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | --- | --- |", ] for row in results: case = row["case"].replace("|", "\\|").replace("\n", " ").replace("\r", " ") status = "FAIL" if row["failures"] else "PASS" cells = [case, str(row["concurrency"]), str(row["resources"]), str(row["seed"])] cells += [milliseconds(row[key]) for key in ("wall_ns", "median_ns", "p95_ns", "maximum_ns", "timeout_ns")] - cells += [str(row["failures"]), status] + error = row.get("first_error", "").replace("|", "\\|").replace("\n", " ").replace("\r", " ") + cells += [str(row["failures"]), status, error] lines.append("| " + " | ".join(cells) + " |") lines += ["", f"Reported {len(results)} completed case batches. Missing measurements are not passes."] else: diff --git a/.github/scripts/test_summarize_authz_performance.py b/.github/scripts/test_summarize_authz_performance.py index 7e39142e62..c8de1b7e35 100644 --- a/.github/scripts/test_summarize_authz_performance.py +++ b/.github/scripts/test_summarize_authz_performance.py @@ -28,12 +28,13 @@ def test_console_and_json_records_have_identical_rendering(self): self.assertIn("130.00 ms | 140.00 ms | 30000.00 ms | 0 | PASS", console) def test_partial_failure_keeps_rows_without_gating_slow_requests(self): - text = self.record(case="denied_user", failures=2) + "\n" + self.record(case="allowed_read", maximum_ns=8000000000) + text = self.record(case="denied_user", failures=2, first_error="unavailable: unexpected EOF") + "\n" + self.record(case="allowed_read", maximum_ns=8000000000) rendered, errors = summary.render(text, "failure") self.assertEqual(errors, 0) self.assertIn("BDD step outcome: **failure**", rendered) self.assertIn("8000.00 ms | 30000.00 ms | 0 | PASS |", rendered) self.assertIn("| 2 | FAIL |", rendered) + self.assertIn("unavailable: unexpected EOF", rendered) self.assertIn("Latency is report-only", rendered) self.assertLess(rendered.index("allowed_read"), rendered.index("denied_user")) diff --git a/tests-bdd/cukes/resources/platform.template b/tests-bdd/cukes/resources/platform.template index 207eca2711..38bfb2dc8c 100644 --- a/tests-bdd/cukes/resources/platform.template +++ b/tests-bdd/cukes/resources/platform.template @@ -56,6 +56,11 @@ services: # enabled: false # refresh_interval: 30s server: +{{- if .httpWriteTimeout }} + # BDD-specific: allow the scale test's client deadline to control slow requests. + http: + writeTimeout: {{ .httpWriteTimeout }} +{{- end }} public_hostname: {{ .hostname }} tls: enabled: false diff --git a/tests-bdd/cukes/steps_authorization_scale.go b/tests-bdd/cukes/steps_authorization_scale.go index 147d7bbcb5..0d9b7305d0 100644 --- a/tests-bdd/cukes/steps_authorization_scale.go +++ b/tests-bdd/cukes/steps_authorization_scale.go @@ -38,6 +38,7 @@ type authorizationPerformanceResult struct { Maximum time.Duration `json:"maximum_ns"` Timeout time.Duration `json:"timeout_ns"` Failures int `json:"failures"` + FirstError string `json:"first_error,omitempty"` } func parseAuthorizationScaleCases(table *godog.Table) ([]authorizationScaleCase, error) { @@ -203,6 +204,7 @@ func runAuthorizationScaleCase(ctx context.Context, scenario *PlatformScenarioCo for _, err := range requestErrors { if err != nil { failure = err + result.FirstError = err.Error() break } } diff --git a/tests-bdd/cukes/steps_localplatform.go b/tests-bdd/cukes/steps_localplatform.go index 042133d1ef..100a3c690b 100644 --- a/tests-bdd/cukes/steps_localplatform.go +++ b/tests-bdd/cukes/steps_localplatform.go @@ -60,6 +60,7 @@ type platformStartOptions struct { kcProvisionPath *template.Template provisionDefaultPolicy bool ersConfig *ERSInlineConfig + httpWriteTimeout time.Duration } func (s *LocalPlatformStepDefinitions) aUser(ctx context.Context, username string, email string, attributes *godog.Table) (context.Context, error) { @@ -171,7 +172,7 @@ func (s *LocalPlatformStepDefinitions) commonLocalPlatform(ctx context.Context, if !exists { version = platformImageEnvironmentLocalImage } - platformConfigPath, err := createPlatformConfiguration(localPlatformOptions, scenarioContext.ScenarioOptions, version == debugVersion, options.platformProvisionPath, options.ersConfig) + platformConfigPath, err := createPlatformConfiguration(localPlatformOptions, scenarioContext.ScenarioOptions, version == debugVersion, options.platformProvisionPath, options.ersConfig, options.httpWriteTimeout) if err != nil { return ctx, err } @@ -288,6 +289,15 @@ func (s *LocalPlatformStepDefinitions) aEmptyLocalPlatform(ctx context.Context) return s.commonLocalPlatform(ctx, &platformStartOptions{kcProvisionPath: kt}) } +func (s *LocalPlatformStepDefinitions) aEmptyLocalPlatformWithHTTPWriteTimeout(ctx context.Context, duration string) (context.Context, error) { + timeout, err := time.ParseDuration(duration) + if err != nil || timeout <= 0 { + return ctx, fmt.Errorf("invalid HTTP write timeout %q", duration) + } + kt := template.Must(template.New("kc").Parse(keycloakBaseTemplate)) + return s.commonLocalPlatform(ctx, &platformStartOptions{kcProvisionPath: kt, httpWriteTimeout: timeout}) +} + func (s *LocalPlatformStepDefinitions) aDefaultLocalPlatform(ctx context.Context) (context.Context, error) { kt := template.Must(template.New("kc").Parse(keycloakBaseTemplate)) return s.commonLocalPlatform(ctx, &platformStartOptions{ @@ -531,7 +541,7 @@ func createPlatformComposeConfiguration(options *LocalDevOptions) (string, error } // createPlatformConfiguration generates a platform configuration from a go text template for platform option settings -func createPlatformConfiguration(options *LocalDevOptions, scenarioOptions *LocalDevScenarioOptions, devMode bool, platformTemplatePath *string, ersConfig *ERSInlineConfig) (string, error) { +func createPlatformConfiguration(options *LocalDevOptions, scenarioOptions *LocalDevScenarioOptions, devMode bool, platformTemplatePath *string, ersConfig *ERSInlineConfig, httpWriteTimeout time.Duration) (string, error) { tempFileName := path.Join(options.CukesDir, "opentdf.yaml") platformKeysDir := options.KeysDir pgHost := "localhost" @@ -550,15 +560,16 @@ func createPlatformConfiguration(options *LocalDevOptions, scenarioOptions *Loca t := template.Must(template.New("platform").Parse(templateSource)) var strBuffer bytes.Buffer if err := t.Execute(&strBuffer, map[string]any{ - "hostname": options.Hostname, - "kcPort": options.keycloakPort, - "platformPort": scenarioOptions.PlatformPort, - "pgPort": options.postgresPort, - "pgDatabase": scenarioOptions.DatabaseName, - "pgHost": pgHost, - "platformKeysDir": platformKeysDir, - "authRealm": scenarioOptions.KeycloakRealm, - "ldapPort": scenarioOptions.LDAPPort, + "hostname": options.Hostname, + "kcPort": options.keycloakPort, + "platformPort": scenarioOptions.PlatformPort, + "pgPort": options.postgresPort, + "pgDatabase": scenarioOptions.DatabaseName, + "pgHost": pgHost, + "platformKeysDir": platformKeysDir, + "authRealm": scenarioOptions.KeycloakRealm, + "ldapPort": scenarioOptions.LDAPPort, + "httpWriteTimeout": httpWriteTimeout, }); err != nil { return tempFileName, err } @@ -597,6 +608,7 @@ func RegisterLocalPlatformStepDefinitions(ctx *godog.ScenarioContext, x *Platfor PlatformCukesContext: x, } ctx.Step(`^an empty local platform$`, platformStepDefinitions.aEmptyLocalPlatform) + ctx.Step(`^an empty local platform with HTTP write timeout "([^"]*)"$`, platformStepDefinitions.aEmptyLocalPlatformWithHTTPWriteTimeout) ctx.Step(`^a default local platform$`, platformStepDefinitions.aDefaultLocalPlatform) ctx.Step(`^I use the platform as "([^"]*)"$`, platformStepDefinitions.iUseThePlatformAs) ctx.Step(`^a user exists with username "([^"]*)" and email "([^"]*)" and the following attributes:$`, platformStepDefinitions.aUser) diff --git a/tests-bdd/cukes/steps_localplatform_config_test.go b/tests-bdd/cukes/steps_localplatform_config_test.go new file mode 100644 index 0000000000..f221b33f5a --- /dev/null +++ b/tests-bdd/cukes/steps_localplatform_config_test.go @@ -0,0 +1,37 @@ +package cukes + +import ( + "os" + "testing" + "time" + + "github.com/stretchr/testify/require" + "gopkg.in/yaml.v2" +) + +func TestPlatformHTTPWriteTimeoutIsOptIn(t *testing.T) { + for _, timeout := range []time.Duration{0, 35 * time.Second} { + t.Run(timeout.String(), func(t *testing.T) { + configPath, err := createPlatformConfiguration( + &LocalDevOptions{CukesDir: t.TempDir()}, &LocalDevScenarioOptions{}, true, nil, nil, timeout, + ) + require.NoError(t, err) + data, err := os.ReadFile(configPath) + require.NoError(t, err) + var config struct { + Server struct { + HTTP *struct { + WriteTimeout string `yaml:"writeTimeout"` + } `yaml:"http"` + } `yaml:"server"` + } + require.NoError(t, yaml.Unmarshal(data, &config)) + if timeout == 0 { + require.Nil(t, config.Server.HTTP, "ordinary scenarios must retain the production default") + } else { + require.NotNil(t, config.Server.HTTP) + require.Equal(t, timeout.String(), config.Server.HTTP.WriteTimeout) + } + }) + } +} diff --git a/tests-bdd/features/authorization-v2-subject-mapping-performance.feature b/tests-bdd/features/authorization-v2-subject-mapping-performance.feature index 23e15c9be0..527e6c6c54 100644 --- a/tests-bdd/features/authorization-v2-subject-mapping-performance.feature +++ b/tests-bdd/features/authorization-v2-subject-mapping-performance.feature @@ -9,6 +9,8 @@ Feature: v2 multi-resource decisions at large policy scale retain their namespace. This avoids repeatedly validating the entire attribute during setup. Latency is reported without a performance gate until a baseline is established. Incorrect decisions, request errors, and request timeouts fail the scenario. + The server write timeout exceeds the client deadline so slow completed responses + can be measured instead of being cut off by the default ten-second write timeout. Scenario Outline: Varied multi-resource decisions at concurrency Given a user exists with username "scale-user" and email "scale-user@example.com" and the following attributes: @@ -17,7 +19,7 @@ Feature: v2 multi-resource decisions at large policy scale And a user exists with username "other-user" and email "other-user@example.com" and the following attributes: | name | value | | department | ["sales"] | - And an empty local platform + And an empty local platform with HTTP write timeout "35s" And I submit a request to create a namespace with name "scale.example" and reference id "scale_ns" And I send a request to create an attribute referenced as "scale_attr" in namespace "scale_ns" named "access-level" with rule "anyOf" and 6012 generated values in batches of 25 Then the response should be successful From 10a45864cbcea7fe2cc2b16e289bfe6c81e0bc61 Mon Sep 17 00:00:00 2001 From: strantalis Date: Wed, 9 Sep 2026 14:39:23 -0400 Subject: [PATCH 09/10] test(authz): exercise mixed entitlement traffic under load Signed-off-by: strantalis --- .../scripts/summarize-authz-performance.py | 92 ++++--- .../test_summarize_authz_performance.py | 43 +++- tests-bdd/cukes/steps_authorization.go | 5 +- tests-bdd/cukes/steps_authorization_scale.go | 242 +++++++++++------- .../cukes/steps_authorization_scale_policy.go | 117 +++++++++ .../cukes/steps_authorization_scale_test.go | 117 +++++++++ .../cukes/steps_subjectmappings_scale.go | 32 ++- ...ion-v2-subject-mapping-performance.feature | 120 ++++++--- 8 files changed, 595 insertions(+), 173 deletions(-) create mode 100644 tests-bdd/cukes/steps_authorization_scale_policy.go diff --git a/.github/scripts/summarize-authz-performance.py b/.github/scripts/summarize-authz-performance.py index 944e557ce3..29be63e92c 100644 --- a/.github/scripts/summarize-authz-performance.py +++ b/.github/scripts/summarize-authz-performance.py @@ -1,20 +1,49 @@ #!/usr/bin/env python3 -"""Render structured authorization measurements, including failed BDD runs.""" +"""Render mixed authorization load measurements, including failed BDD runs.""" import argparse import json from pathlib import Path MARKER = "AUTHZ_PERFORMANCE " NUMBERS = ( - "seed", "concurrency", "resources", "wall_ns", "median_ns", "p95_ns", - "maximum_ns", "timeout_ns", "failures", + "seed", "concurrency", "requests", "resources_requested", "wall_ns", + "median_ns", "p95_ns", "maximum_ns", "timeout_ns", "failures", ) +def validate_result(result): + if any(type(result.get(key)) is not int or result[key] < 0 for key in NUMBERS): + raise ValueError("invalid numeric field") + if not result["concurrency"] or result["requests"] < result["concurrency"] or not result["timeout_ns"]: + raise ValueError("invalid workload dimensions") + cases = result.get("cases") + if not isinstance(cases, list) or not cases: + raise ValueError("missing case results") + names = set() + for case in cases: + if any(not isinstance(case.get(key), str) or not case[key] for key in ("name", "user", "action")): + raise ValueError("invalid case identity") + if case["name"] in names: + raise ValueError("duplicate case") + names.add(case["name"]) + if any(type(case.get(key)) is not int or case[key] < 0 for key in ("requests", "failures")): + raise ValueError("invalid case counts") + if case["failures"] > case["requests"] or not isinstance(case.get("first_error", ""), str): + raise ValueError("invalid case failures") + resources, expected = case.get("resources"), case.get("expected") + if not isinstance(resources, list) or not resources or any(not isinstance(r, str) or not r for r in resources): + raise ValueError("invalid resources") + if not isinstance(expected, list) or len(resources) != len(expected) or any(d not in ("PERMIT", "DENY") for d in expected): + raise ValueError("invalid expectations") + if sum(c["requests"] for c in cases) != result["requests"] or sum(c["failures"] for c in cases) != result["failures"]: + raise ValueError("case counts do not match workload totals") + if sum(c["requests"] * len(c["resources"]) for c in cases) != result["resources_requested"]: + raise ValueError("resource counts do not match workload total") + + def read_results(text): results, malformed = [], 0 for line in text.splitlines(): - # Support both console logs and go test -json artifacts. if line.startswith("{"): try: output = json.loads(line).get("Output") @@ -26,49 +55,54 @@ def read_results(text): continue try: result = json.loads(line.split(MARKER, 1)[1]) - if not isinstance(result.get("case"), str) or not result["case"]: - raise ValueError("missing case") - if any(type(result.get(key)) is not int or result[key] < 0 for key in NUMBERS): - raise ValueError("invalid numeric field") - if not result["concurrency"] or not result["resources"] or not result["timeout_ns"]: - raise ValueError("invalid dimensions") - if result["failures"] > result["concurrency"]: - raise ValueError("invalid failure count") - if not isinstance(result.get("first_error", ""), str): - raise ValueError("invalid error detail") + validate_result(result) results.append(result) - except (ValueError, TypeError, AttributeError): + except (ValueError, TypeError, AttributeError, KeyError): malformed += 1 - return sorted(results, key=lambda row: (row["concurrency"], row["case"], row["seed"])), malformed + return sorted(results, key=lambda row: (row["concurrency"], row["seed"])), malformed def milliseconds(nanoseconds): - return f"{nanoseconds / 1_000_000:.2f} ms" + return f"{nanoseconds / 1_000_000:,.2f} ms" + + +def cell(text): + return str(text).replace("|", "\\|").replace("\n", " ").replace("\r", " ") def render(text, outcome): results, malformed = read_results(text) - lines = ["### Authorization v2 concurrency performance", "", f"BDD step outcome: **{outcome}**.", ""] + lines = ["### Authorization v2 concurrency performance", "", f"BDD step outcome: **{outcome}**.", "", + "**Workload:** workers continuously draw from the entitlement case pool. Each request independently selects a case; users, actions, and resources vary within the same load run.", "", + "**Performance: REPORT ONLY.** Correctness PASS means all requests completed with the expected resource decisions. Errors and request timeouts fail; no latency baseline is enforced.", ""] if results: lines += [ - "| Case | Concurrency | Resources | Seed | Wall | Median | p95 | Maximum | Timeout | Failures | Requests | First error |", - "| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | --- | --- |", + "| Concurrency | Requests | Cases used | Median | p95 | Maximum | Requests/s | Failures | Correctness |", + "| ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | --- |", ] for row in results: - case = row["case"].replace("|", "\\|").replace("\n", " ").replace("\r", " ") - status = "FAIL" if row["failures"] else "PASS" - cells = [case, str(row["concurrency"]), str(row["resources"]), str(row["seed"])] - cells += [milliseconds(row[key]) for key in ("wall_ns", "median_ns", "p95_ns", "maximum_ns", "timeout_ns")] - error = row.get("first_error", "").replace("|", "\\|").replace("\n", " ").replace("\r", " ") - cells += [str(row["failures"]), status, error] + used = sum(c["requests"] > 0 for c in row["cases"]) + rate = f"{row['requests'] / (row['wall_ns'] / 1_000_000_000):,.2f}" if row["wall_ns"] else "n/a" + cells = [str(row["concurrency"]), str(row["requests"]), f"{used}/{len(row['cases'])}"] + cells += [milliseconds(row[key]) for key in ("median_ns", "p95_ns", "maximum_ns")] + cells += [rate, str(row["failures"]), "FAIL" if row["failures"] else "PASS"] lines.append("| " + " | ".join(cells) + " |") - lines += ["", f"Reported {len(results)} completed case batches. Missing measurements are not passes."] + lines += ["", "Fixture setup is excluded. Latency covers the whole multi-resource request. Throughput includes failed requests; inspect correctness alongside it. Cases selected zero times remain visible below."] + for row in results: + lines += ["", "
", f"Case selection and failures at concurrency {row['concurrency']}", "", + f"Seed: {row['seed']}. Request timeout: {row['timeout_ns'] / 1_000_000_000:g} s. Load duration: {row['wall_ns'] / 1_000_000_000:,.2f} s. Resources requested: {row['resources_requested']:,}.", "", + "| Case | User | Action | Resources | Expected decisions | Selected | Failures | First error |", + "| --- | --- | --- | --- | --- | ---: | ---: | --- |"] + for case in row["cases"]: + cells = [case["name"], case["user"], case["action"], ", ".join(case["resources"]), ", ".join(case["expected"]), + case["requests"], case["failures"], case.get("first_error", "")] + lines.append("| " + " | ".join(cell(c) for c in cells) + " |") + lines += ["", "
"] else: lines.append("No authorization measurements were produced. Check BDD setup and logs; this is not a passing performance result.") if malformed: lines += ["", f"**Summary error: {malformed} malformed performance record(s).**"] - lines += ["", "Fixture setup is excluded. PASS means requests completed with the expected decisions. Failures include request errors, timeouts, and incorrect decisions. Latency is report-only until a baseline is established; the request timeout is not a latency target.", ""] - return "\n".join(lines), malformed + return "\n".join(lines) + "\n", malformed def main(): diff --git a/.github/scripts/test_summarize_authz_performance.py b/.github/scripts/test_summarize_authz_performance.py index c8de1b7e35..2837cf91b0 100644 --- a/.github/scripts/test_summarize_authz_performance.py +++ b/.github/scripts/test_summarize_authz_performance.py @@ -13,10 +13,15 @@ class SummaryTests(unittest.TestCase): def record(self, **changes): - row = dict(case="allowed_read", concurrency=50, resources=3, seed=4625, + row = dict(concurrency=50, requests=200, resources_requested=200, seed=4625, wall_ns=150000000, median_ns=100000000, p95_ns=130000000, - maximum_ns=140000000, timeout_ns=30000000000, failures=0) + maximum_ns=140000000, timeout_ns=30000000000, failures=0, + cases=[dict(name="allowed read", user="engineer", action="read", resources=["engineering"], expected=["PERMIT"], requests=100, failures=0), + dict(name="denied user", user="visitor", action="read", resources=["engineering"], expected=["DENY"], requests=100, failures=0)]) row.update(changes) + row["cases"][0]["failures"] = row["failures"] + if row["failures"]: + row["cases"][0]["first_error"] = "unavailable: unexpected EOF" return summary.MARKER + json.dumps(row) def test_console_and_json_records_have_identical_rendering(self): @@ -25,18 +30,30 @@ def test_console_and_json_records_have_identical_rendering(self): encoded, _ = summary.render(json.dumps({"Action": "output", "Output": record + "\n"}), "success") self.assertEqual(console, encoded) self.assertEqual(errors, 0) - self.assertIn("130.00 ms | 140.00 ms | 30000.00 ms | 0 | PASS", console) + self.assertIn("| 50 | 200 | 2/2 | 100.00 ms | 130.00 ms | 140.00 ms |", console) + self.assertIn("Request timeout: 30 s", console) + self.assertIn("
", console) + self.assertIn("PERFORMANCE", console.upper()) - def test_partial_failure_keeps_rows_without_gating_slow_requests(self): - text = self.record(case="denied_user", failures=2, first_error="unavailable: unexpected EOF") + "\n" + self.record(case="allowed_read", maximum_ns=8000000000) + def test_failure_keeps_completed_loads_and_does_not_gate_latency(self): + text = self.record(failures=2) + "\n" + self.record(concurrency=10, maximum_ns=8000000000) rendered, errors = summary.render(text, "failure") self.assertEqual(errors, 0) self.assertIn("BDD step outcome: **failure**", rendered) - self.assertIn("8000.00 ms | 30000.00 ms | 0 | PASS |", rendered) + self.assertIn("8,000.00 ms", rendered) + self.assertIn("| 0 | PASS |", rendered) self.assertIn("| 2 | FAIL |", rendered) self.assertIn("unavailable: unexpected EOF", rendered) - self.assertIn("Latency is report-only", rendered) - self.assertLess(rendered.index("allowed_read"), rendered.index("denied_user")) + self.assertIn("Performance: REPORT ONLY", rendered) + self.assertLess(rendered.index("| 10 |"), rendered.index("| 50 |")) + + def test_unsampled_cases_remain_visible(self): + record = json.loads(self.record().split(summary.MARKER)[1]) + record["cases"].append(dict(name="not selected", user="analyst", action="write", resources=["projects"], expected=["DENY"], requests=0, failures=0)) + rendered, errors = summary.render(summary.MARKER + json.dumps(record), "success") + self.assertEqual(errors, 0) + self.assertIn("| 2/3 |", rendered) + self.assertIn("not selected | analyst | write | projects | DENY | 0 | 0 |", rendered) def test_cli_handles_missing_log_after_setup_failure(self): with tempfile.TemporaryDirectory() as directory: @@ -48,14 +65,14 @@ def test_cli_handles_missing_log_after_setup_failure(self): self.assertIn("No authorization measurements", result.stdout) self.assertIn("BDD step outcome: **failure**", result.stdout) - def test_missing_and_malformed_measurements_are_not_passes(self): - rendered, errors = summary.render("setup failed", "failure") - self.assertIn("No authorization measurements", rendered) - self.assertEqual(errors, 0) + def test_malformed_and_inconsistent_counts_are_not_passes(self): rendered, errors = summary.render(self.record() + "\n" + summary.MARKER + "{}", "failure") self.assertEqual(errors, 1) self.assertIn("malformed performance record", rendered) - self.assertIn("allowed_read", rendered) + self.assertIn("allowed read", rendered) + for changes in (dict(requests=201), dict(resources_requested=201)): + _, errors = summary.render(self.record(**changes), "success") + self.assertEqual(errors, 1) if __name__ == "__main__": diff --git a/tests-bdd/cukes/steps_authorization.go b/tests-bdd/cukes/steps_authorization.go index ff813a4442..e9a1f63379 100644 --- a/tests-bdd/cukes/steps_authorization.go +++ b/tests-bdd/cukes/steps_authorization.go @@ -546,7 +546,10 @@ func (s *AuthorizationServiceStepDefinitions) theDecisionResponseForResourceShou } func RegisterAuthorizationStepDefinitions(ctx *godog.ScenarioContext) { - ctx.Step(`^I exercise the authorization cases with (\d+) concurrent requests each, seed (\d+), and request timeout "([^"]*)" for attribute "([^"]*)":$`, exerciseAuthorizationCases) + ctx.Step(`^the following scale attributes exist in namespace "([^"]*)":$`, createScaleAttributes) + ctx.Step(`^the following scale grants exist:$`, createScaleGrants) + ctx.Step(`^the following scale resources are defined:$`, defineScaleResources) + ctx.Step(`^I send (\d+) randomly selected authorization requests with concurrency (\d+), seed (\d+), and request timeout "([^"]*)":$`, exerciseAuthorizationLoad) stepDefinitions := AuthorizationServiceStepDefinitions{} ctx.Step(`^there is a "([^"]*)" subject entity with value "([^"]*)" and referenced as "([^"]*)"$`, stepDefinitions.thereIsASubjectEntityWithValueAndReferencedAs) ctx.Step(`^there is a claims subject entity referenced as "([^"]*)" with claims:$`, stepDefinitions.thereIsAClaimsSubjectEntityReferencedAsWithClaims) diff --git a/tests-bdd/cukes/steps_authorization_scale.go b/tests-bdd/cukes/steps_authorization_scale.go index 0d9b7305d0..670b59a864 100644 --- a/tests-bdd/cukes/steps_authorization_scale.go +++ b/tests-bdd/cukes/steps_authorization_scale.go @@ -20,60 +20,84 @@ import ( const authorizationPerformanceMarker = "AUTHZ_PERFORMANCE " type authorizationScaleCase struct { - name string - entity string - action string - values []string - expected map[string]authz.Decision + name, entity, action string + resources []string + expected map[string]authz.Decision + request *authz.GetDecisionMultiResourceRequest +} + +type authorizationCaseResult struct { + Name string `json:"name"` + User string `json:"user"` + Action string `json:"action"` + Resources []string `json:"resources"` + Expected []string `json:"expected"` + Requests int `json:"requests"` + Failures int `json:"failures"` + FirstError string `json:"first_error,omitempty"` } type authorizationPerformanceResult struct { - Case string `json:"case"` - Seed int `json:"seed"` - Concurrency int `json:"concurrency"` - Resources int `json:"resources"` - Wall time.Duration `json:"wall_ns"` - Median time.Duration `json:"median_ns"` - P95 time.Duration `json:"p95_ns"` - Maximum time.Duration `json:"maximum_ns"` - Timeout time.Duration `json:"timeout_ns"` - Failures int `json:"failures"` - FirstError string `json:"first_error,omitempty"` + Seed int `json:"seed"` + Concurrency int `json:"concurrency"` + Requests int `json:"requests"` + ResourcesRequested int `json:"resources_requested"` + Wall time.Duration `json:"wall_ns"` + Median time.Duration `json:"median_ns"` + P95 time.Duration `json:"p95_ns"` + Maximum time.Duration `json:"maximum_ns"` + Timeout time.Duration `json:"timeout_ns"` + Failures int `json:"failures"` + Cases []authorizationCaseResult `json:"cases"` } -func parseAuthorizationScaleCases(table *godog.Table) ([]authorizationScaleCase, error) { - headers := []string{"case", "entity", "action", valuesKey, "expected"} +func scaleTableRows(table *godog.Table, headers ...string) ([][]string, error) { if table == nil || len(table.Rows) < 2 || len(table.Rows[0].Cells) != len(headers) { - return nil, errors.New("authorization case table requires case, entity, action, values, expected columns") + return nil, fmt.Errorf("table requires columns: %s", strings.Join(headers, ", ")) } for i, header := range headers { - if table.Rows[0].Cells[i].Value != header { + if strings.TrimSpace(table.Rows[0].Cells[i].Value) != header { return nil, fmt.Errorf("expected column %q", header) } } - cases := make([]authorizationScaleCase, 0, len(table.Rows)-1) - names := make(map[string]bool) + rows := make([][]string, 0, len(table.Rows)-1) for _, row := range table.Rows[1:] { if len(row.Cells) != len(headers) { - return nil, errors.New("authorization case row has incorrect column count") + return nil, errors.New("incorrect table column count") } - item := authorizationScaleCase{ - name: strings.TrimSpace(row.Cells[0].Value), entity: strings.TrimSpace(row.Cells[1].Value), - action: strings.TrimSpace(row.Cells[2].Value), values: strings.Split(row.Cells[3].Value, ","), - expected: make(map[string]authz.Decision), + cells := make([]string, len(headers)) + for i, cell := range row.Cells { + cells[i] = strings.TrimSpace(cell.Value) + if cells[i] == "" { + return nil, fmt.Errorf("empty %s cell", headers[i]) + } } - if item.name == "" || names[item.name] || item.entity == "" || item.action == "" { - return nil, errors.New("cases require unique names, entities, and actions") + rows = append(rows, cells) + } + return rows, nil +} + +func parseAuthorizationScaleCases(table *godog.Table) ([]authorizationScaleCase, error) { + rows, err := scaleTableRows(table, "case", "user", "action", "resources", "expected") + if err != nil { + return nil, err + } + cases := make([]authorizationScaleCase, 0, len(rows)) + names := make(map[string]bool) + for _, row := range rows { + item := authorizationScaleCase{name: row[0], entity: row[1], action: row[2], resources: strings.Split(row[3], ","), expected: make(map[string]authz.Decision)} + if names[item.name] { + return nil, fmt.Errorf("duplicate case %q", item.name) } names[item.name] = true - expected := strings.Split(row.Cells[4].Value, ",") - if len(item.values) != len(expected) { - return nil, fmt.Errorf("case %s has mismatched values and expectations", item.name) + expected := strings.Split(row[4], ",") + if len(item.resources) != len(expected) { + return nil, fmt.Errorf("case %s has mismatched resources and expectations", item.name) } - for i, value := range item.values { - item.values[i] = strings.TrimSpace(value) - if item.values[i] == "" { - return nil, fmt.Errorf("case %s has an empty value", item.name) + for i, resource := range item.resources { + item.resources[i] = strings.TrimSpace(resource) + if item.resources[i] == "" { + return nil, fmt.Errorf("case %s has an empty resource", item.name) } decision, ok := authz.Decision_value["DECISION_"+strings.TrimSpace(expected[i])] if !ok || (authz.Decision(decision) != authz.Decision_DECISION_PERMIT && authz.Decision(decision) != authz.Decision_DECISION_DENY) { @@ -108,9 +132,9 @@ func validateScaleDecision(response *authz.GetDecisionMultiResourceResponse, exp return nil } -func exerciseAuthorizationCases(ctx context.Context, concurrency, seed int, requestTimeout, attributeRef string, table *godog.Table) (context.Context, error) { - if concurrency < 1 || seed < 0 { - return ctx, errors.New("concurrency must be positive and seed nonnegative") +func exerciseAuthorizationLoad(ctx context.Context, requests, concurrency, seed int, requestTimeout string, table *godog.Table) (context.Context, error) { + if concurrency < 1 || requests < concurrency || seed < 0 { + return ctx, errors.New("requests must be at least concurrency, concurrency positive, and seed nonnegative") } timeout, err := time.ParseDuration(requestTimeout) if err != nil || timeout <= 0 { @@ -121,92 +145,116 @@ func exerciseAuthorizationCases(ctx context.Context, concurrency, seed int, requ return ctx, err } scenario := GetPlatformScenarioContext(ctx) - attribute, ok := scenario.GetObject(attributeRef).(*policy.Attribute) - if !ok || attribute.GetFqn() == "" { - return ctx, fmt.Errorf("missing attribute %q", attributeRef) - } - // Shuffle all cases rather than sampling, so no expected path is omitted. - random := rand.New(rand.NewPCG(uint64(seed), uint64(concurrency))) //nolint:gosec // reproducible test order, not security randomness - random.Shuffle(len(cases), func(i, j int) { cases[i], cases[j] = cases[j], cases[i] }) - var failures []error - for _, item := range cases { + for i := range cases { + item := &cases[i] chain, err := buildEntityChainFromIDs(scenario, item.entity) if err != nil { return ctx, err } - request := &authz.GetDecisionMultiResourceRequest{ + item.request = &authz.GetDecisionMultiResourceRequest{ EntityIdentifier: &authz.EntityIdentifier{Identifier: &authz.EntityIdentifier_EntityChain{EntityChain: chain}}, Action: &policy.Action{Name: item.action}, } - for i, value := range item.values { - request.Resources = append(request.Resources, &authz.Resource{ + for i, name := range item.resources { + fqns, ok := scenario.GetObject("scale-resource/" + name).([]string) + if !ok || len(fqns) == 0 { + return ctx, fmt.Errorf("case %s: missing resource %q", item.name, name) + } + item.request.Resources = append(item.request.Resources, &authz.Resource{ EphemeralId: fmt.Sprintf("resource%d", i), - Resource: &authz.Resource_AttributeValues_{AttributeValues: &authz.Resource_AttributeValues{ - Fqns: []string{attribute.GetFqn() + "/value/" + value}, - }}, + Resource: &authz.Resource_AttributeValues_{AttributeValues: &authz.Resource_AttributeValues{Fqns: fqns}}, }) } - result, err := runAuthorizationScaleCase(ctx, scenario, item, request, concurrency, seed, timeout, random) - encoded, encodeErr := json.Marshal(result) - if encodeErr != nil { - return ctx, encodeErr - } - // A structured record survives both console and Go test JSON output formats. - fmt.Println(authorizationPerformanceMarker + string(encoded)) //nolint:forbidigo // structured CI record, independent of the configured log handler - if err != nil { - failures = append(failures, fmt.Errorf("case %s: %w", item.name, err)) - } } - return ctx, errors.Join(failures...) + result, runErr := runAuthorizationScaleLoad(ctx, cases, requests, concurrency, seed, timeout, scenario.SDK.AuthorizationV2.GetDecisionMultiResource) + encoded, err := json.Marshal(result) + if err != nil { + return ctx, err + } + fmt.Println(authorizationPerformanceMarker + string(encoded)) //nolint:forbidigo // structured CI record, independent of the configured log handler + return ctx, runErr +} + +// Preselect uniformly with replacement so scheduling cannot change the workload. +// The same seed selects the same cases at every concurrency level. +func selectAuthorizationCases(caseCount, requests, seed int) []int { + random := rand.New(rand.NewPCG(uint64(seed), 0)) //nolint:gosec // reproducible workload selection, not security randomness + selected := make([]int, requests) + for i := range selected { + selected[i] = random.IntN(caseCount) + } + return selected } -func runAuthorizationScaleCase(ctx context.Context, scenario *PlatformScenarioContext, item authorizationScaleCase, request *authz.GetDecisionMultiResourceRequest, concurrency, seed int, timeout time.Duration, random *rand.Rand) (authorizationPerformanceResult, error) { - result := authorizationPerformanceResult{Case: item.name, Seed: seed, Concurrency: concurrency, Resources: len(item.values), Timeout: timeout} - // Bound request completion without treating the timeout as a latency baseline. - requestCtx, cancel := context.WithTimeout(ctx, timeout) - defer cancel() - requests := make([]*authz.GetDecisionMultiResourceRequest, concurrency) +type scaleDecisionFunc func(context.Context, *authz.GetDecisionMultiResourceRequest) (*authz.GetDecisionMultiResourceResponse, error) + +func runAuthorizationScaleLoad(ctx context.Context, cases []authorizationScaleCase, requests, concurrency, seed int, timeout time.Duration, decide scaleDecisionFunc) (authorizationPerformanceResult, error) { + result := authorizationPerformanceResult{Seed: seed, Concurrency: concurrency, Requests: requests, Timeout: timeout, Cases: make([]authorizationCaseResult, len(cases))} + selected := selectAuthorizationCases(len(cases), requests, seed) + durations := make([]time.Duration, requests) + requestErrors := make([]error, requests) + jobs := make(chan int, requests) for i := range requests { - requests[i] = proto.CloneOf(request) - resources := requests[i].GetResources() - random.Shuffle(len(resources), func(i, j int) { resources[i], resources[j] = resources[j], resources[i] }) + jobs <- i } - durations := make([]time.Duration, concurrency) - requestErrors := make([]error, concurrency) + close(jobs) start := make(chan struct{}) - var workers sync.WaitGroup - for i := range requests { + var workers, ready sync.WaitGroup + ready.Add(concurrency) + for range concurrency { workers.Go(func() { + ready.Done() <-start - started := time.Now() - response, err := scenario.SDK.AuthorizationV2.GetDecisionMultiResource(requestCtx, requests[i]) - durations[i] = time.Since(started) - if err == nil { - err = validateScaleDecision(response, item.expected) + for i := range jobs { + item := cases[selected[i]] + request := proto.CloneOf(item.request) + // Each request gets a fresh deadline, including later work on the same worker. + requestCtx, cancel := context.WithTimeout(ctx, timeout) + started := time.Now() + response, err := decide(requestCtx, request) + durations[i] = time.Since(started) + cancel() + if err == nil { + err = validateScaleDecision(response, item.expected) + } + requestErrors[i] = err } - requestErrors[i] = err }) } + ready.Wait() started := time.Now() close(start) workers.Wait() result.Wall = time.Since(started) - for _, err := range requestErrors { - if err != nil { + for i, item := range cases { + row := authorizationCaseResult{Name: item.name, User: item.entity, Action: item.action, Resources: item.resources} + for j := range item.resources { + row.Expected = append(row.Expected, strings.TrimPrefix(item.expected[fmt.Sprintf("resource%d", j)].String(), "DECISION_")) + } + result.Cases[i] = row + } + var firstError error + for i, caseIndex := range selected { + row := &result.Cases[caseIndex] + row.Requests++ + result.ResourcesRequested += len(row.Resources) + if err := requestErrors[i]; err != nil { result.Failures++ + row.Failures++ + if row.FirstError == "" { + row.FirstError = err.Error() + } + if firstError == nil { + firstError = fmt.Errorf("case %s: %w", row.Name, err) + } } } slices.Sort(durations) - result.Median = durations[(concurrency-1)/2] - result.P95 = durations[(95*concurrency-1)/100] - result.Maximum = durations[concurrency-1] - var failure error - for _, err := range requestErrors { - if err != nil { - failure = err - result.FirstError = err.Error() - break - } + result.Median = durations[(requests-1)/2] + result.P95 = durations[(95*requests-1)/100] + result.Maximum = durations[requests-1] + if firstError != nil { + return result, fmt.Errorf("%d/%d authorization requests failed: %w", result.Failures, requests, firstError) } - return result, failure + return result, nil } diff --git a/tests-bdd/cukes/steps_authorization_scale_policy.go b/tests-bdd/cukes/steps_authorization_scale_policy.go new file mode 100644 index 0000000000..6b2cebcfa4 --- /dev/null +++ b/tests-bdd/cukes/steps_authorization_scale_policy.go @@ -0,0 +1,117 @@ +package cukes + +import ( + "context" + "fmt" + "strings" + + "github.com/cucumber/godog" + "github.com/opentdf/platform/protocol/go/policy" + "github.com/opentdf/platform/protocol/go/policy/attributes" + "github.com/opentdf/platform/protocol/go/policy/subjectmapping" +) + +func createScaleAttributes(ctx context.Context, namespaceRef string, table *godog.Table) (context.Context, error) { + rows, err := scaleTableRows(table, "attribute", "rule", valuesKey) + if err != nil { + return ctx, err + } + scenario := GetPlatformScenarioContext(ctx) + namespace, ok := scenario.GetObject(namespaceRef).(string) + if !ok { + return ctx, fmt.Errorf("missing namespace %q", namespaceRef) + } + for _, row := range rows { + rule, err := parseAttributeRule(row[1]) + if err != nil { + return ctx, err + } + values := strings.Split(row[2], ",") + for i := range values { + values[i] = strings.TrimSpace(values[i]) + } + response, err := scenario.SDK.Attributes.CreateAttribute(ctx, &attributes.CreateAttributeRequest{NamespaceId: namespace, Name: row[0], Rule: rule, Values: values}) + if err != nil { + return ctx, err + } + if response.GetAttribute().GetId() == "" { + return ctx, fmt.Errorf("attribute %s returned no identity", row[0]) + } + scenario.RecordObject(row[0], response.GetAttribute()) + } + return ctx, nil +} + +func scaleAttributeValue(scenario *PlatformScenarioContext, reference string) (*policy.Value, string, error) { + attributeRef, valueName, ok := strings.Cut(strings.TrimSpace(reference), "/") + if !ok { + return nil, "", fmt.Errorf("expected attribute/value, got %q", reference) + } + attribute, ok := scenario.GetObject(attributeRef).(*policy.Attribute) + if ok && attribute.GetFqn() != "" { + for _, value := range attribute.GetValues() { + if value.GetValue() == valueName { + return value, attribute.GetFqn() + "/value/" + valueName, nil + } + } + } + return nil, "", fmt.Errorf("unknown attribute value %q", reference) +} + +func createScaleGrants(ctx context.Context, table *godog.Table) (context.Context, error) { + rows, err := scaleTableRows(table, "attribute value", "selector", "matches", "actions") + if err != nil { + return ctx, err + } + scenario := GetPlatformScenarioContext(ctx) + for _, row := range rows { + value, _, err := scaleAttributeValue(scenario, row[0]) + if err != nil { + return ctx, err + } + response, err := scenario.SDK.SubjectMapping.CreateSubjectMapping(ctx, &subjectmapping.CreateSubjectMappingRequest{ + AttributeValueId: value.GetId(), Actions: GetActionsFromValues(&row[3], nil), + NewSubjectConditionSet: &subjectmapping.SubjectConditionSetCreate{SubjectSets: []*policy.SubjectSet{{ + ConditionGroups: []*policy.ConditionGroup{{ + BooleanOperator: policy.ConditionBooleanTypeEnum_CONDITION_BOOLEAN_TYPE_ENUM_OR, + Conditions: []*policy.Condition{{SubjectExternalSelectorValue: row[1], Operator: policy.SubjectMappingOperatorEnum_SUBJECT_MAPPING_OPERATOR_ENUM_IN, SubjectExternalValues: strings.Split(row[2], ",")}}, + }}, + }}}, + }) + if err != nil { + return ctx, err + } + if response.GetSubjectMapping().GetId() == "" { + return ctx, fmt.Errorf("grant for %s returned no identity", row[0]) + } + if err := validateScaleMappingActions(response.GetSubjectMapping().GetActions(), row[3]); err != nil { + return ctx, fmt.Errorf("grant for %s: %w", row[0], err) + } + } + return ctx, nil +} + +func defineScaleResources(ctx context.Context, table *godog.Table) (context.Context, error) { + rows, err := scaleTableRows(table, "resource", "attributes") + if err != nil { + return ctx, err + } + scenario := GetPlatformScenarioContext(ctx) + names := make(map[string]bool) + for _, row := range rows { + if names[row[0]] { + return ctx, fmt.Errorf("duplicate resource %q", row[0]) + } + names[row[0]] = true + var fqns []string + for _, reference := range strings.Split(row[1], ",") { + _, fqn, err := scaleAttributeValue(scenario, reference) + if err != nil { + return ctx, err + } + fqns = append(fqns, fqn) + } + scenario.RecordObject("scale-resource/"+row[0], fqns) + } + return ctx, nil +} diff --git a/tests-bdd/cukes/steps_authorization_scale_test.go b/tests-bdd/cukes/steps_authorization_scale_test.go index c764705a1f..90770f7594 100644 --- a/tests-bdd/cukes/steps_authorization_scale_test.go +++ b/tests-bdd/cukes/steps_authorization_scale_test.go @@ -1,9 +1,14 @@ package cukes import ( + "context" + "slices" + "sync/atomic" "testing" + "time" authz "github.com/opentdf/platform/protocol/go/authorization/v2" + "github.com/opentdf/platform/protocol/go/policy" "github.com/stretchr/testify/require" "google.golang.org/protobuf/proto" ) @@ -47,3 +52,115 @@ func TestScaleCasesRejectMissingTable(t *testing.T) { _, err := parseAuthorizationScaleCases(nil) require.Error(t, err) } + +func TestScaleLoadSamplesDifferentCasesReproducibly(t *testing.T) { + selected := selectAuthorizationCases(24, 200, 4625) + require.Equal(t, selected, selectAuthorizationCases(24, 200, 4625)) + require.NotEqual(t, selected, selectAuthorizationCases(24, 200, 4626)) + seen := make(map[int]bool) + for _, index := range selected { + require.Less(t, index, 24) + require.GreaterOrEqual(t, index, 0) + seen[index] = true + } + require.Len(t, seen, 24, "the checked-in seed should exercise the complete current case pool") + require.Greater(t, len(slices.Compact(slices.Clone(selected))), 24, "cases must be interleaved rather than grouped into homogeneous batches") +} + +func loadTestCase(name string) authorizationScaleCase { + return authorizationScaleCase{ + name: name, entity: "test-user", action: "read", resources: []string{name}, + expected: map[string]authz.Decision{"resource0": authz.Decision_DECISION_PERMIT}, + request: &authz.GetDecisionMultiResourceRequest{Resources: []*authz.Resource{{ + EphemeralId: "resource0", Resource: &authz.Resource_AttributeValues_{AttributeValues: &authz.Resource_AttributeValues{Fqns: []string{name}}}, + }}}, + } +} + +func permittedLoadResponse(request *authz.GetDecisionMultiResourceRequest) *authz.GetDecisionMultiResourceResponse { + return &authz.GetDecisionMultiResourceResponse{ResourceDecisions: []*authz.ResourceDecision{{EphemeralResourceId: request.GetResources()[0].GetEphemeralId(), Decision: authz.Decision_DECISION_PERMIT}}} +} + +func TestScaleLoadMixesRequestsAcrossBoundedWorkers(t *testing.T) { + cases := []authorizationScaleCase{loadTestCase("engineering"), loadTestCase("projects"), loadTestCase("clearance")} + const concurrency = 4 + started := make(chan string, concurrency) + release := make(chan struct{}) + var calls, active, maximum atomic.Int32 + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + type completion struct { + result authorizationPerformanceResult + err error + } + done := make(chan completion, 1) + go func() { + result, err := runAuthorizationScaleLoad(ctx, cases, 40, concurrency, 4625, time.Second, func(ctx context.Context, request *authz.GetDecisionMultiResourceRequest) (*authz.GetDecisionMultiResourceResponse, error) { + current := active.Add(1) + defer active.Add(-1) + for previous := maximum.Load(); current > previous; previous = maximum.Load() { + if maximum.CompareAndSwap(previous, current) { + break + } + } + if calls.Add(1) <= concurrency { + started <- request.GetResources()[0].GetAttributeValues().GetFqns()[0] + select { + case <-release: + case <-ctx.Done(): + return nil, ctx.Err() + } + } + return permittedLoadResponse(request), nil + }) + done <- completion{result, err} + }() + first := make(map[string]bool) + for range concurrency { + select { + case name := <-started: + first[name] = true + case <-ctx.Done(): + t.Fatal("workers did not start concurrently") + } + } + close(release) + finished := <-done + require.NoError(t, finished.err) + require.Greater(t, len(first), 1, "the same concurrent group should contain different cases") + require.EqualValues(t, concurrency, maximum.Load()) + require.EqualValues(t, 40, calls.Load()) + require.Equal(t, 40, finished.result.Requests) + total := 0 + for _, item := range finished.result.Cases { + total += item.Requests + require.Positive(t, item.Requests) + } + require.Equal(t, 40, total) +} + +func TestScaleLoadUsesFreshDeadlineAndReportsFailure(t *testing.T) { + calls := 0 + result, err := runAuthorizationScaleLoad(context.Background(), []authorizationScaleCase{loadTestCase("test")}, 2, 1, 4625, 10*time.Millisecond, + func(ctx context.Context, request *authz.GetDecisionMultiResourceRequest) (*authz.GetDecisionMultiResourceResponse, error) { + calls++ + if calls == 1 { + <-ctx.Done() + return nil, ctx.Err() + } + if err := ctx.Err(); err != nil { + return nil, err + } + return permittedLoadResponse(request), nil + }) + require.Error(t, err) + require.Equal(t, 2, calls) + require.Equal(t, 1, result.Failures, "the second request must not inherit the first request's expired deadline") + require.Equal(t, 1, result.Cases[0].Failures) + require.Contains(t, result.Cases[0].FirstError, "deadline exceeded") +} + +func TestScaleMappingRejectsMissingAction(t *testing.T) { + require.NoError(t, validateScaleMappingActions([]*policy.Action{{Name: "write"}, {Name: "read"}}, "read,write")) + require.ErrorContains(t, validateScaleMappingActions([]*policy.Action{{Name: "read"}}, "read,write"), "expected actions") +} diff --git a/tests-bdd/cukes/steps_subjectmappings_scale.go b/tests-bdd/cukes/steps_subjectmappings_scale.go index 3a6ba0705e..47db585ffc 100644 --- a/tests-bdd/cukes/steps_subjectmappings_scale.go +++ b/tests-bdd/cukes/steps_subjectmappings_scale.go @@ -3,6 +3,8 @@ package cukes import ( "context" "fmt" + "slices" + "strings" "github.com/opentdf/platform/protocol/go/policy" "github.com/opentdf/platform/protocol/go/policy/subjectmapping" @@ -18,7 +20,7 @@ func (s *SubjectMappingsStepDefinitions) createScaleSubjectMappings(ctx context. if !ok { return ctx, fmt.Errorf("missing condition set %q", conditionSetRef) } - err := createScaleMappings(ctx, count, func(ctx context.Context, index int) error { + create := func(ctx context.Context, index int) error { response, err := scenario.SDK.SubjectMapping.CreateSubjectMapping(ctx, &subjectmapping.CreateSubjectMappingRequest{ AttributeValueId: attribute.GetValues()[index].GetId(), ExistingSubjectConditionSetId: conditionSet.GetId(), Actions: GetActionsFromValues(&action, nil), @@ -29,7 +31,35 @@ func (s *SubjectMappingsStepDefinitions) createScaleSubjectMappings(ctx context. if response.GetSubjectMapping().GetId() == "" { return fmt.Errorf("subject mapping %d returned no identity", index) } + if err := validateScaleMappingActions(response.GetSubjectMapping().GetActions(), action); err != nil { + return fmt.Errorf("subject mapping %d: %w", index, err) + } return nil + } + // Resolve any new action names before concurrent mapping creation starts. + // Concurrent create-or-list calls can otherwise return an incomplete action set. + if err := create(ctx, 0); err != nil { + return ctx, err + } + err := createScaleMappings(ctx, count-1, func(ctx context.Context, index int) error { + return create(ctx, index+1) }) return ctx, err } + +func validateScaleMappingActions(actions []*policy.Action, expected string) error { + want := strings.Split(strings.ToLower(expected), ",") + for i := range want { + want[i] = strings.TrimSpace(want[i]) + } + got := make([]string, len(actions)) + for i, action := range actions { + got[i] = strings.ToLower(action.GetName()) + } + slices.Sort(want) + slices.Sort(got) + if !slices.Equal(got, want) { + return fmt.Errorf("expected actions %v, got %v", want, got) + } + return nil +} diff --git a/tests-bdd/features/authorization-v2-subject-mapping-performance.feature b/tests-bdd/features/authorization-v2-subject-mapping-performance.feature index 527e6c6c54..b745b2b012 100644 --- a/tests-bdd/features/authorization-v2-subject-mapping-performance.feature +++ b/tests-bdd/features/authorization-v2-subject-mapping-performance.feature @@ -1,61 +1,117 @@ @authorization @authz-v2 @performance @scale -Feature: v2 multi-resource decisions at large policy scale - GetDecisionMultiResource must remain responsive when the policy database contains the - subject-mapping and resource-mapping cardinality from the reported regression. Fixture setup is - not timed. The measured operation is a synchronized group of requests to the public v2 - authorization endpoint. Every case runs at every concurrency level. A fixed seed - shuffles case and resource order reproducibly. The extra value has no subject mapping. - Subject mappings use the default unnamespaced policy path; attribute and resource mappings - retain their namespace. This avoids repeatedly validating the entire attribute during setup. - Latency is reported without a performance gate until a baseline is established. - Incorrect decisions, request errors, and request timeouts fail the scenario. - The server write timeout exceeds the client deadline so slow completed responses - can be measured instead of being cut off by the default ten-second write timeout. +Feature: Mixed authorization traffic at large policy scale + Concurrent workers continuously select requests from a pool of entitlement cases. + The pool varies users, actions, anyOf/allOf/hierarchy rules, combined attributes, + and multiple resources. Every resource includes a value from the large attribute + so the measured requests exercise the large-policy lookup path. + The seed selects the same request mix at every concurrency level. Each selection + is independent; the summary reports how often each case was selected, including zero. + Setup is excluded. Latency is report-only; incorrect decisions, errors, and client + timeouts fail. The fixture's server write timeout exceeds the client deadline. - Scenario Outline: Varied multi-resource decisions at concurrency - Given a user exists with username "scale-user" and email "scale-user@example.com" and the following attributes: - | name | value | - | department | ["engineering"] | - And a user exists with username "other-user" and email "other-user@example.com" and the following attributes: + Scenario Outline: Random entitlement requests at concurrency + Given a user exists with username "engineer" and email "engineer@example.com" and the following attributes: + | name | value | + | population | ["load"] | + | department | ["engineering"] | + | projects | ["alpha","beta"] | + | clearance | ["high"] | + And a user exists with username "analyst" and email "analyst@example.com" and the following attributes: | name | value | + | population | ["load"] | | department | ["sales"] | + | projects | ["alpha"] | + | clearance | ["low"] | + And a user exists with username "visitor" and email "visitor@example.com" and the following attributes: + | name | value | + | population | ["load"] | + | department | ["none"] | + | projects | ["none"] | + | clearance | ["none"] | And an empty local platform with HTTP write timeout "35s" And I submit a request to create a namespace with name "scale.example" and reference id "scale_ns" And I send a request to create an attribute referenced as "scale_attr" in namespace "scale_ns" named "access-level" with rule "anyOf" and 6012 generated values in batches of 25 Then the response should be successful And a condition group referenced as "scale_condition" with an "or" operator with conditions: - | selector_value | operator | values | - | .attributes.department[] | in | engineering | + | selector_value | operator | values | + | .attributes.population[] | in | load | And a subject set referenced as "scale_subject_set" containing the condition groups "scale_condition" And I send a request to create a subject condition set referenced as "scale_condition_set" containing subject sets "scale_subject_set" Then the response should be successful - And I create 6011 subject mappings for attribute "scale_attr" using condition set "scale_condition_set" with action "read" + And I create 6011 subject mappings for attribute "scale_attr" using condition set "scale_condition_set" with action "read,write" And I create 6000 resource mappings for attribute "scale_attr" in namespace "scale_ns" - And there is a "user_name" subject entity with value "scale-user" and referenced as "scale-user" - And there is a "user_name" subject entity with value "other-user" and referenced as "other-user" - When I exercise the authorization cases with concurrent requests each, seed 4625, and request timeout "30s" for attribute "scale_attr": - | case | entity | action | values | expected | - | allowed_read | scale-user | read | v0000,v3005,v6010 | PERMIT,PERMIT,PERMIT | - | denied_action | scale-user | write | v0000,v3005,v6010 | DENY,DENY,DENY | - | denied_user | other-user | read | v0000,v3005,v6010 | DENY,DENY,DENY | - | mixed_values | scale-user | read | v0000,v6011,v6010 | PERMIT,DENY,PERMIT | + And the following scale attributes exist in namespace "scale_ns": + | attribute | rule | values | + | department | anyOf | engineering,sales | + | project | allOf | alpha,beta | + | clearance | hierarchy | critical,high,medium,low | + And the following scale grants exist: + | attribute value | selector | matches | actions | + | department/engineering | .attributes.department[] | engineering | read | + | department/sales | .attributes.department[] | sales | read | + | project/alpha | .attributes.projects[] | alpha | read,write | + | project/beta | .attributes.projects[] | beta | read,write | + | clearance/high | .attributes.clearance[] | high | read | + | clearance/low | .attributes.clearance[] | low | read | + And the following scale resources are defined: + | resource | attributes | + | engineering | scale_attr/v0000,department/engineering | + | sales | scale_attr/v3005,department/sales | + | either-team | scale_attr/v6010,department/engineering,department/sales | + | alpha | scale_attr/v0000,project/alpha | + | projects | scale_attr/v3005,project/alpha,project/beta | + | medium | scale_attr/v6010,clearance/medium | + | low | scale_attr/v0000,clearance/low | + | critical | scale_attr/v3005,clearance/critical | + | team-project | scale_attr/v6010,department/engineering,project/alpha,project/beta | + | all-rules | scale_attr/v0000,department/engineering,department/sales,project/alpha,project/beta,clearance/medium | + | unmapped | scale_attr/v6011 | + And there is a "user_name" subject entity with value "engineer" and referenced as "engineer" + And there is a "user_name" subject entity with value "analyst" and referenced as "analyst" + And there is a "user_name" subject entity with value "visitor" and referenced as "visitor" + When I send 200 randomly selected authorization requests with concurrency , seed 4625, and request timeout "30s": + | case | user | action | resources | expected | + | anyOf matching team | engineer | read | engineering | PERMIT | + | anyOf other team | engineer | read | sales | DENY | + | anyOf one matching value | analyst | read | either-team | PERMIT | + | anyOf no matching value | visitor | read | either-team | DENY | + | allOf all values granted | engineer | read | projects | PERMIT | + | allOf missing one grant | analyst | read | projects | DENY | + | allOf single value | analyst | read | alpha | PERMIT | + | allOf no grants | visitor | read | projects | DENY | + | hierarchy higher grant | engineer | read | medium,low | PERMIT,PERMIT | + | hierarchy lower grant | analyst | read | medium | DENY | + | hierarchy exact grant | analyst | read | low | PERMIT | + | hierarchy above clearance | engineer | read | critical | DENY | + | hierarchy no grant | visitor | read | low | DENY | + | write granted | engineer | write | projects | PERMIT | + | write partial grants | analyst | write | alpha,projects | PERMIT,DENY | + | write not granted | engineer | write | engineering | DENY | + | combined attributes pass | engineer | read | team-project,all-rules | PERMIT,PERMIT | + | combined attributes fail | analyst | read | team-project,all-rules | DENY,DENY | + | combined action denied | engineer | write | team-project | DENY | + | mixed resource decisions | engineer | read | engineering,sales,projects | PERMIT,DENY,PERMIT | + | different user decisions | analyst | read | engineering,sales,projects | DENY,PERMIT,DENY | + | no attribute grants | visitor | read | engineering,sales,projects | DENY,DENY,DENY | + | no mapping for value | engineer | read | unmapped | DENY | + | mixed mapped and unmapped | engineer | read | engineering,unmapped | PERMIT,DENY | @concurrency-1 - Examples: One request + Examples: One worker | concurrency | | 1 | @concurrency-10 - Examples: Ten concurrent requests + Examples: Ten workers | concurrency | | 10 | @concurrency-25 - Examples: Twenty-five concurrent requests + Examples: Twenty-five workers | concurrency | | 25 | @concurrency-50 - Examples: Fifty concurrent requests + Examples: Fifty workers | concurrency | | 50 | From 157f9b13a1fd2c33dd31c5c0248a37eda2ab065b Mon Sep 17 00:00:00 2001 From: strantalis Date: Wed, 9 Sep 2026 17:46:48 -0400 Subject: [PATCH 10/10] test(authz): align scale policy and generated resource traffic Signed-off-by: strantalis --- .../scripts/summarize-authz-performance.py | 16 +- .../test_summarize_authz_performance.py | 13 + tests-bdd/README.md | 27 ++ tests-bdd/cukes/steps_authorization.go | 2 + tests-bdd/cukes/steps_authorization_scale.go | 55 +++- .../steps_authorization_scale_generated.go | 266 ++++++++++++++++++ ...teps_authorization_scale_generated_test.go | 158 +++++++++++ .../cukes/steps_authorization_scale_policy.go | 7 +- tests-bdd/cukes/steps_localplatform.go | 8 +- .../cukes/steps_resourcemappings_scale.go | 11 +- tests-bdd/cukes/steps_subjectmappings.go | 2 +- .../cukes/steps_subjectmappings_scale.go | 39 +-- ...ion-v2-subject-mapping-performance.feature | 120 +++----- 13 files changed, 595 insertions(+), 129 deletions(-) create mode 100644 tests-bdd/cukes/steps_authorization_scale_generated.go create mode 100644 tests-bdd/cukes/steps_authorization_scale_generated_test.go diff --git a/.github/scripts/summarize-authz-performance.py b/.github/scripts/summarize-authz-performance.py index 29be63e92c..95f3a9a620 100644 --- a/.github/scripts/summarize-authz-performance.py +++ b/.github/scripts/summarize-authz-performance.py @@ -19,6 +19,8 @@ def validate_result(result): cases = result.get("cases") if not isinstance(cases, list) or not cases: raise ValueError("missing case results") + if not isinstance(result.get("fixture", ""), str): + raise ValueError("invalid fixture description") names = set() for case in cases: if any(not isinstance(case.get(key), str) or not case[key] for key in ("name", "user", "action")): @@ -30,6 +32,9 @@ def validate_result(result): raise ValueError("invalid case counts") if case["failures"] > case["requests"] or not isinstance(case.get("first_error", ""), str): raise ValueError("invalid case failures") + variants, used = case.get("variants", 0), case.get("variants_used", 0) + if type(variants) is not int or type(used) is not int or not 0 <= used <= min(variants, case["requests"]): + raise ValueError("invalid variant counts") resources, expected = case.get("resources"), case.get("expected") if not isinstance(resources, list) or not resources or any(not isinstance(r, str) or not r for r in resources): raise ValueError("invalid resources") @@ -76,6 +81,8 @@ def render(text, outcome): "**Workload:** workers continuously draw from the entitlement case pool. Each request independently selects a case; users, actions, and resources vary within the same load run.", "", "**Performance: REPORT ONLY.** Correctness PASS means all requests completed with the expected resource decisions. Errors and request timeouts fail; no latency baseline is enforced.", ""] if results: + if results[0].get("fixture"): + lines += [cell(results[0]["fixture"]), ""] lines += [ "| Concurrency | Requests | Cases used | Median | p95 | Maximum | Requests/s | Failures | Correctness |", "| ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | --- |", @@ -87,15 +94,16 @@ def render(text, outcome): cells += [milliseconds(row[key]) for key in ("median_ns", "p95_ns", "maximum_ns")] cells += [rate, str(row["failures"]), "FAIL" if row["failures"] else "PASS"] lines.append("| " + " | ".join(cells) + " |") - lines += ["", "Fixture setup is excluded. Latency covers the whole multi-resource request. Throughput includes failed requests; inspect correctness alongside it. Cases selected zero times remain visible below."] + lines += ["", "Fixture setup is excluded. Latency covers the whole multi-resource request. Throughput includes failed requests; inspect correctness alongside it. Cases selected zero times remain visible below. For generated cases, resource labels name pools; the variants column shows distinct request variants used/available."] for row in results: lines += ["", "
", f"Case selection and failures at concurrency {row['concurrency']}", "", f"Seed: {row['seed']}. Request timeout: {row['timeout_ns'] / 1_000_000_000:g} s. Load duration: {row['wall_ns'] / 1_000_000_000:,.2f} s. Resources requested: {row['resources_requested']:,}.", "", - "| Case | User | Action | Resources | Expected decisions | Selected | Failures | First error |", - "| --- | --- | --- | --- | --- | ---: | ---: | --- |"] + "| Case | User | Action | Resources | Expected decisions | Selected | Failures | First error | Variants used/available |", + "| --- | --- | --- | --- | --- | ---: | ---: | --- | ---: |"] for case in row["cases"]: cells = [case["name"], case["user"], case["action"], ", ".join(case["resources"]), ", ".join(case["expected"]), - case["requests"], case["failures"], case.get("first_error", "")] + case["requests"], case["failures"], case.get("first_error", ""), + f"{case.get('variants_used', 0)}/{case['variants']}" if case.get("variants") else "fixed"] lines.append("| " + " | ".join(cell(c) for c in cells) + " |") lines += ["", "
"] else: diff --git a/.github/scripts/test_summarize_authz_performance.py b/.github/scripts/test_summarize_authz_performance.py index 2837cf91b0..d9befd6861 100644 --- a/.github/scripts/test_summarize_authz_performance.py +++ b/.github/scripts/test_summarize_authz_performance.py @@ -55,6 +55,19 @@ def test_unsampled_cases_remain_visible(self): self.assertIn("| 2/3 |", rendered) self.assertIn("not selected | analyst | write | projects | DENY | 0 | 0 |", rendered) + def test_generated_variants_and_fixture_are_visible(self): + record = json.loads(self.record().split(summary.MARKER)[1]) + record["fixture"] = "6,011 distinct subject mappings; 5 users" + record["cases"][0].update(variants=100, variants_used=61) + rendered, errors = summary.render(summary.MARKER + json.dumps(record), "success") + self.assertEqual(errors, 0) + self.assertIn("6,011 distinct subject mappings; 5 users", rendered) + self.assertIn("Variants used/available", rendered) + self.assertIn("61/100", rendered) + record["cases"][0]["variants_used"] = 101 + _, errors = summary.render(summary.MARKER + json.dumps(record), "success") + self.assertEqual(errors, 1) + def test_cli_handles_missing_log_after_setup_failure(self): with tempfile.TemporaryDirectory() as directory: result = subprocess.run( diff --git a/tests-bdd/README.md b/tests-bdd/README.md index 64dce7cb97..9ae9f63f21 100644 --- a/tests-bdd/README.md +++ b/tests-bdd/README.md @@ -401,3 +401,30 @@ All 3 scenarios should pass (57 steps, 0 failures). You'll see LDAP testcontaine - Build out step definitions for platform services - Continue to explore AI Agent for scenario generation - Build cross version/platform/sdk fixtures + +### Authorization policy scale fixture + +`features/authorization-v2-subject-mapping-performance.feature` creates 6,000 +project values with `allOf`, four classification levels with `hierarchy`, and +seven regions with `anyOf`. Each of the 6,011 subject mappings has its own +condition set: the matching user entitlement OR one of four synthetic approved +client IDs. The 6,000 resource mappings each have two to five aliases. + +Five Keycloak users hold 3, 10, 50, 500, and zero projects. Generated resources +combine projects, classification, and regions. The initial 1,000 documents contain +55% single-project, 40% 2-20 projects, and 5% 21-30 projects. Load excludes documents +exceeding 20 total attribute FQNs. Another 100 documents provide permitted +examples for the four entitled users; an additional unmapped value tests denial. +No document files are uploaded: authorization receives their attribute FQNs. + +Each of 200 requests selects a case and then a resource variant from that case's +pool, with replacement. Identical resource combinations are removed from each +pool. The seed fixes both selections before workers start. Cases vary users, read/write/delete, one or three resources, and expected +permit/deny combinations. Cases are sampled uniformly to exercise both permits +and denies; this is not a measured customer traffic distribution. The same +workload runs at concurrency 1, 10, 25, and 50. Setup is excluded from timings. + +The CI summary reports policy dimensions, latency, failures, case selection, +and distinct variants used/available. Failures include a variant index so the +seeded request can be reconstructed. Latency remains report-only; request errors, +incorrect decisions, and the 30-second client deadline fail the test. diff --git a/tests-bdd/cukes/steps_authorization.go b/tests-bdd/cukes/steps_authorization.go index e9a1f63379..bc3970eb4d 100644 --- a/tests-bdd/cukes/steps_authorization.go +++ b/tests-bdd/cukes/steps_authorization.go @@ -546,6 +546,8 @@ func (s *AuthorizationServiceStepDefinitions) theDecisionResponseForResourceShou } func RegisterAuthorizationStepDefinitions(ctx *godog.ScenarioContext) { + ctx.Step(`^representative scale users hold subsets of (\d+) project values with seed (\d+)$`, prepareScaleUsers) + ctx.Step(`^I send (\d+) generated authorization requests with concurrency (\d+), seed (\d+), request timeout "([^"]*)", attribute "([^"]*)", and (\d+) documents$`, exerciseGeneratedAuthorizationLoad) ctx.Step(`^the following scale attributes exist in namespace "([^"]*)":$`, createScaleAttributes) ctx.Step(`^the following scale grants exist:$`, createScaleGrants) ctx.Step(`^the following scale resources are defined:$`, defineScaleResources) diff --git a/tests-bdd/cukes/steps_authorization_scale.go b/tests-bdd/cukes/steps_authorization_scale.go index 670b59a864..04a5ea0707 100644 --- a/tests-bdd/cukes/steps_authorization_scale.go +++ b/tests-bdd/cukes/steps_authorization_scale.go @@ -24,20 +24,24 @@ type authorizationScaleCase struct { resources []string expected map[string]authz.Decision request *authz.GetDecisionMultiResourceRequest + variants []*authz.GetDecisionMultiResourceRequest } type authorizationCaseResult struct { - Name string `json:"name"` - User string `json:"user"` - Action string `json:"action"` - Resources []string `json:"resources"` - Expected []string `json:"expected"` - Requests int `json:"requests"` - Failures int `json:"failures"` - FirstError string `json:"first_error,omitempty"` + Name string `json:"name"` + User string `json:"user"` + Action string `json:"action"` + Resources []string `json:"resources"` + Expected []string `json:"expected"` + Variants int `json:"variants,omitempty"` + VariantsUsed int `json:"variants_used,omitempty"` + Requests int `json:"requests"` + Failures int `json:"failures"` + FirstError string `json:"first_error,omitempty"` } type authorizationPerformanceResult struct { + Fixture string `json:"fixture,omitempty"` Seed int `json:"seed"` Concurrency int `json:"concurrency"` Requests int `json:"requests"` @@ -166,7 +170,15 @@ func exerciseAuthorizationLoad(ctx context.Context, requests, concurrency, seed }) } } + return reportAuthorizationLoad(ctx, cases, requests, concurrency, seed, timeout) +} + +func reportAuthorizationLoad(ctx context.Context, cases []authorizationScaleCase, requests, concurrency, seed int, timeout time.Duration) (context.Context, error) { + scenario := GetPlatformScenarioContext(ctx) result, runErr := runAuthorizationScaleLoad(ctx, cases, requests, concurrency, seed, timeout, scenario.SDK.AuthorizationV2.GetDecisionMultiResource) + if fixture, ok := scenario.GetObject("scale-fixture-description").(string); ok { + result.Fixture = fixture + } encoded, err := json.Marshal(result) if err != nil { return ctx, err @@ -191,6 +203,14 @@ type scaleDecisionFunc func(context.Context, *authz.GetDecisionMultiResourceRequ func runAuthorizationScaleLoad(ctx context.Context, cases []authorizationScaleCase, requests, concurrency, seed int, timeout time.Duration, decide scaleDecisionFunc) (authorizationPerformanceResult, error) { result := authorizationPerformanceResult{Seed: seed, Concurrency: concurrency, Requests: requests, Timeout: timeout, Cases: make([]authorizationCaseResult, len(cases))} selected := selectAuthorizationCases(len(cases), requests, seed) + // Preselect resource variants too, independently of goroutine scheduling. + variantRandom := rand.New(rand.NewPCG(uint64(seed), 1)) //nolint:gosec // reproducible test data + variants := make([]int, requests) + for i, caseIndex := range selected { + if count := len(cases[caseIndex].variants); count > 0 { + variants[i] = variantRandom.IntN(count) + } + } durations := make([]time.Duration, requests) requestErrors := make([]error, requests) jobs := make(chan int, requests) @@ -207,7 +227,11 @@ func runAuthorizationScaleLoad(ctx context.Context, cases []authorizationScaleCa <-start for i := range jobs { item := cases[selected[i]] - request := proto.CloneOf(item.request) + template := item.request + if len(item.variants) > 0 { + template = item.variants[variants[i]] + } + request := proto.CloneOf(template) // Each request gets a fresh deadline, including later work on the same worker. requestCtx, cancel := context.WithTimeout(ctx, timeout) started := time.Now() @@ -227,18 +251,29 @@ func runAuthorizationScaleLoad(ctx context.Context, cases []authorizationScaleCa workers.Wait() result.Wall = time.Since(started) for i, item := range cases { - row := authorizationCaseResult{Name: item.name, User: item.entity, Action: item.action, Resources: item.resources} + row := authorizationCaseResult{Name: item.name, User: item.entity, Action: item.action, Resources: item.resources, Variants: len(item.variants)} for j := range item.resources { row.Expected = append(row.Expected, strings.TrimPrefix(item.expected[fmt.Sprintf("resource%d", j)].String(), "DECISION_")) } result.Cases[i] = row } + usedVariants := make([]map[int]bool, len(cases)) + for i := range usedVariants { + usedVariants[i] = make(map[int]bool) + } var firstError error for i, caseIndex := range selected { row := &result.Cases[caseIndex] row.Requests++ + if len(cases[caseIndex].variants) > 0 { + usedVariants[caseIndex][variants[i]] = true + row.VariantsUsed = len(usedVariants[caseIndex]) + } result.ResourcesRequested += len(row.Resources) if err := requestErrors[i]; err != nil { + if len(cases[caseIndex].variants) > 0 { + err = fmt.Errorf("variant %d: %w", variants[i], err) + } result.Failures++ row.Failures++ if row.FirstError == "" { diff --git a/tests-bdd/cukes/steps_authorization_scale_generated.go b/tests-bdd/cukes/steps_authorization_scale_generated.go new file mode 100644 index 0000000000..a622d6e4f6 --- /dev/null +++ b/tests-bdd/cukes/steps_authorization_scale_generated.go @@ -0,0 +1,266 @@ +package cukes + +import ( + "context" + "errors" + "fmt" + "math/rand/v2" + "slices" + "strings" + "time" + + authz "github.com/opentdf/platform/protocol/go/authorization/v2" + "github.com/opentdf/platform/protocol/go/policy" +) + +const ( + scaleUsersKey = "scale-users" + scaleUnmapped = "unmapped" + scaleRead = "read" + scaleWrite = "write" + scaleResourceStream = 4 + scaleMaxFQNs = 20 +) + +var ( + scaleClearances = []string{"critical", "high", "medium", "low"} + scaleRegions = []string{"region-a", "region-b", "region-c", "region-d", "region-e", "region-f", "region-g"} +) + +type scaleUser struct { + name string + projects []int + clearance int + region string +} + +type scaleDocument struct { + name string + projects []int + clearance int + regions []string +} + +// A small directory represents entitlement sizes, not directory throughput. +// Independent subsets overlap naturally and span the whole attribute. +// +//nolint:mnd // fixture archetype sizes and independent random stream are defined here +func generateScaleUsers(values, seed int) []scaleUser { + random := rand.New(rand.NewPCG(uint64(seed), 2)) //nolint:gosec // reproducible fixture + users := []scaleUser{ + {name: "few-grants", clearance: 2, region: scaleRegions[0]}, + {name: "moderate-grants", clearance: 1, region: scaleRegions[1]}, + {name: "many-grants", clearance: 1, region: scaleRegions[2]}, + {name: "large-grants", clearance: 0, region: scaleRegions[3]}, + {name: "no-grants", clearance: 3, region: scaleRegions[4]}, + } + for i, count := range []int{3, 10, 50, 500, 0} { + users[i].projects = random.Perm(values)[:min(count, values)] + } + return users +} + +func prepareScaleUsers(ctx context.Context, values, seed int) (context.Context, error) { + if values < 500 || seed < 0 { + return ctx, errors.New("scale users require at least 500 values and a nonnegative seed") + } + users := generateScaleUsers(values, seed) + for _, user := range users { + projects := make([]string, len(user.projects)) + for i, value := range user.projects { + projects[i] = fmt.Sprintf("v%04d", value) + } + var err error + _, err = registerLocalUser(ctx, user.name, user.name+"@example.com", map[string]any{ + "projects": projects, "clearance": []string{scaleClearances[user.clearance]}, "regions": []string{user.region}, + }) + if err != nil { + return ctx, err + } + _, err = (&AuthorizationServiceStepDefinitions{}).thereIsASubjectEntityWithValueAndReferencedAs(ctx, "user_name", user.name, user.name) + if err != nil { + return ctx, err + } + } + GetPlatformScenarioContext(ctx).RecordObject(scaleUsersKey, users) + return ctx, nil +} + +// The generated population follows the scale harness: 55% single-project, +// 40% 2-20 projects, and 5% deliberately over the 20-FQN boundary. Authorization +// traffic excludes over-limit documents, as the harness does. Add permitted +// examples for each user so sparse random intersections don't yield only denies. +// +//nolint:mnd // document proportions and ranges mirror the scale fixture described in the feature +func generateScaleDocuments(values, count, seed int, users []scaleUser) []scaleDocument { + random := rand.New(rand.NewPCG(uint64(seed), 3)) //nolint:gosec // reproducible fixture + documents := make([]scaleDocument, 0, count+100) + for i := range count { + projects := 1 + if i >= count*55/100 { + projects = 2 + random.IntN(19) + } + if i >= count*95/100 { + projects = 21 + random.IntN(10) + } + doc := scaleDocument{name: fmt.Sprintf("document-%04d", i), projects: random.Perm(values)[:projects], clearance: random.IntN(4)} + for _, region := range random.Perm(len(scaleRegions))[:1+random.IntN(3)] { + doc.regions = append(doc.regions, scaleRegions[region]) + } + documents = append(documents, doc) + } + for _, user := range users { + if len(user.projects) == 0 { + continue + } + for i := range 25 { + size := 1 + if i%2 != 0 { + size = 2 + random.IntN(min(16, len(user.projects))-1) + } + projects := make([]int, size) + for j, index := range random.Perm(len(user.projects))[:size] { + projects[j] = user.projects[index] + } + documents = append(documents, scaleDocument{ + name: fmt.Sprintf("%s-permitted-%02d", user.name, i), projects: projects, + clearance: user.clearance + random.IntN(4-user.clearance), regions: []string{user.region}, + }) + } + } + // A known value without a subject mapping tests fail-closed behavior. + documents = append(documents, scaleDocument{name: scaleUnmapped, projects: []int{values}, clearance: 3, regions: slices.Clone(scaleRegions)}) + return documents +} + +// Independent fixture oracle: all projects, sufficient clearance, and any region +// must match. Both configured actions grant access; other actions must deny. +func expectedScaleDocument(user scaleUser, doc scaleDocument, action string) authz.Decision { + if action != scaleRead && action != scaleWrite { + return authz.Decision_DECISION_DENY + } + if user.clearance > doc.clearance || !slices.Contains(doc.regions, user.region) { + return authz.Decision_DECISION_DENY + } + for _, value := range doc.projects { + if !slices.Contains(user.projects, value) { + return authz.Decision_DECISION_DENY + } + } + return authz.Decision_DECISION_PERMIT +} + +func scaleDocumentFQNs(doc scaleDocument, attribute *policy.Attribute, classification, region *policy.Attribute) []string { + fqns := make([]string, 0, len(doc.projects)+1+len(doc.regions)) + for _, value := range doc.projects { + fqns = append(fqns, fmt.Sprintf("%s/value/v%04d", attribute.GetFqn(), value)) + } + fqns = append(fqns, classification.GetFqn()+"/value/"+scaleClearances[doc.clearance]) + for _, value := range doc.regions { + fqns = append(fqns, region.GetFqn()+"/value/"+value) + } + slices.Sort(fqns) + return fqns +} + +func buildGeneratedScaleCases(ctx context.Context, attributeRef string, count, seed int) ([]authorizationScaleCase, error) { + scenario := GetPlatformScenarioContext(ctx) + users, ok := scenario.GetObject(scaleUsersKey).([]scaleUser) + if !ok { + return nil, errors.New("scale users must be prepared before platform setup") + } + attribute, ok := scenario.GetObject(attributeRef).(*policy.Attribute) + if !ok || len(attribute.GetValues()) < 501 { + return nil, errors.New("missing large scale attribute") + } + classification, ok := scenario.GetObject("classification").(*policy.Attribute) + if !ok { + return nil, errors.New("missing classification attribute") + } + region, ok := scenario.GetObject("region").(*policy.Attribute) + if !ok { + return nil, errors.New("missing region attribute") + } + documents := generateScaleDocuments(len(attribute.GetValues())-1, count, seed, users) + generated := len(documents) + documents = slices.DeleteFunc(documents, func(doc scaleDocument) bool { return len(doc.projects)+1+len(doc.regions) > scaleMaxFQNs }) + scenario.RecordObject("scale-fixture-description", fmt.Sprintf("%d project values (allOf), 4 classification levels (hierarchy), 7 regions (anyOf); %d distinct subject mappings; %d resource mappings with 2-5 aliases; %d users holding 3/10/50/500/0 projects. %d eligible resource documents, %d over-limit documents excluded. Includes 100 permitted examples and one additional unmapped value. Both case and resource variant selection are seeded; case categories are sampled uniformly, not weighted as customer traffic.", len(attribute.GetValues())-1, len(attribute.GetValues())-1+len(scaleClearances)+len(scaleRegions), len(attribute.GetValues())-1, len(users), len(documents), generated-len(documents))) + var cases []authorizationScaleCase + groups := make(map[string]int) + uniqueVariants := make(map[string]bool) + for _, user := range users { + chain, err := buildEntityChainFromIDs(scenario, user.name) + if err != nil { + return nil, err + } + for _, action := range []string{scaleRead, scaleWrite, "delete"} { + add := func(category string, docs []scaleDocument) { + request := &authz.GetDecisionMultiResourceRequest{ + EntityIdentifier: &authz.EntityIdentifier{Identifier: &authz.EntityIdentifier_EntityChain{EntityChain: chain}}, + Action: &policy.Action{Name: action}, + } + expected := make(map[string]authz.Decision) + decisions := make([]string, len(docs)) + labels := make([]string, len(docs)) + for i, doc := range docs { + id := fmt.Sprintf("resource%d", i) + expected[id] = expectedScaleDocument(user, doc, action) + decisions[i] = strings.TrimPrefix(expected[id].String(), "DECISION_") + labels[i] = category + " pool" + request.Resources = append(request.Resources, &authz.Resource{ + EphemeralId: id, + Resource: &authz.Resource_AttributeValues_{AttributeValues: &authz.Resource_AttributeValues{Fqns: scaleDocumentFQNs(doc, attribute, classification, region)}}, + }) + } + name := strings.Join([]string{user.name, action, category, strings.Join(decisions, "/")}, " ") + index, exists := groups[name] + if !exists { + index = len(cases) + groups[name] = index + cases = append(cases, authorizationScaleCase{name: name, entity: user.name, action: action, resources: labels, expected: expected}) + } + var fingerprint strings.Builder + fingerprint.WriteString(name) + for _, resource := range request.GetResources() { + fingerprint.WriteString("|") + fingerprint.WriteString(strings.Join(resource.GetAttributeValues().GetFqns(), ",")) + } + if key := fingerprint.String(); !uniqueVariants[key] { + uniqueVariants[key] = true + cases[index].variants = append(cases[index].variants, request) + } + } + for _, doc := range documents { + category := "single-project" + if len(doc.projects) > 1 { + category = "multi-project" + } + if doc.name == scaleUnmapped { + category = scaleUnmapped + } + add(category, []scaleDocument{doc}) + } + // Pair a user's permitted document with randomly selected documents. This + // covers mixed decisions within one request as well as between requests. + random := rand.New(rand.NewPCG(uint64(seed), scaleResourceStream)) //nolint:gosec // reproducible fixture + for _, doc := range documents { + if strings.HasPrefix(doc.name, user.name+"-permitted-") { + add("three-resource", []scaleDocument{doc, documents[random.IntN(len(documents))], documents[random.IntN(len(documents))]}) + } + } + } + } + return cases, nil +} + +func exerciseGeneratedAuthorizationLoad(ctx context.Context, requests, concurrency, seed int, requestTimeout, attributeRef string, documents int) (context.Context, error) { + timeout, err := time.ParseDuration(requestTimeout) + if err != nil || timeout <= 0 || concurrency < 1 || requests < concurrency || seed < 0 || documents < 100 { + return ctx, errors.New("invalid generated load dimensions or timeout") + } + cases, err := buildGeneratedScaleCases(ctx, attributeRef, documents, seed) + if err != nil { + return ctx, err + } + return reportAuthorizationLoad(ctx, cases, requests, concurrency, seed, timeout) +} diff --git a/tests-bdd/cukes/steps_authorization_scale_generated_test.go b/tests-bdd/cukes/steps_authorization_scale_generated_test.go new file mode 100644 index 0000000000..046e4344b3 --- /dev/null +++ b/tests-bdd/cukes/steps_authorization_scale_generated_test.go @@ -0,0 +1,158 @@ +package cukes + +import ( + "context" + "fmt" + "slices" + "strings" + "testing" + "time" + + authz "github.com/opentdf/platform/protocol/go/authorization/v2" + "github.com/opentdf/platform/protocol/go/policy" + "github.com/stretchr/testify/require" +) + +func TestScaleDocumentExpectations(t *testing.T) { + user := scaleUser{projects: []int{2, 7}, clearance: 1, region: scaleRegions[1]} + tests := []struct { + name string + projects []int + clearance int + regions []string + action string + want authz.Decision + }{ + {"all projects and higher clearance", []int{2, 7}, 2, []string{"region-a", scaleRegions[1]}, "read", authz.Decision_DECISION_PERMIT}, + {"write and exact clearance", []int{7}, 1, []string{scaleRegions[1]}, "write", authz.Decision_DECISION_PERMIT}, + {"missing one project", []int{2, 8}, 2, []string{scaleRegions[1]}, "read", authz.Decision_DECISION_DENY}, + {"insufficient clearance", []int{2}, 0, []string{scaleRegions[1]}, "read", authz.Decision_DECISION_DENY}, + {"no matching region", []int{2}, 2, []string{"region-c"}, "read", authz.Decision_DECISION_DENY}, + {"ungranted action", []int{2}, 2, []string{scaleRegions[1]}, "delete", authz.Decision_DECISION_DENY}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + require.Equal(t, tc.want, expectedScaleDocument(user, scaleDocument{projects: tc.projects, clearance: tc.clearance, regions: tc.regions}, tc.action)) + }) + } +} + +func TestScaleFixtureDistributionAndDistinctConditions(t *testing.T) { + users := generateScaleUsers(6000, 4625) + require.Equal(t, users, generateScaleUsers(6000, 4625)) + require.NotEqual(t, users, generateScaleUsers(6000, 4626)) + for i, count := range []int{3, 10, 50, 500, 0} { + require.Len(t, users[i].projects, count) + for _, project := range users[i].projects { + require.Less(t, project, 6000) + } + } + docs := generateScaleDocuments(6000, 1000, 4625, users) + require.Equal(t, docs, generateScaleDocuments(6000, 1000, 4625, users)) + require.Len(t, docs, 1101) + single, multi, over := 0, 0, 0 + seen := map[int]bool{} + for _, doc := range docs[:1000] { + switch { + case len(doc.projects) == 1: + single++ + case len(doc.projects) <= 20: + multi++ + default: + over++ + } + require.Len(t, slices.Compact(slices.Sorted(slices.Values(doc.projects))), len(doc.projects)) + for _, project := range doc.projects { + seen[project] = true + } + } + require.Equal(t, 550, single) + require.Equal(t, 400, multi) + require.Equal(t, 50, over) + require.Greater(t, len(seen), 3000, "resource combinations must span the large policy") + for _, doc := range docs[1000:1100] { + for _, user := range users { + if strings.HasPrefix(doc.name, user.name+"-permitted-") { + require.Equal(t, authz.Decision_DECISION_PERMIT, expectedScaleDocument(user, doc, "read")) + } + } + } + for _, user := range users { + require.Equal(t, authz.Decision_DECISION_DENY, expectedScaleDocument(user, docs[1100], "read")) + } + conditions := scaleValueConditions(".attributes.projects[]", []string{"v3000", "project-v3000"}) + group := conditions.GetSubjectSets()[0].GetConditionGroups()[0] + require.Equal(t, policy.ConditionBooleanTypeEnum_CONDITION_BOOLEAN_TYPE_ENUM_OR, group.GetBooleanOperator()) + require.Len(t, group.GetConditions(), 2) + require.Equal(t, []string{"v3000", "project-v3000"}, group.GetConditions()[0].GetSubjectExternalValues()) + require.Equal(t, ".clientId", group.GetConditions()[1].GetSubjectExternalSelectorValue()) + require.Len(t, group.GetConditions()[1].GetSubjectExternalValues(), 4) +} + +func TestGeneratedScaleCasesVaryRealResourcesAndKeepValidRequests(t *testing.T) { + scenario := &PlatformScenarioContext{objects: make(map[string]any)} + ctx := context.WithValue(context.Background(), platformScenarioContextKey{}, scenario) + ctx, err := prepareScaleUsers(ctx, 6000, 4625) + require.NoError(t, err) + scenario.RecordObject("projects", &policy.Attribute{Fqn: "https://scale.example/attr/project", Values: make([]*policy.Value, 6001)}) + scenario.RecordObject("classification", &policy.Attribute{Fqn: "https://scale.example/attr/classification"}) + scenario.RecordObject("region", &policy.Attribute{Fqn: "https://scale.example/attr/region"}) + cases, err := buildGeneratedScaleCases(ctx, "projects", 1000, 4625) + require.NoError(t, err) + seen := map[string]bool{} + mixed := false + for _, item := range cases { + require.NotEmpty(t, item.variants) + unique := make(map[string]bool) + for _, request := range item.variants { + var signature strings.Builder + require.Len(t, request.GetResources(), len(item.expected)) + for _, resource := range request.GetResources() { + fqns := resource.GetAttributeValues().GetFqns() + signature.WriteString("|") + signature.WriteString(strings.Join(fqns, ",")) + require.LessOrEqual(t, len(fqns), 20) + require.GreaterOrEqual(t, len(fqns), 3) + for _, fqn := range fqns { + seen[fqn] = true + } + } + require.False(t, unique[signature.String()], "reported variants must contain different resource attributes") + unique[signature.String()] = true + } + if item.expected["resource0"] == authz.Decision_DECISION_PERMIT && item.expected["resource1"] == authz.Decision_DECISION_DENY { + mixed = true + } + } + require.True(t, mixed, "multi-resource traffic must include different decisions in one request") + require.Greater(t, len(seen), 2500, "eligible documents must still span thousands of values after excluding over-limit requests") + selected := selectAuthorizationCases(len(cases), 200, 4625) + selectedUsers, selectedActions := map[string]bool{}, map[string]bool{} + for _, index := range selected { + selectedUsers[cases[index].entity] = true + selectedActions[cases[index].action] = true + } + require.Len(t, selectedUsers, 5) + require.Len(t, selectedActions, 3) + t.Logf("%d case categories, %d distinct FQNs in the resource pool", len(cases), len(seen)) +} + +func TestScaleLoadSelectsResourceVariants(t *testing.T) { + item := loadTestCase("generated") + for i := range 50 { + item.variants = append(item.variants, loadTestCase(fmt.Sprintf("document-%d", i)).request) + } + var previous authorizationPerformanceResult + for _, concurrency := range []int{1, 10} { + result, err := runAuthorizationScaleLoad(t.Context(), []authorizationScaleCase{item}, 200, concurrency, 4625, time.Second, + func(_ context.Context, request *authz.GetDecisionMultiResourceRequest) (*authz.GetDecisionMultiResourceResponse, error) { + return permittedLoadResponse(request), nil + }) + require.NoError(t, err) + require.Greater(t, result.Cases[0].VariantsUsed, 40) + if concurrency > 1 { + require.Equal(t, previous.Cases, result.Cases, "scheduling must not alter selected variants") + } + previous = result + } +} diff --git a/tests-bdd/cukes/steps_authorization_scale_policy.go b/tests-bdd/cukes/steps_authorization_scale_policy.go index 6b2cebcfa4..2dd946bae2 100644 --- a/tests-bdd/cukes/steps_authorization_scale_policy.go +++ b/tests-bdd/cukes/steps_authorization_scale_policy.go @@ -71,12 +71,7 @@ func createScaleGrants(ctx context.Context, table *godog.Table) (context.Context } response, err := scenario.SDK.SubjectMapping.CreateSubjectMapping(ctx, &subjectmapping.CreateSubjectMappingRequest{ AttributeValueId: value.GetId(), Actions: GetActionsFromValues(&row[3], nil), - NewSubjectConditionSet: &subjectmapping.SubjectConditionSetCreate{SubjectSets: []*policy.SubjectSet{{ - ConditionGroups: []*policy.ConditionGroup{{ - BooleanOperator: policy.ConditionBooleanTypeEnum_CONDITION_BOOLEAN_TYPE_ENUM_OR, - Conditions: []*policy.Condition{{SubjectExternalSelectorValue: row[1], Operator: policy.SubjectMappingOperatorEnum_SUBJECT_MAPPING_OPERATOR_ENUM_IN, SubjectExternalValues: strings.Split(row[2], ",")}}, - }}, - }}}, + NewSubjectConditionSet: scaleValueConditions(row[1], strings.Split(row[2], ",")), }) if err != nil { return ctx, err diff --git a/tests-bdd/cukes/steps_localplatform.go b/tests-bdd/cukes/steps_localplatform.go index 100a3c690b..edfbae266f 100644 --- a/tests-bdd/cukes/steps_localplatform.go +++ b/tests-bdd/cukes/steps_localplatform.go @@ -64,8 +64,6 @@ type platformStartOptions struct { } func (s *LocalPlatformStepDefinitions) aUser(ctx context.Context, username string, email string, attributes *godog.Table) (context.Context, error) { - scenarioContext := GetPlatformScenarioContext(ctx) - var users []map[string]any attributeMap := map[string]any{} cellMap := map[string]int{} for ri, row := range attributes.Rows { @@ -85,6 +83,12 @@ func (s *LocalPlatformStepDefinitions) aUser(ctx context.Context, username strin } } } + return registerLocalUser(ctx, username, email, attributeMap) +} + +func registerLocalUser(ctx context.Context, username, email string, attributeMap map[string]any) (context.Context, error) { + scenarioContext := GetPlatformScenarioContext(ctx) + var users []map[string]any userObj := scenarioContext.GetObject(userContextKey) if userObj != nil { usersObj, ok := scenarioContext.GetObject(userContextKey).([]map[string]any) diff --git a/tests-bdd/cukes/steps_resourcemappings_scale.go b/tests-bdd/cukes/steps_resourcemappings_scale.go index 4584075b88..a7ee88877f 100644 --- a/tests-bdd/cukes/steps_resourcemappings_scale.go +++ b/tests-bdd/cukes/steps_resourcemappings_scale.go @@ -20,7 +20,7 @@ func createScaleResourceMappings(ctx context.Context, count int, attributeRef, n err = createScaleMappings(ctx, count, func(ctx context.Context, index int) error { response, err := scenario.SDK.ResourceMapping.CreateResourceMapping(ctx, &resourcemapping.CreateResourceMappingRequest{ AttributeValueId: attribute.GetValues()[index].GetId(), NamespaceId: namespace, - Terms: []string{fmt.Sprintf("resource-%04d", index)}, + Terms: scaleResourceTerms(index), }) if err != nil { return fmt.Errorf("create resource mapping %d: %w", index, err) @@ -32,3 +32,12 @@ func createScaleResourceMappings(ctx context.Context, count int, attributeRef, n }) return ctx, err } + +// Two to five aliases per mapping, with distinct public synthetic identifiers. +func scaleResourceTerms(index int) []string { + terms := []string{fmt.Sprintf("resource-%04d", index), fmt.Sprintf("project-v%04d", index)} + for alias := range index % 4 { + terms = append(terms, fmt.Sprintf("alias-%d-v%04d", alias, index)) + } + return terms +} diff --git a/tests-bdd/cukes/steps_subjectmappings.go b/tests-bdd/cukes/steps_subjectmappings.go index 56066c6e84..d20d52c04f 100644 --- a/tests-bdd/cukes/steps_subjectmappings.go +++ b/tests-bdd/cukes/steps_subjectmappings.go @@ -222,7 +222,7 @@ func (s *SubjectMappingsStepDefinitions) iSendARequestToCreateSubjectMappingForE func RegisterSubjectMappingsStepsDefinitions(ctx *godog.ScenarioContext) { subjectMappingStepDefinitions := &SubjectMappingsStepDefinitions{} - ctx.Step(`^I create (\d+) subject mappings for attribute "([^"]*)" using condition set "([^"]*)" with action "([^"]*)"$`, subjectMappingStepDefinitions.createScaleSubjectMappings) + ctx.Step(`^I create (\d+) subject mappings for attribute "([^"]*)" matching selector "([^"]*)" with action "([^"]*)"$`, subjectMappingStepDefinitions.createScaleSubjectMappings) ctx.Step(`a condition group referenced as "([^"]*)" with an "([^"]*)" operator with conditions:$`, subjectMappingStepDefinitions.aConditionGroup) ctx.Step(`^a subject set referenced as "([^"]*)" containing the condition groups "([^"]*)"$`, subjectMappingStepDefinitions.aSubjectSet) ctx.Step(`^I send a request to create a subject condition set referenced as "([^"]*)" containing subject sets "([^"]*)"$`, subjectMappingStepDefinitions.iSendARequestToCreateSubjectConditionSet) diff --git a/tests-bdd/cukes/steps_subjectmappings_scale.go b/tests-bdd/cukes/steps_subjectmappings_scale.go index 47db585ffc..c66f639169 100644 --- a/tests-bdd/cukes/steps_subjectmappings_scale.go +++ b/tests-bdd/cukes/steps_subjectmappings_scale.go @@ -10,20 +10,32 @@ import ( "github.com/opentdf/platform/protocol/go/policy/subjectmapping" ) -func (s *SubjectMappingsStepDefinitions) createScaleSubjectMappings(ctx context.Context, count int, attributeRef, conditionSetRef, action string) (context.Context, error) { +// Each value has a distinct condition set, matching either a user's entitlement +// or an approved client. Sharing one condition set hides policy evaluation costs. +func scaleValueConditions(selector string, values []string) *subjectmapping.SubjectConditionSetCreate { + return &subjectmapping.SubjectConditionSetCreate{SubjectSets: []*policy.SubjectSet{{ + ConditionGroups: []*policy.ConditionGroup{{ + BooleanOperator: policy.ConditionBooleanTypeEnum_CONDITION_BOOLEAN_TYPE_ENUM_OR, + Conditions: []*policy.Condition{ + {SubjectExternalSelectorValue: selector, Operator: policy.SubjectMappingOperatorEnum_SUBJECT_MAPPING_OPERATOR_ENUM_IN, SubjectExternalValues: values}, + {SubjectExternalSelectorValue: ".clientId", Operator: policy.SubjectMappingOperatorEnum_SUBJECT_MAPPING_OPERATOR_ENUM_IN, SubjectExternalValues: []string{"scale-browser", "scale-mail", "scale-gateway", "scale-cli"}}, + }, + }}, + }}} +} + +func (s *SubjectMappingsStepDefinitions) createScaleSubjectMappings(ctx context.Context, count int, attributeRef, selector, action string) (context.Context, error) { scenario := GetPlatformScenarioContext(ctx) attribute, ok := scenario.GetObject(attributeRef).(*policy.Attribute) if !ok || count <= 0 || count > len(attribute.GetValues()) { return ctx, fmt.Errorf("attribute %q must contain at least %d values", attributeRef, count) } - conditionSet, ok := scenario.GetObject(conditionSetRef).(*policy.SubjectConditionSet) - if !ok { - return ctx, fmt.Errorf("missing condition set %q", conditionSetRef) - } create := func(ctx context.Context, index int) error { + value := attribute.GetValues()[index] response, err := scenario.SDK.SubjectMapping.CreateSubjectMapping(ctx, &subjectmapping.CreateSubjectMappingRequest{ - AttributeValueId: attribute.GetValues()[index].GetId(), ExistingSubjectConditionSetId: conditionSet.GetId(), - Actions: GetActionsFromValues(&action, nil), + AttributeValueId: value.GetId(), + NewSubjectConditionSet: scaleValueConditions(selector, []string{value.GetValue(), "project-" + value.GetValue()}), + Actions: GetActionsFromValues(&action, nil), }) if err != nil { return fmt.Errorf("create subject mapping %d: %w", index, err) @@ -31,20 +43,13 @@ func (s *SubjectMappingsStepDefinitions) createScaleSubjectMappings(ctx context. if response.GetSubjectMapping().GetId() == "" { return fmt.Errorf("subject mapping %d returned no identity", index) } - if err := validateScaleMappingActions(response.GetSubjectMapping().GetActions(), action); err != nil { - return fmt.Errorf("subject mapping %d: %w", index, err) - } - return nil + return validateScaleMappingActions(response.GetSubjectMapping().GetActions(), action) } - // Resolve any new action names before concurrent mapping creation starts. - // Concurrent create-or-list calls can otherwise return an incomplete action set. + // Resolve new action names before concurrent mapping creation starts. if err := create(ctx, 0); err != nil { return ctx, err } - err := createScaleMappings(ctx, count-1, func(ctx context.Context, index int) error { - return create(ctx, index+1) - }) - return ctx, err + return ctx, createScaleMappings(ctx, count-1, func(ctx context.Context, index int) error { return create(ctx, index+1) }) } func validateScaleMappingActions(actions []*policy.Action, expected string) error { diff --git a/tests-bdd/features/authorization-v2-subject-mapping-performance.feature b/tests-bdd/features/authorization-v2-subject-mapping-performance.feature index b745b2b012..e4d0d3fb2a 100644 --- a/tests-bdd/features/authorization-v2-subject-mapping-performance.feature +++ b/tests-bdd/features/authorization-v2-subject-mapping-performance.feature @@ -1,100 +1,44 @@ @authorization @authz-v2 @performance @scale Feature: Mixed authorization traffic at large policy scale - Concurrent workers continuously select requests from a pool of entitlement cases. - The pool varies users, actions, anyOf/allOf/hierarchy rules, combined attributes, - and multiple resources. Every resource includes a value from the large attribute - so the measured requests exercise the large-policy lookup path. - The seed selects the same request mix at every concurrency level. Each selection - is independent; the summary reports how often each case was selected, including zero. - Setup is excluded. Latency is report-only; incorrect decisions, errors, and client - timeouts fail. The fixture's server write timeout exceeds the client deadline. + Policy has 6,000 allOf project values, four hierarchy levels, seven anyOf regions, + 6,011 distinct subject mappings, and 6,000 resource mappings with multiple aliases. + Each mapping matches its own user entitlement OR an approved client ID. + Five users hold 3, 10, 50, 500, and zero projects spread across the full policy. + Resources combine projects, classification, and regions. Of 1,000 generated + documents, 55% have one project, 40% have 2-20, and 5% have 21-30. + Documents exceeding 20 total FQNs are excluded from load. Another 100 documents + provide permitted examples, and one unmapped value checks fail-closed behavior. + Workers randomly select a user/action/decision case and then a resource variant. + Requests vary read, write, denied delete, and one or three resources. + The seed reproduces both selections at every concurrency level. Setup is excluded. + Latency is report-only; incorrect decisions, errors, and 30-second timeouts fail. Scenario Outline: Random entitlement requests at concurrency - Given a user exists with username "engineer" and email "engineer@example.com" and the following attributes: - | name | value | - | population | ["load"] | - | department | ["engineering"] | - | projects | ["alpha","beta"] | - | clearance | ["high"] | - And a user exists with username "analyst" and email "analyst@example.com" and the following attributes: - | name | value | - | population | ["load"] | - | department | ["sales"] | - | projects | ["alpha"] | - | clearance | ["low"] | - And a user exists with username "visitor" and email "visitor@example.com" and the following attributes: - | name | value | - | population | ["load"] | - | department | ["none"] | - | projects | ["none"] | - | clearance | ["none"] | + Given representative scale users hold subsets of 6000 project values with seed 4625 And an empty local platform with HTTP write timeout "35s" And I submit a request to create a namespace with name "scale.example" and reference id "scale_ns" - And I send a request to create an attribute referenced as "scale_attr" in namespace "scale_ns" named "access-level" with rule "anyOf" and 6012 generated values in batches of 25 + And I send a request to create an attribute referenced as "projects" in namespace "scale_ns" named "project" with rule "allOf" and 6001 generated values in batches of 25 Then the response should be successful - And a condition group referenced as "scale_condition" with an "or" operator with conditions: - | selector_value | operator | values | - | .attributes.population[] | in | load | - And a subject set referenced as "scale_subject_set" containing the condition groups "scale_condition" - And I send a request to create a subject condition set referenced as "scale_condition_set" containing subject sets "scale_subject_set" - Then the response should be successful - And I create 6011 subject mappings for attribute "scale_attr" using condition set "scale_condition_set" with action "read,write" - And I create 6000 resource mappings for attribute "scale_attr" in namespace "scale_ns" + And I create 6000 subject mappings for attribute "projects" matching selector ".attributes.projects[]" with action "read,write" + And I create 6000 resource mappings for attribute "projects" in namespace "scale_ns" And the following scale attributes exist in namespace "scale_ns": - | attribute | rule | values | - | department | anyOf | engineering,sales | - | project | allOf | alpha,beta | - | clearance | hierarchy | critical,high,medium,low | + | attribute | rule | values | + | classification | hierarchy | critical,high,medium,low | + | region | anyOf | region-a,region-b,region-c,region-d,region-e,region-f,region-g | And the following scale grants exist: - | attribute value | selector | matches | actions | - | department/engineering | .attributes.department[] | engineering | read | - | department/sales | .attributes.department[] | sales | read | - | project/alpha | .attributes.projects[] | alpha | read,write | - | project/beta | .attributes.projects[] | beta | read,write | - | clearance/high | .attributes.clearance[] | high | read | - | clearance/low | .attributes.clearance[] | low | read | - And the following scale resources are defined: - | resource | attributes | - | engineering | scale_attr/v0000,department/engineering | - | sales | scale_attr/v3005,department/sales | - | either-team | scale_attr/v6010,department/engineering,department/sales | - | alpha | scale_attr/v0000,project/alpha | - | projects | scale_attr/v3005,project/alpha,project/beta | - | medium | scale_attr/v6010,clearance/medium | - | low | scale_attr/v0000,clearance/low | - | critical | scale_attr/v3005,clearance/critical | - | team-project | scale_attr/v6010,department/engineering,project/alpha,project/beta | - | all-rules | scale_attr/v0000,department/engineering,department/sales,project/alpha,project/beta,clearance/medium | - | unmapped | scale_attr/v6011 | - And there is a "user_name" subject entity with value "engineer" and referenced as "engineer" - And there is a "user_name" subject entity with value "analyst" and referenced as "analyst" - And there is a "user_name" subject entity with value "visitor" and referenced as "visitor" - When I send 200 randomly selected authorization requests with concurrency , seed 4625, and request timeout "30s": - | case | user | action | resources | expected | - | anyOf matching team | engineer | read | engineering | PERMIT | - | anyOf other team | engineer | read | sales | DENY | - | anyOf one matching value | analyst | read | either-team | PERMIT | - | anyOf no matching value | visitor | read | either-team | DENY | - | allOf all values granted | engineer | read | projects | PERMIT | - | allOf missing one grant | analyst | read | projects | DENY | - | allOf single value | analyst | read | alpha | PERMIT | - | allOf no grants | visitor | read | projects | DENY | - | hierarchy higher grant | engineer | read | medium,low | PERMIT,PERMIT | - | hierarchy lower grant | analyst | read | medium | DENY | - | hierarchy exact grant | analyst | read | low | PERMIT | - | hierarchy above clearance | engineer | read | critical | DENY | - | hierarchy no grant | visitor | read | low | DENY | - | write granted | engineer | write | projects | PERMIT | - | write partial grants | analyst | write | alpha,projects | PERMIT,DENY | - | write not granted | engineer | write | engineering | DENY | - | combined attributes pass | engineer | read | team-project,all-rules | PERMIT,PERMIT | - | combined attributes fail | analyst | read | team-project,all-rules | DENY,DENY | - | combined action denied | engineer | write | team-project | DENY | - | mixed resource decisions | engineer | read | engineering,sales,projects | PERMIT,DENY,PERMIT | - | different user decisions | analyst | read | engineering,sales,projects | DENY,PERMIT,DENY | - | no attribute grants | visitor | read | engineering,sales,projects | DENY,DENY,DENY | - | no mapping for value | engineer | read | unmapped | DENY | - | mixed mapped and unmapped | engineer | read | engineering,unmapped | PERMIT,DENY | + | attribute value | selector | matches | actions | + | classification/critical | .attributes.clearance[] | critical | read,write | + | classification/high | .attributes.clearance[] | high | read,write | + | classification/medium | .attributes.clearance[] | medium | read,write | + | classification/low | .attributes.clearance[] | low | read,write | + | region/region-a | .attributes.regions[] | region-a | read,write | + | region/region-b | .attributes.regions[] | region-b | read,write | + | region/region-c | .attributes.regions[] | region-c | read,write | + | region/region-d | .attributes.regions[] | region-d | read,write | + | region/region-e | .attributes.regions[] | region-e | read,write | + | region/region-f | .attributes.regions[] | region-f | read,write | + | region/region-g | .attributes.regions[] | region-g | read,write | + When I send 200 generated authorization requests with concurrency , seed 4625, request timeout "30s", attribute "projects", and 1000 documents @concurrency-1 Examples: One worker