diff --git a/.github/scripts/summarize-authz-performance.py b/.github/scripts/summarize-authz-performance.py new file mode 100644 index 0000000000..a79b9919b7 --- /dev/null +++ b/.github/scripts/summarize-authz-performance.py @@ -0,0 +1,83 @@ +#!/usr/bin/env python3 +"""Render structured authorization measurements, including failed BDD runs.""" +import argparse +import json +from pathlib import Path + +MARKER = "AUTHZ_PERFORMANCE " +NUMBERS = ( + "seed", "concurrency", "resources", "wall_ns", "median_ns", "p95_ns", + "maximum_ns", "timeout_ns", "failures", +) + + +def read_results(text): + results, malformed = [], 0 + for line in text.splitlines(): + # Support both console logs and go test -json artifacts. + if line.startswith("{"): + try: + output = json.loads(line).get("Output") + if isinstance(output, str): + line = output + except (ValueError, AttributeError): + pass + if MARKER not in line: + continue + try: + result = json.loads(line.split(MARKER, 1)[1]) + if not isinstance(result.get("case"), str) or not result["case"]: + raise ValueError("missing case") + if any(type(result.get(key)) is not int or result[key] < 0 for key in NUMBERS): + raise ValueError("invalid numeric field") + if not result["concurrency"] or not result["resources"] or not result["timeout_ns"]: + raise ValueError("invalid dimensions") + if result["failures"] > result["concurrency"]: + raise ValueError("invalid failure count") + results.append(result) + except (ValueError, TypeError, AttributeError): + malformed += 1 + return sorted(results, key=lambda row: (row["concurrency"], row["case"], row["seed"])), malformed + + +def milliseconds(nanoseconds): + return f"{nanoseconds / 1_000_000:.2f} ms" + + +def render(text, outcome): + results, malformed = read_results(text) + lines = ["### Authorization v2 concurrency performance", "", f"BDD step outcome: **{outcome}**.", ""] + if results: + lines += [ + "| Case | Concurrency | Resources | Seed | Wall | Median | p95 | Maximum | Timeout | Failures | Requests |", + "| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | --- |", + ] + for row in results: + case = row["case"].replace("|", "\\|").replace("\n", " ").replace("\r", " ") + status = "FAIL" if row["failures"] else "PASS" + cells = [case, str(row["concurrency"]), str(row["resources"]), str(row["seed"])] + cells += [milliseconds(row[key]) for key in ("wall_ns", "median_ns", "p95_ns", "maximum_ns", "timeout_ns")] + cells += [str(row["failures"]), status] + lines.append("| " + " | ".join(cells) + " |") + lines += ["", f"Reported {len(results)} completed case batches. Missing measurements are not passes."] + else: + lines.append("No authorization measurements were produced. Check BDD setup and logs; this is not a passing performance result.") + if malformed: + lines += ["", f"**Summary error: {malformed} malformed performance record(s).**"] + lines += ["", "Fixture setup is excluded. PASS means requests completed with the expected decisions. Failures include request errors, timeouts, and incorrect decisions. Latency is report-only until a baseline is established; the request timeout is not a latency target.", ""] + return "\n".join(lines), malformed + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("log", type=Path) + parser.add_argument("--bdd-outcome", default="unknown", choices=["success", "failure", "cancelled", "skipped", "unknown"]) + args = parser.parse_args() + text = args.log.read_text() if args.log.exists() else "" + summary, malformed = render(text, args.bdd_outcome) + print(summary) + return bool(malformed) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/.github/scripts/test_summarize_authz_performance.py b/.github/scripts/test_summarize_authz_performance.py new file mode 100644 index 0000000000..7e39142e62 --- /dev/null +++ b/.github/scripts/test_summarize_authz_performance.py @@ -0,0 +1,61 @@ +import importlib.util +import json +import subprocess +import sys +import tempfile +from pathlib import Path +import unittest + +spec = importlib.util.spec_from_file_location("summary", Path(__file__).with_name("summarize-authz-performance.py")) +summary = importlib.util.module_from_spec(spec) +spec.loader.exec_module(summary) + + +class SummaryTests(unittest.TestCase): + def record(self, **changes): + row = dict(case="allowed_read", concurrency=50, resources=3, seed=4625, + wall_ns=150000000, median_ns=100000000, p95_ns=130000000, + maximum_ns=140000000, timeout_ns=30000000000, failures=0) + row.update(changes) + return summary.MARKER + json.dumps(row) + + def test_console_and_json_records_have_identical_rendering(self): + record = self.record() + console, errors = summary.render(record, "success") + encoded, _ = summary.render(json.dumps({"Action": "output", "Output": record + "\n"}), "success") + self.assertEqual(console, encoded) + self.assertEqual(errors, 0) + self.assertIn("130.00 ms | 140.00 ms | 30000.00 ms | 0 | PASS", console) + + def test_partial_failure_keeps_rows_without_gating_slow_requests(self): + text = self.record(case="denied_user", failures=2) + "\n" + self.record(case="allowed_read", maximum_ns=8000000000) + rendered, errors = summary.render(text, "failure") + self.assertEqual(errors, 0) + self.assertIn("BDD step outcome: **failure**", rendered) + self.assertIn("8000.00 ms | 30000.00 ms | 0 | PASS |", rendered) + self.assertIn("| 2 | FAIL |", rendered) + self.assertIn("Latency is report-only", rendered) + self.assertLess(rendered.index("allowed_read"), rendered.index("denied_user")) + + def test_cli_handles_missing_log_after_setup_failure(self): + with tempfile.TemporaryDirectory() as directory: + result = subprocess.run( + [sys.executable, str(Path(__file__).with_name("summarize-authz-performance.py")), + str(Path(directory) / "missing.log"), "--bdd-outcome", "failure"], + capture_output=True, text=True, check=True, + ) + self.assertIn("No authorization measurements", result.stdout) + self.assertIn("BDD step outcome: **failure**", result.stdout) + + def test_missing_and_malformed_measurements_are_not_passes(self): + rendered, errors = summary.render("setup failed", "failure") + self.assertIn("No authorization measurements", rendered) + self.assertEqual(errors, 0) + rendered, errors = summary.render(self.record() + "\n" + summary.MARKER + "{}", "failure") + self.assertEqual(errors, 1) + self.assertIn("malformed performance record", rendered) + self.assertIn("allowed_read", rendered) + + +if __name__ == "__main__": + unittest.main() diff --git a/.github/workflows/checks.yaml b/.github/workflows/checks.yaml index e55262e42e..f4c8333cc8 100644 --- a/.github/workflows/checks.yaml +++ b/.github/workflows/checks.yaml @@ -527,12 +527,22 @@ jobs: go-version-file: go.work cache: false + - name: Test authorization summary renderer + run: python3 -m unittest discover -s .github/scripts -p 'test_summarize_authz_performance.py' + - name: Build local platform-cukes image for testing run: docker build -t platform-cukes . - name: Run BDD Tests + id: bdd + shell: bash + run: | + CUKES_LOG_HANDLER=console go test ./tests-bdd -v -timeout=20m --tags=cukes --godog.random --godog.format="cucumber:$(pwd)/cukes_platform_report.json,pretty:$(pwd)/cukes_platform_report.log,pretty" ./features 2>&1 | tee cukes_test_output.log + + - name: Summarize authorization performance + if: ${{ !cancelled() }} run: | - CUKES_LOG_HANDLER=console go test ./tests-bdd -v --tags=cukes --godog.random --godog.format="cucumber:$(pwd)/cukes_platform_report.json,pretty:$(pwd)/cukes_platform_report.log,pretty" ./features + python3 .github/scripts/summarize-authz-performance.py cukes_test_output.log --bdd-outcome "${{ steps.bdd.outcome || 'skipped' }}" >> "$GITHUB_STEP_SUMMARY" - name: Check for undefined steps run: | @@ -557,6 +567,7 @@ jobs: path: | cukes_platform_report.json cukes_platform_report.log + cukes_test_output.log retention-days: 1 # test otdfctl CLI e2e against platform PR branch diff --git a/tests-bdd/cukes/scale_setup.go b/tests-bdd/cukes/scale_setup.go new file mode 100644 index 0000000000..3eaa49a98f --- /dev/null +++ b/tests-bdd/cukes/scale_setup.go @@ -0,0 +1,42 @@ +package cukes + +import ( + "context" + "errors" + "fmt" + "sync" + + "github.com/opentdf/platform/protocol/go/policy" +) + +// Setup uses bounded API calls and finishes before authorization is measured. +func createScaleMappings(ctx context.Context, count int, create func(context.Context, int) error) error { + // Match the default HTTP idle pool so thousands of setup calls reuse connections. + const batchSize = 2 + for start := 0; start < count; start += batchSize { + end := min(start+batchSize, count) + failures := make([]error, end-start) + var workers sync.WaitGroup + for index := start; index < end; index++ { + workers.Go(func() { failures[index-start] = create(ctx, index) }) + } + workers.Wait() + if err := errors.Join(failures...); err != nil { + return err + } + } + return nil +} + +func scaleMappingInputs(ctx context.Context, count int, attributeRef, namespaceRef string) (*PlatformScenarioContext, *policy.Attribute, string, error) { + scenario := GetPlatformScenarioContext(ctx) + attribute, ok := scenario.GetObject(attributeRef).(*policy.Attribute) + if !ok || count <= 0 || count > len(attribute.GetValues()) { + return nil, nil, "", fmt.Errorf("attribute %q must contain at least %d values for mapping setup", attributeRef, count) + } + namespace, ok := scenario.GetObject(namespaceRef).(string) + if !ok { + return nil, nil, "", fmt.Errorf("missing namespace %q", namespaceRef) + } + return scenario, attribute, namespace, nil +} diff --git a/tests-bdd/cukes/steps_attributes.go b/tests-bdd/cukes/steps_attributes.go index ba8dae68c0..5585edd228 100644 --- a/tests-bdd/cukes/steps_attributes.go +++ b/tests-bdd/cukes/steps_attributes.go @@ -4,7 +4,10 @@ import ( "context" "errors" "fmt" + "log/slog" "strings" + "sync" + "time" "github.com/cucumber/godog" "github.com/opentdf/platform/protocol/go/policy" @@ -19,6 +22,19 @@ type AttributesStepDefinitions struct { PlatformCukesContext *PlatformTestSuiteContext } +func parseAttributeRule(rule string) (policy.AttributeRuleTypeEnum, error) { + switch strings.TrimSpace(rule) { + case "anyOf": + return policy.AttributeRuleTypeEnum_ATTRIBUTE_RULE_TYPE_ENUM_ANY_OF, nil + case "allOf": + return policy.AttributeRuleTypeEnum_ATTRIBUTE_RULE_TYPE_ENUM_ALL_OF, nil + case "hierarchy": + return policy.AttributeRuleTypeEnum_ATTRIBUTE_RULE_TYPE_ENUM_HIERARCHY, nil + default: + return policy.AttributeRuleTypeEnum_ATTRIBUTE_RULE_TYPE_ENUM_UNSPECIFIED, fmt.Errorf("unknown attribute rule type %s", rule) + } +} + func (s *AttributesStepDefinitions) aAttributeDef(ctx context.Context, _ string, _ string) (context.Context, error) { return ctx, nil } @@ -98,16 +114,9 @@ func (s *AttributesStepDefinitions) iSendARequestToCreateAnAttributeWithGenerate return ctx, fmt.Errorf("unable to get namespace id for %s", namespaceRef) } - var ruleType policy.AttributeRuleTypeEnum - switch strings.TrimSpace(rule) { - case "anyOf": - ruleType = policy.AttributeRuleTypeEnum_ATTRIBUTE_RULE_TYPE_ENUM_ANY_OF - case "allOf": - ruleType = policy.AttributeRuleTypeEnum_ATTRIBUTE_RULE_TYPE_ENUM_ALL_OF - case "hierarchy": - ruleType = policy.AttributeRuleTypeEnum_ATTRIBUTE_RULE_TYPE_ENUM_HIERARCHY - default: - return ctx, fmt.Errorf("unknown attribute rule type %s", rule) + ruleType, err := parseAttributeRule(rule) + if err != nil { + return ctx, err } values := make([]string, 0, valueCount) @@ -128,6 +137,86 @@ func (s *AttributesStepDefinitions) iSendARequestToCreateAnAttributeWithGenerate return ctx, nil } +func (s *AttributesStepDefinitions) iSendARequestToCreateAnAttributeWithBatchedGeneratedValues(ctx context.Context, referenceID, namespaceRef, name, rule string, valueCount, batchSize int) (context.Context, error) { + scenarioContext := GetPlatformScenarioContext(ctx) + scenarioContext.ClearError() + if valueCount < 1 { + return ctx, errors.New("generated value count must be positive") + } + if batchSize < 1 { + return ctx, errors.New("generated value batch size must be positive") + } + + namespaceID, ok := scenarioContext.GetObject(strings.TrimSpace(namespaceRef)).(string) + if !ok { + return ctx, fmt.Errorf("unable to get namespace id for %s", namespaceRef) + } + ruleType, err := parseAttributeRule(rule) + if err != nil { + return ctx, err + } + + created, err := scenarioContext.SDK.Attributes.CreateAttribute(ctx, &attributes.CreateAttributeRequest{ + NamespaceId: namespaceID, + Name: strings.TrimSpace(name), + Rule: ruleType, + }) + if err != nil { + scenarioContext.SetError(err) + return ctx, nil + } + if created.GetAttribute() == nil { + return ctx, errors.New("create attribute returned no attribute") + } + + started := time.Now() + values := make([]*policy.Value, valueCount) + for batchStart := 0; batchStart < valueCount; batchStart += batchSize { + batchEnd := min(batchStart+batchSize, valueCount) + batchCtx, cancel := context.WithCancel(ctx) + errCh := make(chan error, batchEnd-batchStart) + var wg sync.WaitGroup + for i := batchStart; i < batchEnd; i++ { + wg.Add(1) + go func(index int) { + defer wg.Done() + resp, createErr := scenarioContext.SDK.Attributes.CreateAttributeValue(batchCtx, &attributes.CreateAttributeValueRequest{ + AttributeId: created.GetAttribute().GetId(), + Value: fmt.Sprintf("v%04d", index), + }) + if createErr != nil { + errCh <- fmt.Errorf("create generated attribute value v%04d: %w", index, createErr) + cancel() + return + } + if resp.GetValue() == nil { + errCh <- fmt.Errorf("create generated attribute value v%04d returned no value", index) + cancel() + return + } + values[index] = resp.GetValue() + }(i) + } + wg.Wait() + cancel() + close(errCh) + if batchErr, hasBatchErr := <-errCh; hasBatchErr { + scenarioContext.SetError(batchErr) + return ctx, nil + } + } + + created.GetAttribute().Values = values + scenarioContext.RecordObject(strings.TrimSpace(referenceID), created.GetAttribute()) + scenarioContext.TestSuiteContext.Logger.Info( + "created generated attribute values in batches", + slog.Int("value_count", valueCount), + slog.Int("batch_size", batchSize), + slog.Duration("duration", time.Since(started)), + ) + return ctx, nil +} + func RegisterAttributeStepDefinitions(ctx *godog.ScenarioContext, x *PlatformTestSuiteContext) { stepDefinitions := AttributesStepDefinitions{ PlatformCukesContext: x, @@ -135,4 +224,5 @@ func RegisterAttributeStepDefinitions(ctx *godog.ScenarioContext, x *PlatformTes ctx.Step(`^a (anyOf|allOf|hierarchy) attribute definition with values: "([^"]*)"$`, stepDefinitions.aAttributeDef) ctx.Step(`^I send a request to create an attribute with:$`, stepDefinitions.iSendARequestToCreateAnAttributeWith) ctx.Step(`^I send a request to create an attribute referenced as "([^"]*)" in namespace "([^"]*)" named "([^"]*)" with rule "([^"]*)" and (\d+) generated values$`, stepDefinitions.iSendARequestToCreateAnAttributeWithGeneratedValues) + ctx.Step(`^I send a request to create an attribute referenced as "([^"]*)" in namespace "([^"]*)" named "([^"]*)" with rule "([^"]*)" and (\d+) generated values in batches of (\d+)$`, stepDefinitions.iSendARequestToCreateAnAttributeWithBatchedGeneratedValues) } diff --git a/tests-bdd/cukes/steps_authorization.go b/tests-bdd/cukes/steps_authorization.go index ed8541d443..ff813a4442 100644 --- a/tests-bdd/cukes/steps_authorization.go +++ b/tests-bdd/cukes/steps_authorization.go @@ -546,6 +546,7 @@ func (s *AuthorizationServiceStepDefinitions) theDecisionResponseForResourceShou } func RegisterAuthorizationStepDefinitions(ctx *godog.ScenarioContext) { + ctx.Step(`^I exercise the authorization cases with (\d+) concurrent requests each, seed (\d+), and request timeout "([^"]*)" for attribute "([^"]*)":$`, exerciseAuthorizationCases) stepDefinitions := AuthorizationServiceStepDefinitions{} ctx.Step(`^there is a "([^"]*)" subject entity with value "([^"]*)" and referenced as "([^"]*)"$`, stepDefinitions.thereIsASubjectEntityWithValueAndReferencedAs) ctx.Step(`^there is a claims subject entity referenced as "([^"]*)" with claims:$`, stepDefinitions.thereIsAClaimsSubjectEntityReferencedAsWithClaims) diff --git a/tests-bdd/cukes/steps_authorization_scale.go b/tests-bdd/cukes/steps_authorization_scale.go new file mode 100644 index 0000000000..147d7bbcb5 --- /dev/null +++ b/tests-bdd/cukes/steps_authorization_scale.go @@ -0,0 +1,210 @@ +package cukes + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "math/rand/v2" + "slices" + "strings" + "sync" + "time" + + "github.com/cucumber/godog" + authz "github.com/opentdf/platform/protocol/go/authorization/v2" + "github.com/opentdf/platform/protocol/go/policy" + "google.golang.org/protobuf/proto" +) + +const authorizationPerformanceMarker = "AUTHZ_PERFORMANCE " + +type authorizationScaleCase struct { + name string + entity string + action string + values []string + expected map[string]authz.Decision +} + +type authorizationPerformanceResult struct { + Case string `json:"case"` + Seed int `json:"seed"` + Concurrency int `json:"concurrency"` + Resources int `json:"resources"` + Wall time.Duration `json:"wall_ns"` + Median time.Duration `json:"median_ns"` + P95 time.Duration `json:"p95_ns"` + Maximum time.Duration `json:"maximum_ns"` + Timeout time.Duration `json:"timeout_ns"` + Failures int `json:"failures"` +} + +func parseAuthorizationScaleCases(table *godog.Table) ([]authorizationScaleCase, error) { + headers := []string{"case", "entity", "action", valuesKey, "expected"} + if table == nil || len(table.Rows) < 2 || len(table.Rows[0].Cells) != len(headers) { + return nil, errors.New("authorization case table requires case, entity, action, values, expected columns") + } + for i, header := range headers { + if table.Rows[0].Cells[i].Value != header { + return nil, fmt.Errorf("expected column %q", header) + } + } + cases := make([]authorizationScaleCase, 0, len(table.Rows)-1) + names := make(map[string]bool) + for _, row := range table.Rows[1:] { + if len(row.Cells) != len(headers) { + return nil, errors.New("authorization case row has incorrect column count") + } + item := authorizationScaleCase{ + name: strings.TrimSpace(row.Cells[0].Value), entity: strings.TrimSpace(row.Cells[1].Value), + action: strings.TrimSpace(row.Cells[2].Value), values: strings.Split(row.Cells[3].Value, ","), + expected: make(map[string]authz.Decision), + } + if item.name == "" || names[item.name] || item.entity == "" || item.action == "" { + return nil, errors.New("cases require unique names, entities, and actions") + } + names[item.name] = true + expected := strings.Split(row.Cells[4].Value, ",") + if len(item.values) != len(expected) { + return nil, fmt.Errorf("case %s has mismatched values and expectations", item.name) + } + for i, value := range item.values { + item.values[i] = strings.TrimSpace(value) + if item.values[i] == "" { + return nil, fmt.Errorf("case %s has an empty value", item.name) + } + decision, ok := authz.Decision_value["DECISION_"+strings.TrimSpace(expected[i])] + if !ok || (authz.Decision(decision) != authz.Decision_DECISION_PERMIT && authz.Decision(decision) != authz.Decision_DECISION_DENY) { + return nil, fmt.Errorf("case %s requires explicit PERMIT or DENY expectations", item.name) + } + item.expected[fmt.Sprintf("resource%d", i)] = authz.Decision(decision) + } + cases = append(cases, item) + } + return cases, nil +} + +func validateScaleDecision(response *authz.GetDecisionMultiResourceResponse, expected map[string]authz.Decision) error { + if response == nil || len(response.GetResourceDecisions()) != len(expected) { + return errors.New("unexpected resource decision count") + } + seen := make(map[string]bool, len(expected)) + for _, decision := range response.GetResourceDecisions() { + id := decision.GetEphemeralResourceId() + want, ok := expected[id] + if !ok || seen[id] { + return fmt.Errorf("unexpected or duplicate resource decision %q", id) + } + seen[id] = true + if decision.GetDecision() != want { + return fmt.Errorf("resource %s: expected %s, got %s", id, want, decision.GetDecision()) + } + if len(decision.GetRequiredObligations()) != 0 { + return fmt.Errorf("resource %s returned unexpected obligations", id) + } + } + return nil +} + +func exerciseAuthorizationCases(ctx context.Context, concurrency, seed int, requestTimeout, attributeRef string, table *godog.Table) (context.Context, error) { + if concurrency < 1 || seed < 0 { + return ctx, errors.New("concurrency must be positive and seed nonnegative") + } + timeout, err := time.ParseDuration(requestTimeout) + if err != nil || timeout <= 0 { + return ctx, fmt.Errorf("invalid duration %q", requestTimeout) + } + cases, err := parseAuthorizationScaleCases(table) + if err != nil { + return ctx, err + } + scenario := GetPlatformScenarioContext(ctx) + attribute, ok := scenario.GetObject(attributeRef).(*policy.Attribute) + if !ok || attribute.GetFqn() == "" { + return ctx, fmt.Errorf("missing attribute %q", attributeRef) + } + // Shuffle all cases rather than sampling, so no expected path is omitted. + random := rand.New(rand.NewPCG(uint64(seed), uint64(concurrency))) //nolint:gosec // reproducible test order, not security randomness + random.Shuffle(len(cases), func(i, j int) { cases[i], cases[j] = cases[j], cases[i] }) + var failures []error + for _, item := range cases { + chain, err := buildEntityChainFromIDs(scenario, item.entity) + if err != nil { + return ctx, err + } + request := &authz.GetDecisionMultiResourceRequest{ + EntityIdentifier: &authz.EntityIdentifier{Identifier: &authz.EntityIdentifier_EntityChain{EntityChain: chain}}, + Action: &policy.Action{Name: item.action}, + } + for i, value := range item.values { + request.Resources = append(request.Resources, &authz.Resource{ + EphemeralId: fmt.Sprintf("resource%d", i), + Resource: &authz.Resource_AttributeValues_{AttributeValues: &authz.Resource_AttributeValues{ + Fqns: []string{attribute.GetFqn() + "/value/" + value}, + }}, + }) + } + result, err := runAuthorizationScaleCase(ctx, scenario, item, request, concurrency, seed, timeout, random) + encoded, encodeErr := json.Marshal(result) + if encodeErr != nil { + return ctx, encodeErr + } + // A structured record survives both console and Go test JSON output formats. + fmt.Println(authorizationPerformanceMarker + string(encoded)) //nolint:forbidigo // structured CI record, independent of the configured log handler + if err != nil { + failures = append(failures, fmt.Errorf("case %s: %w", item.name, err)) + } + } + return ctx, errors.Join(failures...) +} + +func runAuthorizationScaleCase(ctx context.Context, scenario *PlatformScenarioContext, item authorizationScaleCase, request *authz.GetDecisionMultiResourceRequest, concurrency, seed int, timeout time.Duration, random *rand.Rand) (authorizationPerformanceResult, error) { + result := authorizationPerformanceResult{Case: item.name, Seed: seed, Concurrency: concurrency, Resources: len(item.values), Timeout: timeout} + // Bound request completion without treating the timeout as a latency baseline. + requestCtx, cancel := context.WithTimeout(ctx, timeout) + defer cancel() + requests := make([]*authz.GetDecisionMultiResourceRequest, concurrency) + for i := range requests { + requests[i] = proto.CloneOf(request) + resources := requests[i].GetResources() + random.Shuffle(len(resources), func(i, j int) { resources[i], resources[j] = resources[j], resources[i] }) + } + durations := make([]time.Duration, concurrency) + requestErrors := make([]error, concurrency) + start := make(chan struct{}) + var workers sync.WaitGroup + for i := range requests { + workers.Go(func() { + <-start + started := time.Now() + response, err := scenario.SDK.AuthorizationV2.GetDecisionMultiResource(requestCtx, requests[i]) + durations[i] = time.Since(started) + if err == nil { + err = validateScaleDecision(response, item.expected) + } + requestErrors[i] = err + }) + } + started := time.Now() + close(start) + workers.Wait() + result.Wall = time.Since(started) + for _, err := range requestErrors { + if err != nil { + result.Failures++ + } + } + slices.Sort(durations) + result.Median = durations[(concurrency-1)/2] + result.P95 = durations[(95*concurrency-1)/100] + result.Maximum = durations[concurrency-1] + var failure error + for _, err := range requestErrors { + if err != nil { + failure = err + break + } + } + return result, failure +} diff --git a/tests-bdd/cukes/steps_authorization_scale_test.go b/tests-bdd/cukes/steps_authorization_scale_test.go new file mode 100644 index 0000000000..c764705a1f --- /dev/null +++ b/tests-bdd/cukes/steps_authorization_scale_test.go @@ -0,0 +1,49 @@ +package cukes + +import ( + "testing" + + authz "github.com/opentdf/platform/protocol/go/authorization/v2" + "github.com/stretchr/testify/require" + "google.golang.org/protobuf/proto" +) + +func TestScaleDecisionValidatesEachResourceRegardlessOfOrder(t *testing.T) { + expected := map[string]authz.Decision{"resource0": authz.Decision_DECISION_PERMIT, "resource1": authz.Decision_DECISION_DENY} + valid := &authz.GetDecisionMultiResourceResponse{ResourceDecisions: []*authz.ResourceDecision{ + {EphemeralResourceId: "resource1", Decision: authz.Decision_DECISION_DENY}, + {EphemeralResourceId: "resource0", Decision: authz.Decision_DECISION_PERMIT}, + }} + require.NoError(t, validateScaleDecision(valid, expected)) + tests := []struct { + name string + change func(*authz.GetDecisionMultiResourceResponse) + }{ + {"incorrect deny", func(r *authz.GetDecisionMultiResourceResponse) { + r.GetResourceDecisions()[0].Decision = authz.Decision_DECISION_PERMIT + }}, + {"duplicate resource", func(r *authz.GetDecisionMultiResourceResponse) { + r.GetResourceDecisions()[0].EphemeralResourceId = "resource0" + }}, + {"unknown resource", func(r *authz.GetDecisionMultiResourceResponse) { + r.GetResourceDecisions()[0].EphemeralResourceId = "unknown" + }}, + {"missing resource", func(r *authz.GetDecisionMultiResourceResponse) { r.ResourceDecisions = r.GetResourceDecisions()[:1] }}, + {"unexpected obligations", func(r *authz.GetDecisionMultiResourceResponse) { + r.GetResourceDecisions()[1].RequiredObligations = []string{"unexpected"} + }}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + response := proto.CloneOf(valid) + tc.change(response) + require.Error(t, validateScaleDecision(response, expected)) + }) + } + require.Error(t, validateScaleDecision(nil, expected)) +} + +func TestScaleCasesRejectMissingTable(t *testing.T) { + _, err := parseAuthorizationScaleCases(nil) + require.Error(t, err) +} diff --git a/tests-bdd/cukes/steps_resourcemappings_scale.go b/tests-bdd/cukes/steps_resourcemappings_scale.go new file mode 100644 index 0000000000..4584075b88 --- /dev/null +++ b/tests-bdd/cukes/steps_resourcemappings_scale.go @@ -0,0 +1,34 @@ +package cukes + +import ( + "context" + "fmt" + + "github.com/cucumber/godog" + "github.com/opentdf/platform/protocol/go/policy/resourcemapping" +) + +func RegisterResourceMappingScaleSteps(ctx *godog.ScenarioContext) { + ctx.Step(`^I create (\d+) resource mappings for attribute "([^"]*)" in namespace "([^"]*)"$`, createScaleResourceMappings) +} + +func createScaleResourceMappings(ctx context.Context, count int, attributeRef, namespaceRef string) (context.Context, error) { + scenario, attribute, namespace, err := scaleMappingInputs(ctx, count, attributeRef, namespaceRef) + if err != nil { + return ctx, err + } + err = createScaleMappings(ctx, count, func(ctx context.Context, index int) error { + response, err := scenario.SDK.ResourceMapping.CreateResourceMapping(ctx, &resourcemapping.CreateResourceMappingRequest{ + AttributeValueId: attribute.GetValues()[index].GetId(), NamespaceId: namespace, + Terms: []string{fmt.Sprintf("resource-%04d", index)}, + }) + if err != nil { + return fmt.Errorf("create resource mapping %d: %w", index, err) + } + if response.GetResourceMapping().GetId() == "" { + return fmt.Errorf("resource mapping %d returned no identity", index) + } + return nil + }) + return ctx, err +} diff --git a/tests-bdd/cukes/steps_subjectmappings.go b/tests-bdd/cukes/steps_subjectmappings.go index 76d7c7a108..56066c6e84 100644 --- a/tests-bdd/cukes/steps_subjectmappings.go +++ b/tests-bdd/cukes/steps_subjectmappings.go @@ -29,7 +29,7 @@ func (s *SubjectMappingsStepDefinitions) iSendARequestToCreateSubjectMapping(ctx cellIndexMap[ci] = c.Value } else { switch cellIndexMap[ci] { - case "namespace_id": + case namespaceIDKey: nsID, ok := scenarioContext.GetObject(strings.TrimSpace(c.Value)).(string) if !ok { return ctx, fmt.Errorf("unable to get namespace id for %s", c.Value) @@ -222,6 +222,7 @@ func (s *SubjectMappingsStepDefinitions) iSendARequestToCreateSubjectMappingForE func RegisterSubjectMappingsStepsDefinitions(ctx *godog.ScenarioContext) { subjectMappingStepDefinitions := &SubjectMappingsStepDefinitions{} + ctx.Step(`^I create (\d+) subject mappings for attribute "([^"]*)" using condition set "([^"]*)" with action "([^"]*)"$`, subjectMappingStepDefinitions.createScaleSubjectMappings) ctx.Step(`a condition group referenced as "([^"]*)" with an "([^"]*)" operator with conditions:$`, subjectMappingStepDefinitions.aConditionGroup) ctx.Step(`^a subject set referenced as "([^"]*)" containing the condition groups "([^"]*)"$`, subjectMappingStepDefinitions.aSubjectSet) ctx.Step(`^I send a request to create a subject condition set referenced as "([^"]*)" containing subject sets "([^"]*)"$`, subjectMappingStepDefinitions.iSendARequestToCreateSubjectConditionSet) diff --git a/tests-bdd/cukes/steps_subjectmappings_scale.go b/tests-bdd/cukes/steps_subjectmappings_scale.go new file mode 100644 index 0000000000..3a6ba0705e --- /dev/null +++ b/tests-bdd/cukes/steps_subjectmappings_scale.go @@ -0,0 +1,35 @@ +package cukes + +import ( + "context" + "fmt" + + "github.com/opentdf/platform/protocol/go/policy" + "github.com/opentdf/platform/protocol/go/policy/subjectmapping" +) + +func (s *SubjectMappingsStepDefinitions) createScaleSubjectMappings(ctx context.Context, count int, attributeRef, conditionSetRef, action string) (context.Context, error) { + scenario := GetPlatformScenarioContext(ctx) + attribute, ok := scenario.GetObject(attributeRef).(*policy.Attribute) + if !ok || count <= 0 || count > len(attribute.GetValues()) { + return ctx, fmt.Errorf("attribute %q must contain at least %d values", attributeRef, count) + } + conditionSet, ok := scenario.GetObject(conditionSetRef).(*policy.SubjectConditionSet) + if !ok { + return ctx, fmt.Errorf("missing condition set %q", conditionSetRef) + } + err := createScaleMappings(ctx, count, func(ctx context.Context, index int) error { + response, err := scenario.SDK.SubjectMapping.CreateSubjectMapping(ctx, &subjectmapping.CreateSubjectMappingRequest{ + AttributeValueId: attribute.GetValues()[index].GetId(), ExistingSubjectConditionSetId: conditionSet.GetId(), + Actions: GetActionsFromValues(&action, nil), + }) + if err != nil { + return fmt.Errorf("create subject mapping %d: %w", index, err) + } + if response.GetSubjectMapping().GetId() == "" { + return fmt.Errorf("subject mapping %d returned no identity", index) + } + return nil + }) + return ctx, err +} diff --git a/tests-bdd/features/authorization-v2-subject-mapping-performance.feature b/tests-bdd/features/authorization-v2-subject-mapping-performance.feature new file mode 100644 index 0000000000..23e15c9be0 --- /dev/null +++ b/tests-bdd/features/authorization-v2-subject-mapping-performance.feature @@ -0,0 +1,59 @@ +@authorization @authz-v2 @performance @scale +Feature: v2 multi-resource decisions at large policy scale + GetDecisionMultiResource must remain responsive when the policy database contains the + subject-mapping and resource-mapping cardinality from the reported regression. Fixture setup is + not timed. The measured operation is a synchronized group of requests to the public v2 + authorization endpoint. Every case runs at every concurrency level. A fixed seed + shuffles case and resource order reproducibly. The extra value has no subject mapping. + Subject mappings use the default unnamespaced policy path; attribute and resource mappings + retain their namespace. This avoids repeatedly validating the entire attribute during setup. + Latency is reported without a performance gate until a baseline is established. + Incorrect decisions, request errors, and request timeouts fail the scenario. + + Scenario Outline: Varied multi-resource decisions at concurrency + Given a user exists with username "scale-user" and email "scale-user@example.com" and the following attributes: + | name | value | + | department | ["engineering"] | + And a user exists with username "other-user" and email "other-user@example.com" and the following attributes: + | name | value | + | department | ["sales"] | + And an empty local platform + And I submit a request to create a namespace with name "scale.example" and reference id "scale_ns" + And I send a request to create an attribute referenced as "scale_attr" in namespace "scale_ns" named "access-level" with rule "anyOf" and 6012 generated values in batches of 25 + Then the response should be successful + And a condition group referenced as "scale_condition" with an "or" operator with conditions: + | selector_value | operator | values | + | .attributes.department[] | in | engineering | + And a subject set referenced as "scale_subject_set" containing the condition groups "scale_condition" + And I send a request to create a subject condition set referenced as "scale_condition_set" containing subject sets "scale_subject_set" + Then the response should be successful + And I create 6011 subject mappings for attribute "scale_attr" using condition set "scale_condition_set" with action "read" + And I create 6000 resource mappings for attribute "scale_attr" in namespace "scale_ns" + And there is a "user_name" subject entity with value "scale-user" and referenced as "scale-user" + And there is a "user_name" subject entity with value "other-user" and referenced as "other-user" + When I exercise the authorization cases with concurrent requests each, seed 4625, and request timeout "30s" for attribute "scale_attr": + | case | entity | action | values | expected | + | allowed_read | scale-user | read | v0000,v3005,v6010 | PERMIT,PERMIT,PERMIT | + | denied_action | scale-user | write | v0000,v3005,v6010 | DENY,DENY,DENY | + | denied_user | other-user | read | v0000,v3005,v6010 | DENY,DENY,DENY | + | mixed_values | scale-user | read | v0000,v6011,v6010 | PERMIT,DENY,PERMIT | + + @concurrency-1 + Examples: One request + | concurrency | + | 1 | + + @concurrency-10 + Examples: Ten concurrent requests + | concurrency | + | 10 | + + @concurrency-25 + Examples: Twenty-five concurrent requests + | concurrency | + | 25 | + + @concurrency-50 + Examples: Fifty concurrent requests + | concurrency | + | 50 | diff --git a/tests-bdd/platform_test.go b/tests-bdd/platform_test.go index 9df363fd23..bd0b9cb946 100644 --- a/tests-bdd/platform_test.go +++ b/tests-bdd/platform_test.go @@ -110,6 +110,7 @@ func runTests() int { cukes.RegisterSmokeStepDefinitions(ctx, platformCukesContext) cukes.RegisterAuthorizationStepDefinitions(ctx) cukes.RegisterSubjectMappingsStepsDefinitions(ctx) + cukes.RegisterResourceMappingScaleSteps(ctx) cukes.RegisterDynamicValueMappingsStepDefinitions(ctx) cukes.RegisterDirectEntitlementsStepDefinitions(ctx) cukes.RegisterRegisteredResourcesStepDefinitions(ctx)