diff --git a/.agents/rules/testing.md b/.agents/rules/testing.md index 1ad6ffc6704..d09b44674d2 100644 --- a/.agents/rules/testing.md +++ b/.agents/rules/testing.md @@ -185,6 +185,7 @@ Available on `PATH` during test execution (from `acceptance/bin/`): - `diff.py DIR1 DIR2` or `diff.py FILE1 FILE2`: recursive diff with test replacements applied. - `print_state.py [-t TARGET] [--backup]`: print deployment state (terraform or direct). - `edit_resource.py TYPE ID < script.py`: fetch resource by ID, execute Python on it (resource in `r`), then update it. TYPE is `jobs` or `pipelines`. +- `verify_no_drift.py PLAN.json`: assert every action in a JSON plan is `skip`. - `gron.py`: flatten JSON into greppable discrete assignments (simpler than `jq` for searching JSON). - `jq` is also available for JSON processing. diff --git a/Taskfile.yml b/Taskfile.yml index 3d58060c1e2..7c499faf4e0 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -8,6 +8,9 @@ vars: # .github/workflows/check.yml and python/Taskfile.yml. RUFF: uvx ruff@0.15.17 TEST_PACKAGES: ./acceptance/internal ./libs/... ./internal/... ./cmd/... ./bundle/... ./experimental/ssh/... . + # Field-support catalog. Covered by TEST_PACKAGES for local/CI runs via ./bundle/..., + # but named separately because the cloud tasks below list their packages explicitly. + FIELDS_PACKAGE: ./bundle/direct/autotest ACCEPTANCE_TEST_FILTER: "" # Single brace-expansion glob covering every //go:embed target in the repo, # computed by grepping `//go:embed` directives. Evaluated lazily by Task so @@ -590,6 +593,31 @@ tasks: cmds: - "go test ./acceptance -run '^TestAccept$' -update -timeout=${LOCAL_TIMEOUT:-60m}" + test-update-fields: + desc: Update the resource field-support catalog (local) + cmds: + - "go test {{.FIELDS_PACKAGE}} -run '^TestFields$' -update -timeout=${LOCAL_TIMEOUT:-60m}" + + # The catalog runs against the fake server as part of `./task test`, and needs no entry of its + # own for that. These two are the real-workspace runs: they need CLOUD_ENV and credentials, and + # hold the same committed reports as the local run, so a divergence is the fake server or the + # engine differing from the backend. + autotest-cloud: + desc: Run the resource field-support catalog against a real workspace (every field; nightly) + cmds: + # -sample 0 says every field: without it a cloud run samples unless the commit title carries + # AUTOTEST_ALL, which is what keeps a PR's integration run affordable. + - "go test {{.FIELDS_PACKAGE}} -run '^TestFields$' -count=1 -sample 0 -timeout=${CLOUD_TIMEOUT:-300m}" + + autotest-cloud-pr: + desc: Run a two-field sample of the field-support catalog against a real workspace (PRs) + cmds: + # Which fields are sampled comes from HEAD, so each commit covers different ground. This + # checks that every resource type still deploys and that the sampled fields' findings read + # as recorded; a change between two passing verdicts only shows in the full run above. + - "go test {{.FIELDS_PACKAGE}} -run '^TestFields$' -count=1 -sample 2 -timeout=${CLOUD_TIMEOUT:-90m}" + + test-update-templates: desc: Update acceptance test template output sources: *ACC_SOURCES_UPDATE @@ -601,6 +629,7 @@ tasks: desc: Update all acceptance test outputs cmds: - task: test-update + - task: test-update-fields # Follows upstream HEAD, so its result changes over time: keep it out of # `generate-check`, which requires byte-for-byte reproducible output. @@ -755,7 +784,7 @@ tasks: --format github-actions \ --rerun-fails \ --jsonfile output.json \ - --packages "./acceptance ./integration/..." \ + --packages "./acceptance ./integration/... {{.FIELDS_PACKAGE}}" \ -- -parallel 4 -timeout=12h integration-short: @@ -769,7 +798,7 @@ tasks: --format github-actions \ --rerun-fails \ --jsonfile output.json \ - --packages "./acceptance ./integration/..." \ + --packages "./acceptance ./integration/... {{.FIELDS_PACKAGE}}" \ -- -parallel 4 -timeout=12h -short integration-short-skiplocal: @@ -783,7 +812,7 @@ tasks: --format github-actions \ --rerun-fails \ --jsonfile output.json \ - --packages "./acceptance ./integration/..." \ + --packages "./acceptance ./integration/... {{.FIELDS_PACKAGE}}" \ -- -parallel 4 -timeout=2h -short dbr-integration: diff --git a/acceptance/bundle/deployment/bind/pipelines/update/out.bind-fail.direct.txt b/acceptance/bundle/deployment/bind/pipelines/update/out.bind-fail.direct.txt index a5e4ca80074..607db25e88c 100644 --- a/acceptance/bundle/deployment/bind/pipelines/update/out.bind-fail.direct.txt +++ b/acceptance/bundle/deployment/bind/pipelines/update/out.bind-fail.direct.txt @@ -3,7 +3,6 @@ Plan: update resources.pipelines.foo Changes detected: - ~ catalog: null -> "main" ~ channel: null -> "CURRENT" ~ deployment: null -> {"kind":"BUNDLE","metadata_file_path":"/Workspace/Users/[USERNAME]/.bundle/test-pipeline-recreate/default/state/metadata.json"} ~ edition: null -> "ADVANCED" diff --git a/acceptance/bundle/deployment/bind/pipelines/update/out.bind-fail.terraform.txt b/acceptance/bundle/deployment/bind/pipelines/update/out.bind-fail.terraform.txt index deacd3b1a20..7e846f456ee 100644 --- a/acceptance/bundle/deployment/bind/pipelines/update/out.bind-fail.terraform.txt +++ b/acceptance/bundle/deployment/bind/pipelines/update/out.bind-fail.terraform.txt @@ -10,7 +10,6 @@ Terraform will perform the following actions: # databricks_pipeline.foo will be updated in-place ~ resource "databricks_pipeline" "foo" { - + catalog = "main" + channel = "CURRENT" + edition = "ADVANCED" id = "[NEW_PIPELINE_ID]" diff --git a/acceptance/bundle/deployment/bind/pipelines/update/out.deploy.requests.terraform.json b/acceptance/bundle/deployment/bind/pipelines/update/out.deploy.requests.terraform.json index 3c8d1946b83..cb62d4d01c9 100644 --- a/acceptance/bundle/deployment/bind/pipelines/update/out.deploy.requests.terraform.json +++ b/acceptance/bundle/deployment/bind/pipelines/update/out.deploy.requests.terraform.json @@ -31,7 +31,6 @@ } } ], - "name": "test-pipeline", - "storage": "dbfs:/pipelines/[NEW_PIPELINE_ID]" + "name": "test-pipeline" } } diff --git a/acceptance/bundle/deployment/bind/pipelines/update/pipeline.json b/acceptance/bundle/deployment/bind/pipelines/update/pipeline.json index 0254d7b549f..fca73e374c0 100644 --- a/acceptance/bundle/deployment/bind/pipelines/update/pipeline.json +++ b/acceptance/bundle/deployment/bind/pipelines/update/pipeline.json @@ -1,5 +1,6 @@ { "name": "lakeflow-pipeline", + "catalog": "main", "libraries": [ { "glob": { diff --git a/acceptance/bundle/resources/model_serving_endpoints/update/email-notifications/out.get_email_notifications.terraform.json b/acceptance/bundle/resources/model_serving_endpoints/update/email-notifications/out.get_email_notifications.terraform.json index 76b5ef08c2c..19765bd501b 100644 --- a/acceptance/bundle/resources/model_serving_endpoints/update/email-notifications/out.get_email_notifications.terraform.json +++ b/acceptance/bundle/resources/model_serving_endpoints/update/email-notifications/out.get_email_notifications.terraform.json @@ -1,5 +1 @@ -{ - "on_update_success": [ - "user1@example.com" - ] -} +null diff --git a/acceptance/bundle/resources/model_serving_endpoints/update/email-notifications/out.plan.direct.json b/acceptance/bundle/resources/model_serving_endpoints/update/email-notifications/out.plan.direct.json index cf8c8b3f0ca..4529fa116f0 100644 --- a/acceptance/bundle/resources/model_serving_endpoints/update/email-notifications/out.plan.direct.json +++ b/acceptance/bundle/resources/model_serving_endpoints/update/email-notifications/out.plan.direct.json @@ -60,11 +60,6 @@ "creation_timestamp": [UNIX_TIME_MILLIS][0], "creator": "[USERNAME]", "description": "", - "email_notifications": { - "on_update_success": [ - "user1@example.com" - ] - }, "id": "[UUID]", "last_updated_timestamp": [UNIX_TIME_MILLIS][0], "name": "[ENDPOINT_ID]", @@ -100,11 +95,6 @@ ] } }, - "email_notifications": { - "on_update_success": [ - "user1@example.com" - ] - }, "name": "[ENDPOINT_ID]" }, "changes": { @@ -126,11 +116,23 @@ "reason": "empty", "remote": "" }, + "email_notifications": { + "action": "update", + "old": { + "on_update_success": [ + "user1@example.com" + ] + }, + "new": { + "on_update_success": [ + "user2@example.com" + ] + } + }, "email_notifications.on_update_success[0]": { "action": "update", "old": "user1@example.com", - "new": "user2@example.com", - "remote": "user1@example.com" + "new": "user2@example.com" }, "route_optimized": { "action": "skip", diff --git a/acceptance/bundle/resources/model_serving_endpoints/update/email-notifications/output.txt b/acceptance/bundle/resources/model_serving_endpoints/update/email-notifications/output.txt index 077c27ab917..fcb206aad5a 100644 --- a/acceptance/bundle/resources/model_serving_endpoints/update/email-notifications/output.txt +++ b/acceptance/bundle/resources/model_serving_endpoints/update/email-notifications/output.txt @@ -6,11 +6,7 @@ Files: 5 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged >>> [CLI] serving-endpoints get [ENDPOINT_ID] -{ - "on_update_success": [ - "user1@example.com" - ] -} +null >>> update_file.py databricks.yml user1@example.com user2@example.com diff --git a/acceptance/bundle/resources/models/readplan-permissions/databricks.yml b/acceptance/bundle/resources/models/readplan-permissions/databricks.yml index 1d4e2f53e6a..9e48a0543da 100644 --- a/acceptance/bundle/resources/models/readplan-permissions/databricks.yml +++ b/acceptance/bundle/resources/models/readplan-permissions/databricks.yml @@ -5,7 +5,7 @@ resources: models: mymodel: name: test-model - description: initial # TO_REMOVE + description: initial permissions: - level: CAN_READ user_name: viewer@example.com diff --git a/acceptance/bundle/resources/models/readplan-permissions/output.txt b/acceptance/bundle/resources/models/readplan-permissions/output.txt index e65f9b2a560..853bd8924c6 100644 --- a/acceptance/bundle/resources/models/readplan-permissions/output.txt +++ b/acceptance/bundle/resources/models/readplan-permissions/output.txt @@ -29,7 +29,7 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged } } -=== Remove manager permission and description, deploy from planUploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +=== Remove manager permission, change description, deploy from planUploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... Updated models.mymodel Updated models.mymodel.permissions Files: 4 uploaded, 0 deleted diff --git a/acceptance/bundle/resources/models/readplan-permissions/script b/acceptance/bundle/resources/models/readplan-permissions/script index f8be26409e3..e0f44832600 100644 --- a/acceptance/bundle/resources/models/readplan-permissions/script +++ b/acceptance/bundle/resources/models/readplan-permissions/script @@ -15,8 +15,11 @@ trace $CLI bundle deploy register_model_id trace print_requests.py //permissions/registered-models -title "Remove manager permission and description, deploy from plan" +title "Remove manager permission, change description, deploy from plan" grep -v TO_REMOVE databricks.yml > updated.yml && mv updated.yml databricks.yml +# Changed rather than removed: MLflow rejects an empty description, so clearing the field +# fails the update. See libs/testserver/models.go. +update_file.py databricks.yml "description: initial" "description: changed" $CLI bundle plan -o json > tmp.plan.json $CLI bundle deploy $(readplanarg tmp.plan.json) register_model_id diff --git a/acceptance/cmd/workspace/create-scope/output.txt b/acceptance/cmd/workspace/create-scope/output.txt index 4487d2945cc..b950aa9d08d 100644 --- a/acceptance/cmd/workspace/create-scope/output.txt +++ b/acceptance/cmd/workspace/create-scope/output.txt @@ -8,18 +8,16 @@ Error: expected JSON object, received sequence >>> [CLI] secrets create-scope --scope-backend-type AZURE_KEYVAULT --json { "scope": "xxxxxxxx-xxx", "scope_backend_type": "AZURE_KEYVAULT", - "keyvault_metadata": { + "backend_azure_keyvault": { "dns_name": "test", "resource_id": "123456" } } -Warning: unknown field: keyvault_metadata - in (inline):4:4 - >>> [CLI] secrets delete-scope xxxxxxxx-xxx ->>> [CLI] secrets create-scope --scope-backend-type AZURE_KEYVAULT --json { +>>> musterr [CLI] secrets create-scope --scope-backend-type AZURE_KEYVAULT --json { "scope": "xxxxxxxx-xxx", "scope_backend_type": "AZURE_KEYVAULT" } +Error: Scope with Azure KeyVault must have AzureKeyVaultSecretScopeMetadata defined! diff --git a/acceptance/cmd/workspace/create-scope/script b/acceptance/cmd/workspace/create-scope/script index 240af51bc41..96480d51b18 100644 --- a/acceptance/cmd/workspace/create-scope/script +++ b/acceptance/cmd/workspace/create-scope/script @@ -5,7 +5,7 @@ trace musterr $CLI secrets create-scope --scope-backend-type AZURE_KEYVAULT --js trace $CLI secrets create-scope --scope-backend-type AZURE_KEYVAULT --json '{ "scope": "xxxxxxxx-xxx", "scope_backend_type": "AZURE_KEYVAULT", - "keyvault_metadata": { + "backend_azure_keyvault": { "dns_name": "test", "resource_id": "123456" } @@ -13,7 +13,9 @@ trace $CLI secrets create-scope --scope-backend-type AZURE_KEYVAULT --json '{ trace $CLI secrets delete-scope xxxxxxxx-xxx -trace $CLI secrets create-scope --scope-backend-type AZURE_KEYVAULT --json '{ +# The API requires the metadata that names the vault: "Scope with Azure KeyVault must have +# AzureKeyVaultSecretScopeMetadata defined!" +trace musterr $CLI secrets create-scope --scope-backend-type AZURE_KEYVAULT --json '{ "scope": "xxxxxxxx-xxx", "scope_backend_type": "AZURE_KEYVAULT" }' diff --git a/bundle/direct/autotest/.gitignore b/bundle/direct/autotest/.gitignore new file mode 100644 index 00000000000..5ddbb82fdf4 --- /dev/null +++ b/bundle/direct/autotest/.gitignore @@ -0,0 +1,3 @@ +# What a run produces for reading rather than for review: the full report, its cloud counterpart, +# and the previous copy of each. output/ holds the committed goldens and nothing else. +logs/ diff --git a/bundle/direct/autotest/README.md b/bundle/direct/autotest/README.md new file mode 100644 index 00000000000..53f717232d6 --- /dev/null +++ b/bundle/direct/autotest/README.md @@ -0,0 +1,273 @@ +# Resource field-support catalog + +This suite answers one question for every field a user can set on a bundle resource: +**if I change this field, does the direct engine notice, apply it, and converge?** + +The answer for every field lands in `output/.txt`. Those files are +the deliverable — a map of where field support is solid and where it is not. A bad +outcome does not fail the test; a *changed* outcome does, because the goldens are +committed. That makes the suite a regression detector for things like an SDK bump +changing a field's type. + +## What it does + +For each resource type with a value library in `testdata/fields`, the suite deploys that +library's `base` as a one-resource bundle, then walks the resource's input struct the way +`cmd/bundle/debug` refschema does. For each field it moves the field through every +ordered pair of a small value set — with `absent` in the set, so adding and removing a +field are just the pairs with `absent` on one side — and records three things per move: +did the plan propose anything, did the apply succeed, and was the next plan clean. + +The pairs are walked as one chain rather than staged one at a time: the values form a +complete digraph, so a single Eulerian circuit covers every ordered pair exactly once and +each move starts where the last one ended. That halves the deploys, and +`TestTransitionsCoverEveryPairInOneChain` is what guarantees nothing is missed. + +A field whose type has no generic value at all — an `any` like `serialized_dashboard` — and a +required field the library gives a single value are reported as not covered rather than +quietly skipped: there is no second value to move to and no absent to move from. + +**Slices and maps** are covered two ways. The container is a field in its own right, whose +values are the config's own and that value with one entry dropped — or, for a list of +scalars the config leaves empty, one and two elements of the element's own type — so with `absent` +in the set, one field covers adding and removing an entry as well as adding and removing +the whole container, all with data the backend has already accepted. Separately, a pattern +like `tasks[*].description` is expanded against the deployed config to the indices that +exist, so fields inside an element are tested like any other. A pattern with nothing +behind it in the config is reported as not covered rather than silently passing. + +Everything runs in-process against the direct engine's own `CalculatePlan` and +`Apply`. There is no CLI subprocess and no bundle-file upload: at thousands of +permutations, a `bundle deploy` each would be dominated by sync. + +Edits are made to the **typed** resource — `libs/structs/structaccess` over +`*resources.Schema` and friends — and synced into the dynamic tree the planner reads the +way any mutator does. That is also what makes "absent" precise: a field is absent when it +holds the zero value and is not in `ForceSendFields`, which is exactly the distinction the +API sees. + +## Verdicts + +| verdict | meaning | +| --- | --- | +| `OK` | planned, applied, next plan clean | +| `OK_RECREATE` | same, but the engine replaced the resource | +| `SUPPRESSED` | the planner diffed the field and dropped the change; detail is the engine's own reason | +| `NOT_OBSERVABLE` | the two values are identical in the state sent to the API, so there is nothing to see — an unset bool and an explicit `false` are the usual case, as is a field the engine consumes before planning (an alert's `file_path`) | +| `NO_PLAN` | the field diff exists but no action was planned | +| `POST_DEPLOY_DRIFT` | applied, yet the plan taken straight afterwards still wants the same change: deploying never converges | +| `POST_DEPLOY_DRIFT_CHILD` | the field converged but another node of the resource did not; a guard that should stay empty | +| `BACKEND_ERROR` | the API rejected the value — usually the value library needs a valid value for this field | +| `DEPLOY_ERROR` | apply failed for a non-API reason | +| `TIMEOUT` | the operation did not finish inside the per-operation deadline | +| `PLAN_ERROR` | planning failed | +| `UNSETTABLE` | the value could not be written into the config at all | +| `BASE_ERROR` | the transition's starting point would not deploy, so nothing was observed | +| `START_NOT_REACHED` | the starting value deployed without error but the field did not end up holding it, so the move under test could not be set up — a field the API refuses to clear cannot start from `absent`. Retried once on a fresh resource before being recorded | +| `UPDATE_IGNORED` | the apply succeeded and the engine sent the write, but the field's remote value is unchanged on two consecutive reads: the backend accepted the request and ignored this field | +| `STALE_READ` | the write did land, but the read straight after the apply did not show it and the next one did — not a support gap, yet a user planning right after a deploy is shown a change that does not exist | +| `BASELINE_DRIFT` | the field drifts with no config change at all, measured once per resource so it is not blamed on every field tested afterwards | +| `COLLATERAL_DRIFT` | the field under test converged, but updating it left some *other* field drifting; the detail names that field, which is where the fix belongs | +| `OK_INERT` | the resource declares that it ignores local changes to this field, and it does — the claim verified rather than assumed | +| `INERT_NOT_HONOURED` | the resource declares the field inert and the change was applied anyway | +| `SKIPPED` | left out by the value library, with a reason | + +`START_NOT_REACHED` and `UPDATE_IGNORED` are two views of the largest class this suite finds: +a field the backend will not clear. The engine sends the write, the backend keeps what it had, +and nothing can then start from `absent`. `SUPPRESSED` with a non-benign reason and +`COLLATERAL_DRIFT` are the next most interesting. `NOT_OBSERVABLE` is not a gap; it is the wire +format telling the truth. + +Two files per resource type. `output/.txt` is committed: one line per finding, then +the count of every verdict — including the passing ones, which no line names. Those counts +are what makes a change in *passing* behaviour visible, since a field that starts being +recreated instead of updated moves one `OK` to `OK_RECREATE` and nothing else would show it. + +`logs/.full.txt` holds every row, the not-covered list, and the evidence indented +under each finding — the post-deploy plan for drift, the whole API error for a rejection. It +is gitignored, because it is full of generated ids and moves whenever any row moves. + +The suite reads `resources.yml` for two things only. A field the resource declares an +`output_only` backend output is skipped rather than tested, since a user cannot meaningfully +set one at all. And a field the resource declares it ignores local changes to is tested +anyway, so the claim is checked: it should come back suppressed with exactly that reason, +which is `OK_INERT`, and anything else is `INERT_NOT_HONOURED`. + +Nothing else is consulted. `resources.yml` is the engine's answer to most of these cases, so +reading it further would just restate the implementation; the `SUPPRESSED` reason string is +the engine explaining itself, which is different. + +## Value library + +`testdata/fields/.yml` is the whole fixture for a type: the resource to deploy, +and the values to move its fields through. There is one per type, and it is the only place a +run reads a resource definition from. + +`fields` supplies values the generic per-kind defaults cannot guess — enums, ids, anything the +backend constrains. A field naming another object needs a name that exists: a real workspace +rejects the generic `x` outright, so the field would report nothing but `BACKEND_ERROR`. + +```yaml +# The resource, rendered into a one-resource databricks.yml and deployed before anything is +# measured. What it declares is what can be tested: a block absent here has no entry for its +# fields to live in, and some blocks only validate as a whole -- a job's git_source needs a +# provider, a url and exactly one ref -- so they cannot be built up one field at a time. +base: + name: test-job-$UNIQUE_NAME + git_source: + git_provider: gitHub + git_url: https://github.com/databricks/cli.test + git_commit: abc123 + +skip: + git_source.git_branch: mutually exclusive with the seeded git_commit + +fields: + git_source.git_commit: [abc123, def456] +``` + +A field with no entry gets two values derived from its type: the first two members of an SDK +enum (excluding the `*_UNSPECIFIED` sentinel, which means "unset" and is normalized away), or +two of its Go kind. That is enough to see a value-to-value move on top of add and remove. + +Values that a real workspace constrains have to be written down even when the Go type does not +say so — a pipeline's `channel` and `edition` are plain strings the backend validates, and a +cluster size is a t-shirt size. The AWS run is what finds these: locally the fake server takes +anything. + +An `OK` that has only ever been seen against the fake server is worth less than one a +workspace has agreed to, and most of all for a field whose value must name something real. The +fake server takes any string, so for such a field `OK` says nothing until it has been checked +against a workspace — see above. That check is impossible for a `local_only` type, whose report +is a record of what the fake server does: `external_locations` has 62 such rows, all in blocks +naming cloud storage the suite does not provision. + +A type that cannot run against a real workspace at all declares `local_only: ` and +is skipped on cloud — an external location needs a storage credential with cloud IAM behind it, +and an instance pool cannot be deleted again afterwards. + +A service that exists on some clouds and not others is a different case, and declares where it +does: `clouds: [aws]` runs the type on an aws workspace and skips it elsewhere, so a cloud-specific +service is still confirmed somewhere rather than nowhere. This mirrors `CloudEnvs.gcp = false` in +`acceptance/bundle/resources/postgres_*/test.toml`; the blanket exclusions in +`acceptance/bundle/invariant/test.toml` drop those configs from every cloud, including the one +that has the service. + +A `skip` key may be a pattern (`aliases[*].id`), matched the way the planner matches its own +field rules, and naming a block skips everything beneath it. + +A field that names the resource gets the run's own unique suffix appended to its values, so +two runs against one workspace never ask for the same name — the second would get "already +exists", which says nothing about the engine. Such a field is recognised by `base` templating +its value with `$UNIQUE_NAME`. The suffix is a placeholder until the value is +written, since a rebuilt resource has a new one while the old is still alive, and it is +redacted in the report so the golden is the same on every run. + +`base` is the whole resource, and is expanded with the `$VARS` an acceptance config would get — +`$CURRENT_USER_NAME`, `$NODE_TYPE_ID`, `$TEST_DEFAULT_WAREHOUSE_ID` and the rest — so a value can +name the workspace's own user rather than a placeholder only the fake server knows. +`$UNIQUE_NAME` is expanded later, per deploy, because a rebuild gets a new one. + +Because `base` becomes the bundle, it can only hold what a bundle can: a field the config format +rejects is rejected here too. That is deliberate — an alert built from a `.dbalert.json` takes +everything but `warehouse_id`, `display_name` and `file_path` from that file, and seeding +`evaluation` in the bundle is a shape the CLI refuses. + +`skip` is for a field no single-field edit can exercise — one that only validates as +part of a set (a job's `git_source`), or whose change is correct but ruinously slow (an +app rename waits out an asynchronous delete). Each entry needs a reason, and shows up in +the report as `SKIPPED` so it is not mistaken for coverage. + +## Running it + +```bash +go test ./bundle/direct/autotest # against the testserver +go test ./bundle/direct/autotest -run 'TestFields/schemas' -v +./task test-update-fields # regenerate the goldens +``` + +The local run is part of `./task test`, so it needs no task of its own. The real-workspace runs +do, since they need `CLOUD_ENV` and credentials: + +```bash +./task autotest-cloud # every field; nightly +./task autotest-cloud-pr # -sample 2; PRs +``` + +A cloud run samples by default -- two fields per type -- because the full sweep takes hours and the +autotest package is in the integration tasks, which a PR runs. Putting `AUTOTEST_ALL` in a commit title +makes that PR's integration run drive every field instead. Locally nothing is sampled: a full run is 11 +seconds, so `./task test` always compares the goldens whole. + +`-sample N` overrides both, and tests N of each type's fields instead of all of them, picked from HEAD so that +successive commits cover different ground and one run's picks follow from its SHA. The reports +stay the same committed goldens: a sampled run is held to the rows belonging to the fields it +picked, plus any row the harness records against itself, which is what catches a type that no +longer deploys at all. The summary counts every field and has no meaningful subset, so it is +not compared -- a change between two *passing* verdicts shows up only in the full run. +`-sample` refuses to run with `-update`, which would truncate the goldens to the sample. + +A value whose own label would be unreadable gets a short alias -- `v1`, `v2` -- which the legend at +the end of the full report maps back, so a subtest is `absent_to_v1` rather than a truncated email +address with a digest on the end. An index is written `tasks_0`, not `tasks[0]`: brackets are a +character class to the `-run` regex, so a filter copied from the output would mean something else. + +One transition is addressable on its own, and the subtest names carry no shell +metacharacters so no quoting is needed. `-v` prints the post-deploy plan behind any +problem verdict: + +```bash +go test ./bundle/direct/autotest -run TestFields/pipelines/.*/dry_run/absent_to_true -v +``` + +### Checking one field against a real workspace + +A whole resource type on cloud is slow (jobs is half an hour) and the interesting question is +usually about a handful of fields. Every field is a subtest, so a few can be driven on their +own: + + CLOUD_ENV=aws go test ./bundle/direct/autotest \ + -run 'TestFields/^apps$/[^/]*/^(budget_policy_id|usage_policy_id)$' + +The golden comparison fails on a partial run, which is expected: read the rows in +`logs/..full.txt` instead. This is how to check a field the local report calls +`OK` when its value looks like it must name something real, a `*_id` or an ARN or a path. The +fake server takes any string, so `OK` there means nothing until a workspace has agreed: both +`budget_policy_id` and `usage_policy_id` on apps read `OK` locally and are rejected outright on +a real workspace, which names an account-level policy neither has. + +On cloud (`CLOUD_ENV` set) the same reports are compared against the same committed +goldens: a divergence means the fake server in `libs/testserver` does not model the API +faithfully, which is worth failing over. The full report goes to +`logs/..full.txt` so a cloud run can be read next to a local one. + +## Out of scope for now + +- remote drift — a change made outside the bundle; this suite only edits config +- `permissions` and `grants`, which are stripped from every fixture before planning: they + are separate plan nodes describing an ACL, and leaving them in made every recreate + report a drifted child against whichever field triggered it. They are supported resource + types in their own right, and `output/configs.txt` leaves them out for the same reason. +- resource types with no value library yet, listed in `output/configs.txt` +- fields under a slice or map, listed at the end of each report + +Two divergences from a real workspace are known and left as findings rather than modelled, +because each needs a behavioural change the acceptance suite currently asserts otherwise: + +- A SQL warehouse reads back `STARTING`, not `RUNNING`, right after create, so a config asking + for `started: false` is already satisfied there. Modelling it means the fake server also has + to move the warehouse to `RUNNING` on a later read, which the engine's waiter depends on. +- A model serving endpoint's `email_notifications` do not take effect from a create, so the field + drifts from the moment the resource exists — recorded as `BASELINE_DRIFT`, with the plan behind + it in the full report. + +A third, a Genie space's title, is fixed: the fake server named an untitled space `""` where the +backend names it `New Agent`, and the missing default was masking a bug in this suite. That is the +shape these entries usually have, which is why they are worth chasing rather than tolerating. + +Two types cannot be driven against the workspace this was verified on, for reasons that are its +capacity rather than the engine's behaviour: a cluster never reaches `RUNNING` because the +instance pool `$TEST_INSTANCE_POOL_ID` names cannot provision instances, and an instance pool +cannot be deleted (which is why that type is `local_only` — every field that recreates hits it). +`clusters` is deliberately *not* marked `local_only`: one workspace's capacity is not a property +of the suite. diff --git a/bundle/direct/autotest/fields_test.go b/bundle/direct/autotest/fields_test.go new file mode 100644 index 00000000000..dd23c12fd75 --- /dev/null +++ b/bundle/direct/autotest/fields_test.go @@ -0,0 +1,1154 @@ +// Package autotest exercises the direct engine end to end -- plan, apply, and the plan +// that follows -- rather than any one resource implementation. It lives beside the +// engine for that reason, not under dresources. +// +// TestFields catalogs how the engine handles a change to every field a user can set on +// a bundle resource. +// +// For each resource it deploys a base config once, then walks the resource's input +// struct and, per field, moves the field through every ordered pair of a small set of +// values -- "absent" included, so adding and removing a field are just the pairs with +// absent on one side. Each move is observed three ways: did the plan propose +// anything, did the apply succeed, and was the next plan clean. +// +// Nothing here shells out to the CLI. A `bundle deploy` per permutation would be +// dominated by bundle-file sync, and there are thousands of permutations. +// +// A bad outcome does not fail the test. The suite is a catalog: every result lands in +// output/.txt, and that golden is the report. A regression shows up +// as a diff in it -- for instance after an SDK bump moves a field's type. +// +// Permissions and grants are out of scope, and are stripped from every config before +// anything is planned: they are separate plan nodes describing an ACL rather than the +// resource, and left in place they make every recreate report a drifted child node +// against whichever field happened to trigger the recreate. +// +// Also out of scope for now, tracked as follow-ups: +// - remote drift (a change made outside the bundle); this suite only edits config +// - configs with more than one resource, or with an -init.sh +// - fields under a slice or map (listed at the end of each report) +package autotest + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/binary" + "encoding/json" + "errors" + "flag" + "fmt" + "hash/fnv" + "maps" + "math/rand/v2" + "os" + "os/exec" + "reflect" + "regexp" + "slices" + "strconv" + "strings" + "sync" + "testing" + "time" + + "github.com/databricks/cli/bundle/deployplan" + "github.com/databricks/cli/bundle/direct/dresources" + "github.com/databricks/cli/libs/diag" + "github.com/databricks/cli/libs/folders" + "github.com/databricks/cli/libs/git" + "github.com/databricks/cli/libs/testdiff" + "github.com/databricks/cli/libs/vfs" + "github.com/databricks/databricks-sdk-go" + "github.com/databricks/databricks-sdk-go/service/iam" + "github.com/stretchr/testify/require" +) + +func TestFields(t *testing.T) { + driven := drivenTypes(t) + + sample := sampleSize(t) + + // -update writes the report as the whole truth for a type, so it cannot come from a run + // that tested a few fields: the missing ones would read as removed. + if sample > 0 && testdiff.OverwriteMode { + t.Fatal("-sample cannot be combined with -update: a sampled run would truncate the reports") + } + + // The order fields are tested in, and the order each field's values are visited in, + // come from this seed. + runSeed := orderSeed + t.Logf("field and value order seeded from orderSeed %d", runSeed) + + var sampleFields uint64 + if sample > 0 { + sampleFields = sampleSeed(t) + t.Logf("testing %d fields per resource type, seeded from HEAD (%d)", sample, sampleFields) + } + + for _, resourceType := range driven { + t.Run(resourceType, func(t *testing.T) { + // One resource type per goroutine: the types are independent, and the + // slowest type then sets the wall time instead of their sum. + t.Parallel() + + // A testserver per type keeps the parallel runs from sharing workspace + // state. On cloud this is the same real workspace either way. + client := newClient(t) + user := workspaceUser(t, client) + + // A value library may name the workspace user or a shared test object, so the same + // variables a bundle gets are expanded here too. UNIQUE_NAME is the exception and + // is left to renderBundle, which knows the deploy it belongs to. + fv, err := loadFieldValues(resourceType, templateVars("", user.UserName)) + require.NoError(t, err) + if fv.localOnly != "" && isCloud() { + t.Skipf("local only: %s", fv.localOnly) + } + if len(fv.clouds) > 0 && isCloud() && !slices.Contains(fv.clouds, cloudName()) { + t.Skipf("not available on %s: the service exists on %s", cloudName(), strings.Join(fv.clouds, ", ")) + } + + // Outlives the field-level subtests that rebuild harnesses. + ctx := t.Context() + + rep := &report{resourceType: resourceType, started: time.Now()} //exhaustruct:ignore + runType(t, ctx, client, user, resourceType, fv, rep, runSeed, sampleFields, sample) + rep.write(t) + }) + } +} + +// runType drives one resource type. Every resource it creates belongs to this test's lifetime, +// which is what `owner` below carries into the subtests: a resource a subtest asks for is +// reused by the transitions after it, so its cleanup cannot be the subtest's. +func runType(t *testing.T, ctx context.Context, client *databricks.WorkspaceClient, user *iam.User, resourceType string, fv *fieldValues, rep *report, runSeed, sampleSeedValue uint64, sample int) { + owner := t + adapter, err := dresources.NewAdapter(dresources.SupportedResources[resourceType], resourceType, client) + require.NoError(t, err) + + // The base deploy establishes that the config itself is deployable. If it is not, + // nothing below can be attributed to a field. + h, err := newBaseline(t, ctx, client, user, resourceType, fv) + if err != nil { + rep.add(result{"(base config)", "", "create", verdictBaseError, oneLine(err.Error()), err.Error()}) + return + } + + // Enumerated from the deployed resource, not just its type: which slice and map entries + // exist decides which fields inside them can be reached at all. + base := h.snapshot() + resource, err := h.resource() + require.NoError(t, err) + fields, uncovered, inert := enumerateFields(resourceType, adapter.InputConfigType(), fv, resource, runSeed, + declaredUnsettable(adapter), h.unique) + rep.addCoverage(fields, uncovered) + fields = pickSample(fields, sampleSeedValue, sample, rep) + + // A label that does not say what the value is needs the legend to: a container's says only how + // many entries it has, and an alias says nothing at all. + for _, f := range fields { + for _, value := range f.values { + if label := f.label(value); isContainer(f.kind) || label != valueLabel(value) { + rep.addLegend(f.path, label, value) + } + } + } + + // Fields the resource declares a user cannot meaningfully set. Recorded with the + // resource's own reason rather than tested: every transition would come back SUPPRESSED + // with exactly that reason. + for _, path := range slices.Sorted(maps.Keys(inert)) { + rep.add(result{path, "", "", verdictSkipped, "backend output: " + inert[path], ""}) + } + + // Fields the resource says it ignores local changes to. Not skipped: the declaration is + // a claim about behaviour, and a transition either bears it out or does not. + ignoredLocally := declaredIgnoredLocally(adapter) + // Every rule the resource declares about deliberately not acting, which is what makes a + // skipped change the expected outcome rather than a failure to reach a value. + decl := declarations{deliberate: declaredDeliberate(adapter), idFields: declaredIDFields(adapter)} + + for path, reason := range fv.skip { + rep.add(result{path, "", "", verdictSkipped, reason, ""}) + } + + // Some resources do not converge even with nothing changed: a field the read never + // echoes diffs forever. Measure that once, against the field that causes it, and leave + // it out of every transition's drift -- otherwise it dirties every post-deploy plan and + // gets blamed on whichever field happened to be under test. + baseline, basePlan, err := baselineDrift(h) + if err != nil { + rep.add(result{"(baseline)", "", "", verdictPlanError, oneLine(err.Error()), err.Error()}) + return + } + for _, path := range slices.Sorted(maps.Keys(baseline)) { + // The plan is the whole diagnosis: a field drifting straight after the create is usually + // one the backend did not apply, and only its remote_state says so. Without it the row + // states the symptom and nothing else, which is not enough to tell a write the API + // ignored from a read that does not echo the field. + rep.add(result{path, "", "", verdictBaselineDrift, "drifts with no config change", withContext("plan right after the base deploy:", basePlan)}) + } + + for _, f := range fields { + // Set once the resource is in an unknown state and could not be replaced. Every + // remaining transition would be observing that rather than the field, and so would + // every later field, so they are recorded as such instead of run. + var broken error + + t.Run(subtestName(f.path), func(t *testing.T) { + // What the field is currently deployed as, when that is known. Transitions are + // generated in runs that share a starting value, so this skips about half of + // the setup deploys -- the single biggest cost in the suite. + deployed, deployedKnown := absent, false + + for _, tr := range f.transitions() { + t.Run(tr.label(), func(t *testing.T) { + if broken != nil { + rep.add(result{f.path, tr.fromLabel, tr.toLabel, verdictBaseError, oneLine(broken.Error()), broken.Error()}) + return + } + + reuse := deployedKnown && valueLabel(deployed) == valueLabel(tr.from) + res := runTransition(t, h, f.path, tr, reuse, baseline, decl) + + // A starting value the deployed resource will not take -- typically a + // field the API refuses to clear -- is not a dead end: a fresh resource + // created without the field genuinely starts absent. Rebuild and try the + // transition once more, so it is observed rather than written off. + if res.verdict == verdictStartNotReached { + // With the starting value written into the config before the create, so + // the new resource is built holding it: a field the API will not move to + // a value can still be created with it. + rebuilt, err := rebuild(owner, ctx, client, user, resourceType, fv, h, preset{f.path, tr.from}) + uncreatable := err != nil + if uncreatable { + // The resource cannot even be created holding that value, which is the + // answer for this transition: a dashboard has to have a display name, + // so nothing starts from it being absent. Recorded, and a plain + // rebuild follows so the remaining transitions have a resource. + res.detail = "cannot create the resource with this starting value: " + oneLine(err.Error()) + res.evidence = withContext("error creating the resource with the starting value:", err.Error()) + rebuilt, err = rebuild(owner, ctx, client, user, resourceType, fv, h) + } + if err != nil { + // No resource left to observe anything on, which is a different + // statement from the field refusing its starting value. + res.verdict = verdictBaseError + res.detail = oneLine(err.Error()) + res.evidence = err.Error() + broken = err + } else { + h, base, baseline = rebuilt, rebuilt.snapshot(), remeasure(rebuilt, baseline, rep) + if !uncreatable { + res = runTransition(t, h, f.path, tr, false, baseline, decl) + } + } + } + + res = checkDeclaredInert(res, ignoredLocally) + rep.add(res) + + // Only a clean apply leaves the field provably at "to". + deployed, deployedKnown = tr.to, res.verdict == verdictOK || res.verdict == verdictRecreate + + if res.verdict == verdictRecreate { + // A recreate replaced the resource without going through rebuild, so the + // baseline belongs to one that no longer exists. + baseline = remeasure(h, baseline, rep) + } + + if broken != nil || res.verdict.leavesResourceUsable() { + return + } + deployedKnown = false + // The resource is in an unknown state; carrying it into the next + // transition would turn one failure into a run of them. + rebuilt, err := rebuild(owner, ctx, client, user, resourceType, fv, h) + if err != nil { + broken = err + return + } + h, base, baseline = rebuilt, rebuilt.snapshot(), remeasure(rebuilt, baseline, rep) + }) + } + }) + if broken != nil { + // The type has no usable resource left, so no later field can be observed either. + rep.add(result{"(rebuild)", "", "", verdictBaseError, oneLine(broken.Error()), broken.Error()}) + return + } + // Put the field back and confirm the resource converged. A field the API cannot + // clear leaves it drifted for good, which would otherwise be blamed on every + // field tested afterwards -- so start over on a fresh resource. A new name is + // what makes that cheap: reusing the old one waits out an asynchronous delete. + h.restore(base) + if !h.converged() { + rebuilt, err := rebuild(owner, ctx, client, user, resourceType, fv, h) + if err != nil { + rep.add(result{f.path, "", "", verdictBaseError, oneLine(err.Error()), err.Error()}) + return + } + h, base, baseline = rebuilt, rebuilt.snapshot(), remeasure(rebuilt, baseline, rep) + } + } +} + +// sample cuts the field list down to -sample entries, drawn without replacement. Sorting first +// makes the draw depend only on the seed, not on the order enumeration happened to produce. +// The chosen paths go to the report, which needs them to compare a partial run against a +// golden written by a full one. +func pickSample(fields []field, seed uint64, sample int, rep *report) []field { + // A type with no more fields than the sample size is covered in full, so its report stays + // comparable whole -- rep.sampled is left nil. + if sample <= 0 || len(fields) <= sample { + return fields + } + ordered := slices.Clone(fields) + slices.SortFunc(ordered, func(a, b field) int { return strings.Compare(a.path, b.path) }) + rng := rand.New(rand.NewPCG(seed, 2)) + rng.Shuffle(len(ordered), func(i, j int) { ordered[i], ordered[j] = ordered[j], ordered[i] }) + ordered = ordered[:sample] + for _, f := range ordered { + rep.addSampled(f.path) + } + return ordered +} + +// sameField reports whether a drifting path is the field under test. The planner may report +// a change against an ancestor rather than the leaf -- a whole "config" block instead of +// "config.auto_capture_config.catalog_name" -- and that is still the field's own drift, not +// some other field's. +func sameField(drifting, path string) bool { + drifting, path = normalizeSelectors(drifting), normalizeSelectors(path) + return drifting == path || + strings.HasPrefix(path, drifting+".") || strings.HasPrefix(path, drifting+"[") || + strings.HasPrefix(drifting, path+".") || strings.HasPrefix(drifting, path+"[") +} + +// selectorPattern matches the key-value selector the planner uses to name a slice element it +// can identify -- "tasks[task_key='seeded']" for what this suite calls "tasks[0]". +var selectorPattern = regexp.MustCompile(`\[(?:[^'=\]]+='[^']*'|[0-9]+)\]`) + +// normalizeSelectors reduces both ways of naming a slice element to one form, so a change the +// planner recorded against "tasks[task_key='seeded'].run_if" is recognised as the same field as +// "tasks[0].run_if". Which element it is does not need to be resolved: this suite drives one +// element at a time, so at most one is in play. +func normalizeSelectors(path string) string { + return selectorPattern.ReplaceAllString(path, "[]") +} + +// relatedChange returns the plan's change for the field, or for whatever ancestor or +// descendant of it the planner actually recorded -- it diffs at its own granularity, so a +// change to "tasks[0].description" can be reported against "tasks". A pending change is +// preferred over a skipped one, since that is the one that explains what will happen. +func relatedChange(plan *deployplan.Plan, node, path string) (*deployplan.ChangeDesc, bool) { + _, change, ok := relatedChangeKey(plan, node, path) + return change, ok +} + +// relatedChangeKey is relatedChange, also returning the key the change was recorded under so +// a caller comparing two plans can look the same key up in both. +func relatedChangeKey(plan *deployplan.Plan, node, path string) (string, *deployplan.ChangeDesc, bool) { + if plan == nil { + return "", nil, false + } + entry, ok := plan.Plan[node] + if !ok { + return "", nil, false + } + + var foundKey string + var found *deployplan.ChangeDesc + for _, key := range slices.Sorted(maps.Keys(entry.Changes)) { + change := entry.Changes[key] + if !sameField(key, path) { + continue + } + if change.Action != deployplan.Skip { + return key, change, true + } + if found == nil { + foundKey, found = key, change + } + } + return foundKey, found, found != nil +} + +// changeAt looks a change up by its exact key. +func changeAt(plan *deployplan.Plan, node, key string) (*deployplan.ChangeDesc, bool) { + if plan == nil { + return nil, false + } + entry, ok := plan.Plan[node] + if !ok { + return nil, false + } + change, ok := entry.Changes[key] + return change, ok +} + +// fieldWasDropped reports whether the plan mentions the field and yet will not act on it. +// The planner records a change at whatever granularity it diffed, which is not always the +// leaf -- a whole "tasks" entry rather than "tasks[0].description" -- so every entry naming +// the field, an ancestor of it, or something under it counts. A field the plan does not +// mention at all is not "dropped": there is nothing to conclude from its absence, and the +// node's own action decides. +func fieldWillChange(plan *deployplan.Plan, node, path string) bool { + // A create or a recreate replaces the resource and records no per-field detail, so there is + // nothing to consult and the node's own action is the answer. + if !hasFieldChanges(plan, node) { + return true + } + change, ok := relatedChange(plan, node, path) + return ok && change.Action != deployplan.Skip +} + +// hasFieldChanges reports whether the plan records per-field detail for the node. +func hasFieldChanges(plan *deployplan.Plan, node string) bool { + if plan == nil { + return false + } + entry, ok := plan.Plan[node] + return ok && len(entry.Changes) > 0 +} + +// remeasure takes the resource's own baseline drift, for a resource that has just replaced the +// one measured before: carrying the old measurement over would either hide drift the new one +// has or blame it on whichever field is under test. +// +// A plan that fails here leaves the previous measurement in place, which is closer than +// nothing, and says so: every verdict after it is measured against a baseline that may not be +// this resource's. +func remeasure(h *bundleHarness, previous map[string]bool, rep *report) map[string]bool { + measured, _, err := baselineDrift(h) + if err == nil { + return measured + } + // Recorded, not just logged: every verdict after this is measured against a baseline that + // belongs to a resource which no longer exists, and a reader of the report has to know. + rep.add(result{"(baseline)", "", "", verdictPlanError, oneLine(err.Error()), err.Error()}) + return previous +} + +// baselineCovers reports whether the drift this field would be blamed for is drift that was +// already there. The comparison is against the key the plan resolved the field to, not the leaf +// under test and not any relation of it: existing drift at "config" explains a change recorded +// against "config", but says nothing about whether a write to "config.foo" landed, and treating +// it as an excuse would let an ignored write pass unnoticed. +func baselineCovers(baseline map[string]bool, plan *deployplan.Plan, node, path string) bool { + if baseline[path] { + return true + } + key, _, ok := relatedChangeKey(plan, node, path) + return ok && baseline[key] +} + +// converged reports whether a plan no longer wants to change the field, at whatever +// granularity the planner recorded it. +func converged(plan *deployplan.Plan, node, path string) bool { + change, ok := relatedChange(plan, node, path) + return !ok || change.Action == deployplan.Skip +} + +// reachedValue reports whether a plan entry for the field means the remote now holds what +// the config asked for. A pending change means it does not; so does a change the planner +// dropped for a reason other than the value already being there -- an app whose compute is +// stopped suppresses a command with "no active deployment", and the command never landed. +func reachedValue(change *deployplan.ChangeDesc, path string, deliberate []dresources.FieldRule) bool { + if change.Action != deployplan.Skip { + return false + } + if benignSuppressions[change.Reason] { + return true + } + // A field the remote type has no place for -- a write-only input like purge_on_delete, which + // the bundle acts on at destroy and the API never returns. The engine's own definition of the + // reason is that the remote is always nil, so there is nothing for the value to be reached + // in; it is in state and that is all there is. + if change.Reason == deployplan.ReasonMissingInRemote { + return true + } + // A skip for a reason the resource declares about this field is the engine doing what it + // says it does, so the config's value is as reached as it will ever be. Two shapes: a field + // whose local changes are dropped never holds what the config asks for (asking whether the + // starting value landed has no answer, and treating "no" as failure buried the useful + // OK_INERT verdict), and a field whose remote value is not compared -- an input_only alias -- + // has the value in state and on the wire, just not visible on a read. + reason, declared := ruleReason(deliberate, path) + return declared && reason == change.Reason +} + +// runTransition moves one field from one value to another and reports what happened. +// startsDeployed says the field already holds tr.from, so the setup deploy can be skipped. +// declarations are the resource's own claims about its fields, read from resources.yml. Two +// slices of the same type as separate parameters would let a caller swap them unnoticed. +type declarations struct { + // deliberate: rules saying the engine does not act on a field, so a skipped change is the + // declared behaviour rather than a value that failed to land. + deliberate []dresources.FieldRule + // idFields: rules naming the fields that compose the resource's ID. + idFields []dresources.FieldRule +} + +func runTransition(t *testing.T, h *bundleHarness, path string, tr transition, startsDeployed bool, baseline map[string]bool, decl declarations) result { + from, to := tr.from, tr.to + res := result{field: path, from: tr.fromLabel, to: tr.toLabel} //exhaustruct:ignore + + // Reach the starting value. + if err := h.setField(path, from); err != nil { + res.verdict = verdictUnsettable + res.detail = err.Error() + return res + } + if !startsDeployed { + if _, diags := h.deploy(); diags.HasError() { + res.verdict = idFieldRequired(from, path, decl, diags, verdictBaseError) + res.detail = firstError(diags) + res.evidence = withContext("error from the deploy:", allErrors(diags)) + return res + } + // A deploy the backend accepted does not mean the field now holds "from": a write it + // ignores leaves the old value there, and the transition below would then be reported + // under a label that is not what happened -- "absent to 168" while the remote still + // holds 720. Cheaper to check than to reason about afterwards. + plan, diags := h.readPlan() + if diags.HasError() { + // Without this plan there is no way to know the field reached "from", and going ahead + // would label whatever happens next as a move that may never have started. + res.verdict = verdictPlanError + res.detail = firstError(diags) + return res + } + if change, ok := relatedChange(plan, h.node, path); ok && !reachedValue(change, path, decl.deliberate) && !baselineCovers(baseline, plan, h.node, path) { + res.verdict = verdictStartNotReached + // The planner's own reason for not acting, when it gave one -- "the app has no active + // deployment" is the whole explanation. The field is already the row's first column, + // so the detail carries only what that column cannot say. + res.detail = change.Reason + res.evidence = withContext("plan after deploying the starting value:", planJSON(plan)) + return res + } + } + + // Now the change under test. + if err := h.setField(path, to); err != nil { + res.verdict = verdictUnsettable + res.detail = err.Error() + return res + } + + pending, plan, diags := h.plan() + if diags.HasError() { + pending.cancel() + res.verdict = verdictPlanError + res.detail = firstError(diags) + return res + } + action := nodeAction(plan, h.node) + // The field's own entries, not just the node's action: another field already drifting keeps + // the node at "update", and reading only that would let a change to *this* field be + // dropped and still come out as OK once the other field is filtered from the post-deploy + // plan. + // A field the plan does not act on, however the node's action reads. Another field already + // drifting keeps the node at "update", and going ahead on that would apply nothing for this + // field and then report OK once the other field is filtered out as known baseline drift. + if action == deployplan.Skip || !fieldWillChange(plan, h.node, path) { + pending.cancel() + res.verdict, res.detail = explainSkip(plan, h.node, path) + return res + } + + if diags := h.apply(pending, plan); diags.HasError() { + switch { + case isTimeout(diags): + res.verdict = verdictTimeout + case isAPIError(diags): + res.verdict = idFieldRequired(to, path, decl, diags, verdictBackendError) + default: + res.verdict = verdictDeployError + } + res.detail = firstError(diags) + res.evidence = withContext("error from the deploy:", allErrors(diags)) + return res + } + + after, diags := h.readPlan() + if diags.HasError() { + res.verdict = verdictPlanError + res.detail = firstError(diags) + return res + } + // A write the backend accepted and then ignored leaves the field's remote value exactly + // as it was before the apply. That is worth separating from other drift: the engine did + // everything right and the request simply had no effect. + // + // One read cannot tell that apart from a read that was merely stale, so take a second + // one. If the value has appeared by then the write did land and the first read was + // behind; if it still has not, the field really was ignored. + if wasIgnored(plan, after, h.node, path) { + second, diags := h.readPlan() + if diags.HasError() { + res.verdict = verdictPlanError + res.detail = firstError(diags) + return res + } + switch { + case wasIgnored(plan, second, h.node, path): + res.verdict = verdictUpdateIgnored + res.evidence = withContext("plan taken twice after the deploy, still unchanged:", planJSON(second)) + t.Logf("the write was accepted but the remote value did not move on two reads:\n%s", res.evidence) + return res + + case converged(second, h.node, path): + // The remote moved and the plan is now clean, so the first read was behind. + res.verdict = verdictStaleRead + res.evidence = withContext("plan taken right after the deploy:", planJSON(after)) + return res + + default: + // The remote moved, but to something other than what was asked for, so this is + // drift rather than a stale read. Classified below, against the later plan. + after = second + } + } + + if own, child, bare := driftDetail(after, h.node, baseline); own != "" || child != "" || bare != "" { + // Whether the field under test is among the drifting ones decides who is to blame. + drifted := strings.Split(own, ",") + // The plan that still wants a change is the whole evidence for this verdict, so + // keep it: -v prints it, and it goes into the full report. + res.evidence = withContext("plan taken right after the deploy:", planJSON(after)) + t.Logf("post-deploy plan still proposes a change:\n%s", res.evidence) + switch { + case bare != "": + // The whole resource is being replaced again, with no field named. + res.verdict, res.detail = verdictDrift, bare + case own == "": + // A child node left behind by a recreate: a different problem, different fix. + res.verdict, res.detail = verdictDriftChild, child + case slices.ContainsFunc(drifted, func(p string) bool { return sameField(p, path) }): + res.verdict, res.detail = verdictDrift, own + default: + res.verdict, res.detail = verdictCollateral, own + } + return res + } + + res.verdict = verdictOK + if action == deployplan.Recreate { + res.verdict = verdictRecreate + } + return res +} + +// newBaseline builds a harness and deploys the config as written. +// +// owner is the test whose lifetime the resource belongs to, which is the one running the +// config -- not the subtest that happened to ask for the rebuild. Registering the cleanup on +// the subtest would destroy the resource as soon as that transition ended, and every +// transition after it would silently be creating a new one. +func newBaseline(owner *testing.T, ctx context.Context, client *databricks.WorkspaceClient, user *iam.User, resourceType string, fv *fieldValues, presets ...preset) (*bundleHarness, error) { + h, err := newHarness(owner, ctx, client, user, resourceType, uniqueName(), fv.base, fv.deps, fv.variables) + if err != nil { + return nil, err + } + owner.Cleanup(func() { _ = h.destroy() }) + + // Applied before the first deploy, so the create carries the value rather than an update + // having to reach it. That is the only way to start from a value the API will not move a + // field to -- most often absent, for a field it refuses to clear. + for _, p := range presets { + if err := h.setField(p.path, p.value); err != nil { + return nil, fmt.Errorf("presetting %s: %w", p.path, err) + } + } + + action, diags := h.deploy() + if diags.HasError() { + // The whole diagnostics, not firstError's one-line form: the caller records this as + // evidence in the full report, where the point is to read the backend's own words. + return nil, errors.New(allErrors(diags)) + } + if action != deployplan.Create { + return nil, fmt.Errorf("expected create, got %s", action) + } + return h, nil +} + +// preset is a field value to write into the config before the resource is first deployed. +type preset struct { + path string + value any +} + +// rebuild starts over on a fresh resource, leaving the old one to be destroyed at the +// end of the test. A new name is what makes this cheap: reusing the old one can mean +// waiting out an asynchronous delete. +func rebuild(owner *testing.T, ctx context.Context, client *databricks.WorkspaceClient, user *iam.User, resourceType string, fv *fieldValues, old *bundleHarness, presets ...preset) (*bundleHarness, error) { + owner.Cleanup(func() { _ = old.destroy() }) + return newBaseline(owner, ctx, client, user, resourceType, fv, presets...) +} + +// explainSkip says why a plan came back with nothing to do for the field under test. +// The distinction matters: a field the planner never diffed is not a finding (an unset +// bool and an explicit false are the same on the wire), whereas a field it diffed and +// then dropped is, and the engine records its own reason for that. +func explainSkip(plan *deployplan.Plan, node, path string) (verdict, string) { + if _, ok := plan.Plan[node]; !ok { + return verdictNoPlan, "no plan entry" + } + // Whichever entry names the field, which is not always the leaf: the planner records a + // change at the granularity it diffed at, so the reason for dropping "tasks[0].foo" may + // be attached to "tasks". + change, ok := relatedChange(plan, node, path) + if !ok { + return verdictNotObservable, "" + } + if change.Reason != "" { + return verdictSuppressed, change.Reason + } + return verdictSuppressed, string(change.Action) +} + +// valueLabel renders a value for the subtest name and the report. A slice or map is +// labelled by size, since the point of testing one is how many entries it has. An identity +// field's value carries the run's own suffix, which is redacted here so the label -- and so +// the golden -- is the same on every run. +func valueLabel(v any) string { + if v == nil { + return "absent" + } + switch value := reflect.ValueOf(v); value.Kind() { + case reflect.Slice, reflect.Array: + return "len" + strconv.Itoa(value.Len()) + case reflect.Map: + return "keys" + strconv.Itoa(value.Len()) + default: + } + switch s := oneLine(fmt.Sprintf("%v", v)); s { + case "": + return "empty" + default: + return shortLabel(s) + } +} + +// subtestName turns a field path into a subtest name. An index is written "tasks_0", not +// "tasks[0]": brackets are a character class to the -run regex, so a filter copied from the test +// output would quietly mean something else. +func subtestName(path string) string { + return indexPattern.ReplaceAllString(path, "_$1") +} + +// indexPattern matches a slice index in a field path. +var indexPattern = regexp.MustCompile(`\[([0-9]+)\]`) + +// labelSafe matches the characters a label keeps verbatim: everything else would need +// quoting to pass a label back to `go test -run`. +var labelSafe = regexp.MustCompile(`[^A-Za-z0-9._@/:=-]+`) + +// shortLabel renders a value as a token that is readable, stable, and safe to paste into a +// -run filter. A value that survives as-is is left alone; anything long or punctuated -- a +// cluster policy definition is a whole JSON document -- is trimmed and given a short digest, +// so two values that share a prefix still get different labels. +func shortLabel(s string) string { + if clean := cleanLabel(s); clean != "" { + return clean + } + + // No alias was assigned for this value -- it is not one of a field's declared values, so + // nothing holds a legend entry for it. A digest keeps the label short and unique. + digest := fnv.New32a() + _, _ = digest.Write([]byte(s)) + safe := labelSafe.ReplaceAllString(s, "_") + return strings.Trim(safe[:min(len(safe), maxLabelLength-5)], "_") + "~" + strconv.FormatUint(uint64(digest.Sum32())%0x10000, 16) +} + +// maxLabelLength is how long a value's own label may be before it is aliased instead. +const maxLabelLength = 24 + +// cleanLabel returns the value unchanged when it reads as a label already -- short, and free of +// characters that would need quoting in a test filter -- and "" when it does not. +func cleanLabel(s string) string { + if len(s) <= maxLabelLength && labelSafe.ReplaceAllString(s, "_") == s { + return s + } + return "" +} + +// allErrors returns every error diagnostic in full -- status, code, message, endpoint -- +// the way the CLI would print it. firstError's one-line form is for the report column. +func allErrors(diags diag.Diagnostics) string { + var out []string + for _, d := range diags { + if d.Severity == diag.Error { + out = append(out, redactIDs(d.Summary)) + } + } + return strings.Join(out, "\n") +} + +func firstError(diags diag.Diagnostics) string { + for _, d := range diags { + if d.Severity == diag.Error { + return oneLine(d.Summary) + } + } + return "" +} + +// apiErrorCodes are the error codes that mean the backend rejected the request rather +// than the CLI failing. Diagnostics carry a rendered message, not the error value, so +// there is nothing to match with errors.As by the time we see them. +// invalidRequestCodes are the subset of apiErrorCodes that say the request itself was wrong, as +// opposed to the caller lacking permission or asking too often. +var invalidRequestCodes = []string{ + "INVALID_PARAMETER_VALUE", + "BAD_REQUEST", +} + +var apiErrorCodes = []string{ + "INVALID_PARAMETER_VALUE", + "BAD_REQUEST", + "RESOURCE_DOES_NOT_EXIST", + "RESOURCE_ALREADY_EXISTS", + "PERMISSION_DENIED", + "REQUEST_LIMIT_EXCEEDED", + "FEATURE_DISABLED", + "NOT_FOUND", +} + +func isTimeout(diags diag.Diagnostics) bool { + for _, d := range diags { + if strings.Contains(d.Summary, context.DeadlineExceeded.Error()) { + return true + } + } + return false +} + +// idFieldRequired downgrades a rejected deploy to OK_ID_FIELD_REQUIRED when the value it was +// carrying was absent on a field the resource declares as part of its ID. Every input is a +// declaration or a fact about the run -- the field is named in provided_id_fields, the value is +// absent, the backend refused -- and none is the error's wording, which cannot be attributed +// to a single field because a recreate carries the whole resource. +// +// The backend having refused still has to hold: an internal CLI failure on the same deploy is a +// real defect and keeps its own verdict. +func idFieldRequired(value any, path string, decl declarations, diags diag.Diagnostics, otherwise verdict) verdict { + // Same nil test valueLabel uses to print "absent", so the verdict and the row's own column + // can never disagree about which value this was. + // + // Not any API error: a rejection is only evidence about the request when it says the request + // was malformed. PERMISSION_DENIED and REQUEST_LIMIT_EXCEEDED are answers about the caller + // and the moment, and would have come back whatever the field held. + if value != nil || !hasErrorCode(diags, invalidRequestCodes) { + return otherwise + } + if _, isID := ruleReason(decl.idFields, path); !isID { + return otherwise + } + return verdictIDFieldRequired +} + +func isAPIError(diags diag.Diagnostics) bool { + return hasErrorCode(diags, apiErrorCodes) +} + +func hasErrorCode(diags diag.Diagnostics, codes []string) bool { + for _, d := range diags { + for _, code := range codes { + if strings.Contains(d.Summary, code) { + return true + } + } + } + return false +} + +// checkDeclaredInert re-reads a result for a field the resource claims to ignore locally. +// Suppressed with the declared reason confirms the claim; anything else means a change the +// resource said it would drop was not dropped. +func checkDeclaredInert(res result, rules []dresources.FieldRule) result { + reason, declared := ruleReason(rules, res.field) + if !declared { + return res + } + + switch res.verdict { + case verdictSuppressed, verdictNotObservable: + // Both mean the change had no effect, which is what inert claims. Which of the two + // appears depends on whether the planner suppresses the change or drops the entry + // outright, and that is an implementation detail of the rule, not of the field. + res.verdict, res.detail = verdictInertConfirmed, reason + case verdictOK, verdictRecreate: + // The change took effect, so the field is not inert after all. + res.detail = fmt.Sprintf("declared inert (%s) but the change was applied", reason) + res.verdict = verdictInertViolated + default: + // An error or a drift says something more specific than the declaration does; leave + // the verdict alone and note that the field was expected to be inert. + res.detail = fmt.Sprintf("%s (declared inert: %s)", res.detail, reason) + } + return res +} + +// baselineDrift returns the fields the resource already wants to change with the config +// exactly as deployed. +func baselineDrift(h *bundleHarness) (map[string]bool, string, error) { + plan, diags := h.readPlan() + if diags.HasError() { + // Returning no drift would leave every later verdict measured against an unknown + // baseline: a field that was already drifting would be blamed on whichever + // transition happened to run. + return nil, "", errors.New(firstError(diags)) + } + if plan == nil { + return nil, "", errors.New("no plan") + } + entry, ok := plan.Plan[h.node] + if !ok { + return nil, "", nil + } + out := map[string]bool{} + for path, change := range entry.Changes { + if change.Action != deployplan.Skip { + out[path] = true + } + } + return out, planJSON(plan), nil +} + +// wasIgnored reports whether the write for one field was accepted and then had no effect: +// the field is still pending in the post-deploy plan, and its remote value is identical to +// what the pre-deploy plan saw. +func wasIgnored(before, after *deployplan.Plan, node, path string) bool { + // Matched by relation, since the planner records a change at the granularity it diffed at: + // a write to "libraries[1].pypi.repo" can be reported against "libraries[1].pypi". But the + // two reads must be compared at the *same* key -- the granularity can differ between them, + // and comparing a leaf's value against its parent object never matches, which would turn + // every ignored write into drift. + beforeKey, beforeChange, ok := relatedChangeKey(before, node, path) + if !ok { + return false + } + afterChange, ok := changeAt(after, node, beforeKey) + if !ok || afterChange.Action == deployplan.Skip { + return false + } + return jsonEqual(beforeChange.Remote, afterChange.Remote) +} + +// jsonEqual compares two plan values, which are decoded as any and so cannot be compared +// directly. +func jsonEqual(a, b any) bool { + left, err := json.Marshal(a) + if err != nil { + return false + } + right, err := json.Marshal(b) + if err != nil { + return false + } + return bytes.Equal(left, right) +} + +// driftDetail describes what a post-deploy plan still wants to change, split into drift +// on the node under test (own: the field paths that did not stick) and drift on any +// other node (child: how a recreate orphaning a permissions or grants node shows up). +// Both are "" when the deploy converged. +func driftDetail(plan *deployplan.Plan, node string, baseline map[string]bool) (own, child, bare string) { + var ownPaths, childNodes []string + for key, entry := range plan.Plan { + if entry.Action == deployplan.Skip { + continue + } + if key != node { + childNodes = append(childNodes, strings.TrimPrefix(key, node+".")+":"+string(entry.Action)) + continue + } + for p, ch := range entry.Changes { + // Exact, not by relation: a baseline recorded against an ancestor must not hide new + // drift reported against something under it. Over-reporting is the safer error for a + // catalog -- a spurious row is visible and can be investigated, a hidden one cannot. + if ch.Action != deployplan.Skip && !baseline[p] { + ownPaths = append(ownPaths, p) + } + } + if len(entry.Changes) == 0 { + // A recreate carries no per-field detail, so there is no path to name: the whole + // resource is being replaced again. Reported as the field's own non-convergence, + // since attributing it to a sibling would name an action where a field belongs. + bare = string(entry.Action) + } + } + slices.Sort(ownPaths) + slices.Sort(childNodes) + return strings.Join(ownPaths, ","), strings.Join(childNodes, " "), bare +} + +// generatedIDs are the shapes of value that differ between runs, all of which turn up inside +// quoted resource names and ids in error messages -- so they have to be redacted or no golden +// would ever be stable. Ordered: the unique suffix first, since it is the most specific. +var generatedIDs = []struct { + pattern *regexp.Regexp + replacement string +}{ + // The suffix this suite gives every resource it creates. + {regexp.MustCompile(`f[0-9a-f]{20}`), "[UNIQUE_NAME]"}, + // The placeholder an identity field's values carry. Only ever seen in a label, where it + // reads better without brackets -- an error message carries the substituted suffix, which + // the rule above catches. + {regexp.MustCompile(regexp.QuoteMeta(uniqueMarker)), "UNIQUE"}, + // A backend-assigned id. Before the UUID rule, and covering both shapes: the fake server + // hands out UUIDs where a real workspace hands out hex, and the two have to redact to the + // same placeholder or every error message naming one would differ between them. + {regexp.MustCompile("id=[0-9A-Fa-f-]{6,}"), "id=[ID]"}, + {regexp.MustCompile(`[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}`), "[UUID]"}, +} + +// workspaceUserName holds the identity this run deploys as, so it can be redacted: it is +// tester@databricks.com against the fake server and a service principal's UUID on a real +// workspace, and it turns up inside seeded values and error messages alike. Written once with +// the same value by every harness, so the mutex only guards the race, not a decision. +var ( + workspaceUserMu sync.Mutex + workspaceUserName string +) + +func rememberWorkspaceUser(name string) { + workspaceUserMu.Lock() + defer workspaceUserMu.Unlock() + workspaceUserName = name +} + +// redactIDs replaces every run-specific id in a message with a stable placeholder. +func redactIDs(s string) string { + for _, id := range generatedIDs { + s = id.pattern.ReplaceAllString(s, id.replacement) + } + + workspaceUserMu.Lock() + user := workspaceUserName + workspaceUserMu.Unlock() + if user != "" { + s = strings.ReplaceAll(s, user, "[USERNAME]") + } + return s +} + +// sampleSize limits how many of a type's fields are tested, for a run that has to be cheap +// rather than exhaustive -- a PR check against a real workspace. Which fields are picked comes +// from the commit, so successive commits cover different ground, and a whole run's picks are +// reproducible from its SHA alone. +var sampleSizeFlag = flag.Int("sample", -1, "test only N fields per resource type (0 tests every field; -1 decides from the environment)") + +// sampleSize is how many fields per type this run tests, 0 meaning all of them. +// +// A cloud run costs minutes per type and hours in total, which is too much for every PR, so on cloud +// it samples unless the commit asks for everything. That is what AUTOTEST_ALL in a commit title means: +// put it there and the PR's integration run drives every field against a real workspace. +// +// Locally a full run is 11 seconds, so there is nothing to save and the default is all of them -- +// which also keeps the committed goldens compared in full by ./task test. +func sampleSize(t *testing.T) int { + if *sampleSizeFlag >= 0 { + return *sampleSizeFlag + } + if !isCloud() { + return 0 + } + if subject := commitSubject(); strings.Contains(subject, autotestAllMarker) { + t.Logf("commit title contains %s, so every field runs against the workspace", autotestAllMarker) + return 0 + } + t.Logf("cloud run without %s in the commit title, so %d fields per type are sampled; "+ + "put %s in a commit title to run them all", autotestAllMarker, defaultCloudSample, autotestAllMarker) + return defaultCloudSample +} + +// autotestAllMarker in a commit title asks a cloud run for every field rather than a sample. +const autotestAllMarker = "AUTOTEST_ALL" + +// defaultCloudSample is how many fields per type a cloud run tests when the commit does not ask for +// all of them. Two is enough to establish that every resource type still deploys and that the fields +// picked still behave as the goldens record. +const defaultCloudSample = 2 + +// commitSubject returns the subject line of HEAD, or "" when it cannot be read. A subprocess because +// libs/git exposes the commit id but not its message, and this runs once per suite. +func commitSubject() string { + out, err := exec.Command("git", "log", "-1", "--format=%s").Output() + if err != nil { + return "" + } + return strings.TrimSpace(string(out)) +} + +// sampleSeed derives the sample's seed from HEAD. Unlike orderSeed it may move freely: a +// sampled run compares each field's rows against the same golden as a full run, and never +// rewrites it, so the seed cannot invalidate anything committed. +func sampleSeed(t *testing.T) uint64 { + wd, err := os.Getwd() + require.NoError(t, err) + root, err := folders.FindDirWithLeaf(wd, git.GitDirectoryName) + require.NoError(t, err) + repo, err := git.NewRepository(t.Context(), vfs.MustNew(root)) + require.NoError(t, err) + commit, err := repo.LatestCommit() + // No fallback: a run that cannot name its commit would pick fields no one can reproduce, + // which is the one property sampling has to keep. + require.NoError(t, err, "sampling needs the commit to seed from") + require.NotEmpty(t, commit, "sampling needs the commit to seed from") + sum := sha256.Sum256([]byte(commit)) + return binary.BigEndian.Uint64(sum[:8]) +} + +// orderSeed fixes the order fields are tested in and the order each field's values are +// visited in. Bump it to explore a different order, and regenerate the reports. +// +// It is a constant rather than something derived -- HEAD was the first attempt -- because +// the reports are committed and some verdicts depend on the order. A field the API cannot +// clear leaves the remote holding an old value, and whether the next transition then reads +// that as an ignored write or as drift depends on what ran before it. A seed taken from the +// tree or the clock moves under the golden: committing the report changes HEAD, so the +// report would never validate at the commit that contains it. +const orderSeed uint64 = 1 + +// withContext labels an evidence block, so a reader of the full report knows what they are +// looking at. The label is its own line, leaving the JSON below it copy-pasteable. +func withContext(label, body string) string { + return label + "\n" + body +} + +// planJSON renders a plan for a -v dump. +func planJSON(plan *deployplan.Plan) string { + body, err := json.MarshalIndent(plan, "", " ") + if err != nil { + return err.Error() + } + return string(body) +} + +func oneLine(s string) string { + s = redactIDs(s) + s = strings.ReplaceAll(s, "\n", " ") + s = strings.Join(strings.Fields(s), " ") + // A long URL in the middle pushes out the part that says what went wrong -- a request that + // never completed reads as `Get "https://…": read tcp …: operation timed out`, and truncating + // at 140 characters kept the host and dropped the cause. The URL identifies nothing a reader + // of the report can use: the resource is already the row's own field, and the id is redacted. + s = urlPattern.ReplaceAllString(s, `"…"`) + if len(s) > 140 { + // Elided in the middle, keeping both ends: the front says which resource and operation, + // the back says what the backend or the network answered. Cutting at the front dropped + // the answer, which is the half a reader needs. + s = s[:90] + "..." + s[len(s)-47:] + } + return s +} + +// urlPattern matches a quoted http(s) URL, which carries a workspace host and a long path and +// crowds out the error that follows it. +var urlPattern = regexp.MustCompile(`"https?://[^"]*"`) diff --git a/bundle/direct/autotest/fixtures_test.go b/bundle/direct/autotest/fixtures_test.go new file mode 100644 index 00000000000..b86d4959f2d --- /dev/null +++ b/bundle/direct/autotest/fixtures_test.go @@ -0,0 +1,77 @@ +package autotest + +import ( + "fmt" + "os" + "path/filepath" + "slices" + "strings" + "testing" + + bundleconfig "github.com/databricks/cli/bundle/config" + "github.com/databricks/cli/bundle/direct/dresources" + "github.com/stretchr/testify/require" +) + +// dataDir holds the files fixtures reference by path -- an app's source directory, a +// dashboard's serialized definition, a pipeline's notebook. These are shared test assets rather +// than resource definitions, so they stay where every suite reads them from; the fixtures +// themselves live in testdata/fields and belong to this suite alone. +const dataDir = "../../../acceptance/bundle/invariant/data" + +// drivenTypes returns every resource type the direct engine supports, which is what this suite +// covers: a type with no value library is a failure rather than a line in a report, so adding a +// resource type to the engine and forgetting the catalog cannot go unnoticed. +// +// Permissions and grants are excluded. They are separate plan nodes whose fields describe an ACL +// rather than the resource, they need a parent to attach to, and the suite strips them from every +// fixture. +func drivenTypes(t *testing.T) []string { + var driven []string + for resourceType := range dresources.SupportedResources { + if strings.Contains(resourceType, ".") { + continue + } + path := filepath.Join(fieldsDir, resourceType+".yml") + _, err := os.Stat(path) + require.NoError(t, err, "%s is supported by the engine but has no value library", resourceType) + driven = append(driven, resourceType) + } + + // A library naming no supported type is a leftover -- a rename that missed one side. + entries, err := filepath.Glob(filepath.Join(fieldsDir, "*.yml")) + require.NoError(t, err) + for _, path := range entries { + resourceType := strings.TrimSuffix(filepath.Base(path), ".yml") + _, supported := dresources.SupportedResources[resourceType] + require.True(t, supported, "%s names no supported resource type", path) + } + + slices.Sort(driven) + return driven +} + +// resourceNode returns the node of the resource under test, which a fixture declares under +// resourceKey. A fixture may declare dependencies of other types alongside it, so this is not +// simply the only node in the config. +func resourceNode(root *bundleconfig.Root, resourceType string) (string, error) { + want := "resources." + resourceType + "." + resourceKey + nodes, err := initializedNodes(root) + if err != nil { + return "", err + } + if !slices.Contains(nodes, want) { + return "", fmt.Errorf("%s is not among the declared resources %v", want, nodes) + } + return want, nil +} + +func initializedNodes(root *bundleconfig.Root) ([]string, error) { + var nodes []string + err := forEachResource(root, func(node string, _ any) error { + nodes = append(nodes, node) + return nil + }) + slices.Sort(nodes) + return nodes, err +} diff --git a/bundle/direct/autotest/harness_test.go b/bundle/direct/autotest/harness_test.go new file mode 100644 index 00000000000..23ce0b4b186 --- /dev/null +++ b/bundle/direct/autotest/harness_test.go @@ -0,0 +1,895 @@ +package autotest + +import ( + "context" + "encoding/json" + "fmt" + "io" + "log/slog" + "math" + "os" + "path/filepath" + "reflect" + "strings" + "testing" + "time" + + "github.com/databricks/cli/bundle" + bundleconfig "github.com/databricks/cli/bundle/config" + "github.com/databricks/cli/bundle/deploy" + "github.com/databricks/cli/bundle/deployplan" + "github.com/databricks/cli/bundle/direct" + "github.com/databricks/cli/bundle/direct/dstate" + bundleenv "github.com/databricks/cli/bundle/env" + "github.com/databricks/cli/bundle/phases" + "github.com/databricks/cli/libs/cmdctx" + "github.com/databricks/cli/libs/cmdio" + "github.com/databricks/cli/libs/dbr" + "github.com/databricks/cli/libs/diag" + "github.com/databricks/cli/libs/env" + "github.com/databricks/cli/libs/log" + "github.com/databricks/cli/libs/logdiag" + "github.com/databricks/cli/libs/structs/structaccess" + "github.com/databricks/cli/libs/structs/structpath" + "github.com/databricks/cli/libs/testserver" + "github.com/databricks/databricks-sdk-go" + "github.com/databricks/databricks-sdk-go/service/iam" + "github.com/google/uuid" + "github.com/stretchr/testify/require" + "go.yaml.in/yaml/v3" +) + +// bundleHarness drives the direct engine over one bundle config in-process. Nothing +// here shells out to the CLI or uploads files: the point of this suite is to exercise +// plan and apply for thousands of field permutations, and a `bundle deploy` per +// permutation would be dominated by bundle-file sync. +type bundleHarness struct { + t *testing.T + ctx context.Context + client *databricks.WorkspaceClient + bundle *bundle.Bundle + node string // "resources.schemas.foo" + statePath string + + // unique is the suffix this harness gave its resource, reused for the values of identity + // fields so no two runs against one workspace ask for the same name. + unique string +} + +func newClient(t *testing.T) *databricks.WorkspaceClient { + if isCloud() { + w, err := databricks.NewWorkspaceClient() + require.NoError(t, err) + return w + } + server := testserver.New(tolerantT{t}) + testserver.AddDefaultHandlers(server) + // This suite performs thousands of updates, and an asynchronous resource that reports + // itself in-progress once costs a full second of SDK backoff each time -- serving + // endpoints alone accounted for more wall time than every other resource type + // combined. Waiter behaviour is covered by the acceptance suite, which leaves the + // simulation on. + server.SettleAsyncImmediately() + //exhaustruct:ignore // an SDK config needs only these three fields to reach a fake server + w, err := databricks.NewWorkspaceClient(&databricks.Config{ + Host: server.URL, + Token: "testtoken", + RateLimitPerSecond: math.MaxInt, + }) + require.NoError(t, err) + return w +} + +func isCloud() bool { return os.Getenv("CLOUD_ENV") != "" } + +// workspaceUser is resolved once per resource type and handed to every harness, so +// PopulateCurrentUser never reaches the API. Locally the name is pinned, which keeps +// workspace paths stable between runs. +func workspaceUser(t *testing.T, client *databricks.WorkspaceClient) *iam.User { + if !isCloud() { + return &iam.User{UserName: testUserName} //exhaustruct:ignore + } + user, err := client.CurrentUser.Me(t.Context(), iam.MeRequest{}) //exhaustruct:ignore + require.NoError(t, err) + return user +} + +// testUserName is pinned locally so workspace paths are stable between runs. +const testUserName = "tester@databricks.com" + +// templateVars mirrors the variables acceptance/acceptance_test.go exports to the +// invariant configs. On cloud the harness that launched us provides the real ids. +// uniqueNameVar names the run's own suffix. Every fixture uses it for the resource name, and it +// is the one variable a value library cannot expand for itself. +const uniqueNameVar = "UNIQUE_NAME" + +func templateVars(uniqueName, userName string) map[string]string { + vars := map[string]string{ + uniqueNameVar: uniqueName, + // Matches defaultSparkVersion in acceptance/acceptance_test.go. + "DEFAULT_SPARK_VERSION": "13.3.x-snapshot-scala2.12", + "NODE_TYPE_ID": nodeTypeID(), + "CURRENT_USER_NAME": userName, + "TEST_DEFAULT_WAREHOUSE_ID": testserver.TestDefaultWarehouseId, + "TEST_INSTANCE_POOL_ID": testserver.TestDefaultInstancePoolId, + } + if !isCloud() { + return vars + } + for key := range vars { + // UNIQUE_NAME is this run's own, and CURRENT_USER_NAME is already the resolved + // workspace user, which is more reliable than an env var that may not be set. + if key == uniqueNameVar || key == "CURRENT_USER_NAME" { + continue + } + if value := os.Getenv(key); value != "" { + vars[key] = value + } + } + return vars +} + +// nodeTypeID mirrors getNodeTypeID in acceptance/acceptance_test.go. +func nodeTypeID() string { + switch cloudName() { + case "azure": + return "Standard_E4ds_v5" + case "gcp": + return "n1-standard-4" + default: + return "i3.xlarge" + } +} + +func retryIntervalMs() string { + if isCloud() { + return "2000" + } + return "10" +} + +// maxWaitSeconds caps the per-resource readiness wait. The testserver answers +// immediately, so anything non-zero there only guards against a polling loop. +func maxWaitSeconds() string { + if isCloud() { + return "300" + } + // Against the fake server every legitimate wait finishes at once, so this cap only + // bounds the waits that cannot succeed -- an app delete, for one, which the fake + // server holds in DELETING the way the real API does. + return "2" +} + +// newHarness renders one invariant config into a temp dir and runs the bundle +// initialize phase over it, leaving a fully-resolved config.Root ready to plan. +// newHarness returns an error rather than failing the test: a harness that cannot be +// built is usually the environment (an expired token, a workspace that rejects the +// config), and the run is more useful if that lands in the report as one bad verdict +// than if it aborts thousands of pending observations. +func newHarness(t *testing.T, ctx context.Context, client *databricks.WorkspaceClient, user *iam.User, resourceType, uniqueName string, base any, deps map[string]any, variables map[string]string) (*bundleHarness, error) { + dir := t.TempDir() + if err := copyDir(dataDir, dir); err != nil { + return nil, err + } + + rememberWorkspaceUser(user.UserName) + yml, err := renderBundle(resourceType, uniqueName, user.UserName, base, deps, variables) + if err != nil { + return nil, err + } + if err := os.WriteFile(filepath.Join(dir, "databricks.yml"), []byte(yml), 0o600); err != nil { + return nil, err + } + + // WithoutCancel because a harness outlives the scope that created it in two ways, + // and t.Context() is cancelled at the end of both: a harness rebuilt inside a + // field-level subtest is reused by later fields, and every harness is destroyed from + // t.Cleanup, which runs *after* t.Context() is cancelled. Either way the SDK rate + // limiter refuses the call with "context canceled" -- silently leaking the resource + // in the cleanup case. Values (dbr, logdiag, cmdio, env) still propagate; the test + // binary's own -timeout remains the backstop. + ctx = dbr.MockRuntime(context.WithoutCancel(ctx), dbr.Environment{}) //exhaustruct:ignore + // The CLI logs through libs/log, whose default handler writes to stderr. Under -v that output + // belongs to no subtest, so a harness built by one resource type appears beneath whichever + // parallel type happened to print last -- which is how "Phase: load" ends up under an unrelated + // transition. Routed through t.Log instead: the framework attributes it, and it stays out of a + // passing run's output. + ctx = log.NewContext(ctx, slog.New(slog.NewTextHandler(testWriter{t}, &slog.HandlerOptions{ + Level: slog.LevelInfo, + AddSource: false, + ReplaceAttr: nil, + }))) + ctx = withBundleVars(ctx, variables) + // Thousands of deploys run through here, so cap how long any one of them waits for + // a resource to become ready. Without a cap a resource that never reaches its + // terminal state stalls the whole suite instead of showing up as one bad verdict. + ctx = env.Set(ctx, bundleenv.ResourceMaxWaitVariable, maxWaitSeconds()) + // The engine's default retry interval is 15s. Recreating a resource whose delete is + // asynchronous (apps) retries on purpose, and at 15s a run costs hours. + ctx = env.Set(ctx, bundleenv.RetryIntervalMsVariable, retryIntervalMs()) + ctx = logdiag.InitContext(ctx) + logdiag.SetCollect(ctx, true) + ctx = cmdctx.SetWorkspaceClient(ctx, client) + // Some applies log progress through cmdio (app deployments, for one), which panics + // without an IO in the context. Discard it: the report is the output that matters. + ctx = cmdio.InContext(ctx, cmdio.NewTestIO(strings.NewReader(""), io.Discard, io.Discard)) + + b, err := bundle.Load(ctx, dir) + if err != nil { + return nil, err + } + b.SetWorkpaceClient(client) + b.AutoApprove = true + + phases.LoadDefaultTarget(ctx, b) + if diags := logdiag.FlushCollected(ctx); diags.HasError() { + return nil, fmt.Errorf("loading %s: %s", resourceType, firstError(diags)) + } + + // Pin the user so PopulateCurrentUser makes no API call: a harness is built per + // config and again on every rebuild, which on cloud would be thousands of identical + // CurrentUser.Me requests. + bundle.ApplyFuncContext(ctx, b, func(ctx context.Context, b *bundle.Bundle) { + b.Config.Workspace.CurrentUser = &bundleconfig.User{User: user} //exhaustruct:ignore + }) + + phases.Initialize(ctx, b) + if diags := logdiag.FlushCollected(ctx); diags.HasError() { + return nil, fmt.Errorf("initializing %s: %s", resourceType, firstError(diags)) + } + + // Drop the sub-resource blocks before anything is planned. They are separate plan + // nodes describing an ACL rather than the resource, and they are out of scope here. + // + // Leaving them in also mis-attributes a known bug: a recreate re-keys the parent but + // not its child's state entry, so the child re-proposes the same update forever -- + // which would be reported against whichever field happened to trigger the recreate, + // once per field. That bug has its own coverage in + // acceptance/bundle/resources/volumes/recreate. + if err := stripSubResources(&b.Config, ctx); err != nil { + return nil, err + } + + node, err := resourceNode(&b.Config, resourceType) + if err != nil { + return nil, err + } + + // A real deploy runs deploy.ResourcePathMkdir before creating resources, because an + // alert or dashboard is created inside ${workspace.resource_path} and the backend 404s + // on a missing parent. This suite plans and applies directly, so it has to do the same + // step itself -- otherwise the whole type reports one BASE_ERROR that says nothing about + // any field. + bundle.ApplySeq(ctx, b, deploy.ResourcePathMkdir()) + if diags := logdiag.FlushCollected(ctx); diags.HasError() { + return nil, fmt.Errorf("creating the resource path for %s: %s", resourceType, firstError(diags)) + } + + harness := &bundleHarness{ + t: t, + ctx: ctx, + client: client, + bundle: b, + node: node, + statePath: filepath.Join(dir, "resources.json"), + unique: uniqueName, + } + + return harness, nil +} + +// opCtx returns a context with a fresh diagnostics sink. logdiag's error flag is sticky +// -- FlushCollected empties the collected diagnostics but leaves HasError true -- and +// CalculatePlan short-circuits to a bare "planning failed" whenever that flag is set. A +// per-operation context is what keeps one failed transition from poisoning every plan +// after it. +func (h *bundleHarness) opCtx() (context.Context, func()) { + ctx := logdiag.IsolatedContext(h.ctx) + logdiag.SetCollect(ctx, true) + // A deadline so one operation that cannot finish is recorded as TIMEOUT instead of + // stalling the run. Apps are the reason: renaming one recreates it, and the delete + // half waits for the old name to leave DELETING, which the API does not cap. + return context.WithTimeout(ctx, opTimeout()) +} + +func opTimeout() time.Duration { + if isCloud() { + return 10 * time.Minute + } + return 20 * time.Second +} + +// plan opens the state fresh each time: dstate.Open panics on an already-open +// receiver, and Finalize resets it, so one DeploymentBundle serves one operation. +func (h *bundleHarness) plan() (*pendingApply, *deployplan.Plan, diag.Diagnostics) { + ctx, cancel := h.opCtx() + db := &direct.DeploymentBundle{} //exhaustruct:ignore + if err := db.StateDB.Open(ctx, h.statePath, dstate.WithRecovery(false), dstate.WithWrite(false)); err != nil { + // A pendingApply even here: every caller cancels what plan hands back, and a nil one + // would turn a state-file problem into a panic. + return &pendingApply{ctx: ctx, cancel: cancel, db: db}, nil, diag.FromErr(err) + } + plan, err := db.CalculatePlan(ctx, h.client, &h.bundle.Config) + diags := logdiag.FlushCollected(ctx) + if err != nil && !diags.HasError() { + diags = append(diags, diag.FromErr(err)...) + } + return &pendingApply{ctx: ctx, cancel: cancel, db: db}, plan, diags +} + +// readPlan is plan for a caller that will not apply: the operation context is released +// straight away rather than left holding a ten-minute timer until the test ends. +func (h *bundleHarness) readPlan() (*deployplan.Plan, diag.Diagnostics) { + pending, plan, diags := h.plan() + pending.cancel() + return plan, diags +} + +// pendingApply carries a planned DeploymentBundle together with the context it was +// planned under, so the apply reports into the same isolated diagnostics sink. +type pendingApply struct { + ctx context.Context + cancel func() + db *direct.DeploymentBundle +} + +// apply consumes the DeploymentBundle returned by plan. +func (h *bundleHarness) apply(p *pendingApply, plan *deployplan.Plan) diag.Diagnostics { + defer p.cancel() + ctx, db := p.ctx, p.db + if err := db.StateDB.UpgradeToWrite(); err != nil { + // Finalize even here: the upgrade can fail with a WAL already created, and leaving it + // behind makes every later operation fail with an unexpected-WAL error instead. + _, _ = db.StateDB.Finalize(ctx) + return diag.FromErr(err) + } + if err := db.InitForApply(ctx, h.client, plan); err != nil { + _, _ = db.StateDB.Finalize(ctx) + return diag.FromErr(err) + } + db.Apply(ctx, h.client, plan) + diags := logdiag.FlushCollected(ctx) + if _, err := db.StateDB.Finalize(ctx); err != nil { + diags = append(diags, diag.FromErr(err)...) + } + return diags +} + +// deploy plans and applies, returning the planned action for the node under test. +func (h *bundleHarness) deploy() (deployplan.ActionType, diag.Diagnostics) { + pending, plan, diags := h.plan() + if diags.HasError() { + pending.cancel() + return deployplan.Undefined, diags + } + return nodeAction(plan, h.node), h.apply(pending, plan) +} + +// destroy deletes everything in state. A nil configRoot makes the planner treat +// every state entry as a delete (same call phases.Destroy makes). +func (h *bundleHarness) destroy() diag.Diagnostics { + ctx, cancel := h.opCtx() + db := &direct.DeploymentBundle{} //exhaustruct:ignore + if err := db.StateDB.Open(ctx, h.statePath, dstate.WithRecovery(false), dstate.WithWrite(false)); err != nil { + cancel() + return diag.FromErr(err) + } + plan, err := db.CalculatePlan(ctx, h.client, nil) + if err != nil { + _, _ = db.StateDB.Finalize(ctx) + cancel() + return diag.FromErr(err) + } + return h.apply(&pendingApply{ctx: ctx, cancel: cancel, db: db}, plan) +} + +func nodeAction(plan *deployplan.Plan, node string) deployplan.ActionType { + if plan == nil { + return deployplan.Undefined + } + entry, ok := plan.Plan[node] + if !ok { + return deployplan.Undefined + } + return entry.Action +} + +// hasDrift reports whether any node still has work planned. +func hasDrift(plan *deployplan.Plan) bool { + for _, entry := range plan.Plan { + if entry.Action != deployplan.Skip { + return true + } + } + return false +} + +// edit runs fn over the typed resource under test and syncs the result back into the +// dynamic tree, which is what the planner reads (config.Root.GetResourceConfig). Enter +// and exit around a typed edit is the same contract every bundle mutator follows. +func (h *bundleHarness) edit(fn func(resource any) error) error { + if err := h.bundle.Config.MarkMutatorEntry(h.ctx); err != nil { + return err + } + resource, err := structaccess.GetByString(&h.bundle.Config, h.node) + if err == nil { + err = fn(resource) + } + if exitErr := h.bundle.Config.MarkMutatorExit(h.ctx); exitErr != nil { + return exitErr + } + return err +} + +// resource returns the typed resource under test, e.g. *resources.Schema. +func (h *bundleHarness) resource() (any, error) { + return structaccess.GetByString(&h.bundle.Config, h.node) +} + +// depKey is the bundle key a dependency is declared under: its own resource type, singularised +// only in the sense that it is unique per type, so base references read ${resources.....}. +func depKey(depType string) string { + return depType +} + +// withBundleVars passes a fixture's variables the way a user does, so the config keeps the +// ${var...} reference its format requires instead of a literal resolved before validation. +func withBundleVars(ctx context.Context, variables map[string]string) context.Context { + // fatcontext objects to building a context in a loop, which is about a context that grows per + // request; this one is built once per harness and holds at most a handful of variables. Its + // autofix turns the assignment into a declaration, which drops every variable. + //nolint:fatcontext + for name, value := range variables { + ctx = env.Set(ctx, "BUNDLE_VAR_"+name, value) + } + return ctx +} + +// testWriter sends the CLI's log output to the test that owns it. +type testWriter struct { + t *testing.T +} + +func (w testWriter) Write(p []byte) (int, error) { + w.t.Log(strings.TrimRight(string(p), "\n")) + return len(p), nil +} + +// resourceKey is the name every fixture's resource is declared under. One resource per bundle, +// so the name carries nothing and only has to be stable. +const resourceKey = "foo" + +// renderBundle writes the fixture's base out as a bundle. The value library holds the whole +// resource, so this is the only place a databricks.yml comes from: the base is marshalled back +// to YAML, indented under its resource type, and given the bundle block. +// +// $UNIQUE_NAME survives loadFieldValues unexpanded and is expanded here, because it belongs to +// one deploy: a rebuild gets a new name, while a value library is read once for the whole run. +func renderBundle(resourceType, uniqueName, userName string, base any, deps map[string]any, variables map[string]string) (string, error) { + // Marshalled as one document rather than spliced together as indented text: re-indenting + // someone else's YAML has to reason about block scalars, where a blank line is content. + resources := map[string]any{resourceType: map[string]any{resourceKey: base}} + // A dependency is declared under its own type with its own key, so base can reference it as + // ${resources...} the way a bundle normally would. The key is the type's + // own name, which keeps a reference readable and cannot collide with the resource under test. + for depType, body := range deps { + if depType == resourceType { + return "", fmt.Errorf("dep %s is the resource type under test", depType) + } + resources[depType] = map[string]any{depKey(depType): body} + } + + document := map[string]any{ + "bundle": map[string]any{"name": "test-bundle-$UNIQUE_NAME"}, + "resources": resources, + } + // Declared without a value: the value comes from the environment below, since a default here + // would be resolved before the config-format validations run. + if len(variables) > 0 { + declared := map[string]any{} + for name := range variables { + declared[name] = map[string]any{"description": "set by the field catalog"} + } + document["variables"] = declared + } + + body, err := yaml.Marshal(document) + if err != nil { + return "", err + } + + vars := templateVars(uniqueName, userName) + var missing string + yml := os.Expand(string(body), func(key string) string { + if value, ok := vars[key]; ok { + return value + } + // A bundle's own interpolation shares this syntax -- ${var.secret_value}, + // ${resources.postgres_projects.x.name} -- and belongs to the config, not to the suite. + // Told apart by the dot: every variable the suite provides is a bare upper-case name, and + // every bundle reference is dotted. Restored verbatim so the config still carries it. + if strings.Contains(key, ".") { + return "${" + key + "}" + } + // Expanding an unknown variable to "" turns a required field into null, and the failure + // then surfaces as a confusing validation error much later. + missing = key + return "" + }) + if missing != "" { + return "", fmt.Errorf("%s.yml uses $%s, which this suite does not provide", resourceType, missing) + } + return yml, nil +} + +func copyDir(src, dst string) error { + return filepath.WalkDir(src, func(path string, d os.DirEntry, err error) error { + if err != nil { + return err + } + rel, err := filepath.Rel(src, path) + if err != nil { + return err + } + target := filepath.Join(dst, rel) + if d.IsDir() { + return os.MkdirAll(target, 0o700) + } + data, err := os.ReadFile(path) + if err != nil { + return err + } + return os.WriteFile(target, data, 0o600) + }) +} + +// setField writes a value into the resource under test, or removes the field when the +// value is nil -- which is how "absent" is expressed, since a Go struct has no absent. +func (h *bundleHarness) setField(path string, value any) error { + node, err := structpath.ParsePath(path) + if err != nil { + return err + } + return h.edit(func(resource any) error { + return setNode(resource, node, substituteUnique(value, h.unique)) + }) +} + +// substituteUnique resolves the placeholder in an identity field's value against the suffix +// this harness's resource actually carries. A rebuild changes that suffix, and the resource it +// replaced is still alive under the old one until the test ends. +func substituteUnique(value any, unique string) any { + text, ok := value.(string) + if !ok || !strings.Contains(text, uniqueMarker) { + return value + } + return strings.ReplaceAll(text, uniqueMarker, unique) +} + +// setNode is setField without the bundle: the edit itself, for the unit tests. +func setNode(resource any, node *structpath.PathNode, value any) error { + if value == nil { + return removeField(resource, node) + } + if err := ensurePath(resource, node); err != nil { + return err + } + converted, err := coerce(resource, node, value) + if err != nil { + return err + } + return structaccess.Set(resource, node, converted) +} + +// removeField makes a field absent. For a struct field that means the zero value with the +// field dropped from ForceSendFields, which is what structaccess.Set does with nil; a map +// entry and a slice element have to go instead of being zeroed. +func removeField(resource any, node *structpath.PathNode) error { + parent := node.Parent() + // An error here means the path does not lead anywhere in this config -- a nil object on + // the way down -- which is the same conclusion as an absent parent: nothing to remove. + container, err := structaccess.Get(resource, parent) + if err != nil || isAbsent(container) { + return nil //nolint:nilerr // an unreachable parent means the field is already absent + } + + if index, isIndex := node.Index(); isIndex { + return removeIndex(resource, parent, container, index) + } + if key, hasKey := node.StringKey(); hasKey { + if value := reflect.ValueOf(container); value.Kind() == reflect.Map { + value.SetMapIndex(reflect.ValueOf(key).Convert(value.Type().Key()), reflect.Value{}) + return nil + } + } + return structaccess.Set(resource, node, nil) +} + +// removeIndex drops the last element of a slice, which is the only index that can be made +// absent: removing any earlier one shifts its successor into the same path, so the field +// would still be there holding a different value and the transition would be recorded as a +// move from absent that never happened. +func removeIndex(resource any, parent *structpath.PathNode, container any, index int) error { + value := reflect.ValueOf(container) + if value.Kind() != reflect.Slice { + return fmt.Errorf("cannot remove %s[%d]: parent is %s", parent, index, value.Kind()) + } + if index < 0 || index >= value.Len() { + return nil + } + if index != value.Len()-1 { + return fmt.Errorf("cannot make %s[%d] absent: %s[%d] would shift into it", parent, index, parent, index+1) + } + // A fresh slice, not a re-slice: appending into the original backing array would also + // change any copy of the slice the suite is holding on to. + trimmed := reflect.MakeSlice(value.Type(), 0, index) + trimmed = reflect.AppendSlice(trimmed, value.Slice(0, index)) + return structaccess.Set(resource, parent, trimmed.Interface()) +} + +// ensurePath makes the places a path passes through exist, which is what writing the same +// nested key into databricks.yml would do: an absent object is allocated, and a list too +// short for an index is grown. It works top down, so the type of each missing level comes +// from the level above it, which by then exists. +func ensurePath(resource any, node *structpath.PathNode) error { + nodes := node.AsSlice() + for i, prefix := range nodes { + if index, isIndex := prefix.Index(); isIndex { + if err := growSlice(resource, prefix.Parent(), index); err != nil { + return err + } + continue + } + if i == len(nodes)-1 { + // The leaf is what the caller is about to write. + break + } + if _, nextIsIndex := nodes[i+1].Index(); nextIsIndex { + // growSlice creates the list itself, at the length the index needs. + continue + } + if current, err := structaccess.Get(resource, prefix); err == nil && !isAbsent(current) { + continue + } + typ, err := typeAt(resource, prefix) + if err != nil { + return err + } + empty, err := emptyValue(typ) + if err != nil { + return fmt.Errorf("cannot create %s: %w", prefix, err) + } + if err := structaccess.Set(resource, prefix, empty); err != nil { + return err + } + } + return nil +} + +// growSlice extends a list until an index is addressable, appending zero elements. The +// index is either the one the config already had, or the first of a list this suite is +// putting back after removing its only entry. +func growSlice(resource any, parent *structpath.PathNode, index int) error { + container, err := structaccess.Get(resource, parent) + if err != nil { + return err + } + // An empty list reads back as absent -- omitempty hides it -- so the type comes from the + // declaration rather than from the value. + typ, err := typeAt(resource, parent) + if err != nil { + return err + } + if typ.Kind() != reflect.Slice { + return fmt.Errorf("cannot index %s: %s", parent, typ.Kind()) + } + + grown := reflect.MakeSlice(typ, index+1, index+1) + if !isAbsent(container) { + value := reflect.ValueOf(container) + if value.Len() > index { + return nil + } + reflect.Copy(grown, value) + } + return structaccess.Set(resource, parent, grown.Interface()) +} + +// typeAt returns the declared type of the field at node. The parent has to exist, which +// ensurePath guarantees by allocating top down. +func typeAt(resource any, node *structpath.PathNode) (reflect.Type, error) { + parent := node.Parent() + typ := reflect.TypeOf(resource) + if !parent.IsRoot() { + value, err := structaccess.Get(resource, parent) + if err != nil { + return nil, err + } + if isAbsent(value) { + return nil, fmt.Errorf("%s is absent", parent) + } + typ = reflect.TypeOf(value) + } + for typ.Kind() == reflect.Pointer { + typ = typ.Elem() + } + + if _, isIndex := node.Index(); isIndex { + if typ.Kind() != reflect.Slice && typ.Kind() != reflect.Array { + return nil, fmt.Errorf("cannot index %s", typ.Kind()) + } + return typ.Elem(), nil + } + key, ok := node.StringKey() + if !ok { + return nil, fmt.Errorf("unsupported path %s", node) + } + switch typ.Kind() { + case reflect.Map: + return typ.Elem(), nil + case reflect.Struct: + field, _, ok := structaccess.FindStructFieldByKeyType(typ, key) + if !ok { + return nil, fmt.Errorf("field %q not found in %s", key, typ) + } + return field.Type, nil + default: + return nil, fmt.Errorf("cannot access %q on %s", key, typ.Kind()) + } +} + +// coerce converts a value from the value library into the field's own type. A scalar is +// left to structaccess, which converts between the numeric and string kinds; a map or list +// has to go through the type's own JSON unmarshaller, since []any is assignable to nothing +// and an SDK struct or enum decodes itself. +func coerce(resource any, node *structpath.PathNode, value any) (any, error) { + switch reflect.ValueOf(value).Kind() { + case reflect.Map, reflect.Slice: + default: + return value, nil + } + + typ, err := typeAt(resource, node) + if err != nil { + return nil, err + } + for typ.Kind() == reflect.Pointer { + typ = typ.Elem() + } + if reflect.TypeOf(value).AssignableTo(typ) { + // Already the field's own type: a container value read back off the resource. + return value, nil + } + + body, err := json.Marshal(value) + if err != nil { + return nil, err + } + converted := reflect.New(typ) + if err := json.Unmarshal(body, converted.Interface()); err != nil { + return nil, fmt.Errorf("cannot use %v as %s: %w", value, typ, err) + } + return converted.Elem().Interface(), nil +} + +// emptyValue builds the empty container to put at an absent level of a path. +func emptyValue(typ reflect.Type) (any, error) { + switch typ.Kind() { + case reflect.Pointer: + return reflect.New(typ.Elem()).Interface(), nil + case reflect.Map: + return reflect.MakeMap(typ).Interface(), nil + case reflect.Slice: + return reflect.MakeSlice(typ, 0, 0).Interface(), nil + case reflect.Struct: + return reflect.New(typ).Elem().Interface(), nil + default: + return nil, fmt.Errorf("%s is not a container", typ) + } +} + +// isAbsent reports whether a value read out of the config is not there at all. A nil +// pointer or map reads back as a typed nil, which is not a nil interface. +func isAbsent(value any) bool { + if value == nil { + return true + } + switch v := reflect.ValueOf(value); v.Kind() { + case reflect.Pointer, reflect.Map, reflect.Slice, reflect.Interface: + return v.IsNil() + default: + return false + } +} + +// snapshot serializes the resource under test, so a field can be put back to what the +// base config had once its transitions are done. JSON is the same representation the API +// uses, and the SDK's own unmarshaller restores ForceSendFields, so an absent field comes +// back absent rather than as an explicit empty value. +func (h *bundleHarness) snapshot() []byte { + resource, err := h.resource() + require.NoError(h.t, err) + body, err := json.Marshal(resource) + require.NoError(h.t, err) + return body +} + +// restore puts the resource back to a snapshot taken earlier. +func (h *bundleHarness) restore(snapshot []byte) { + _ = h.edit(func(resource any) error { + value := reflect.ValueOf(resource).Elem() + value.Set(reflect.Zero(value.Type())) + return json.Unmarshal(snapshot, resource) + }) +} + +// subResourceKinds are the child plan nodes this suite removes from every config; see +// stripSubResources. +var subResourceKinds = []string{"permissions", "grants"} + +// stripSubResources drops the permissions and grants blocks from every resource of an +// initialized config. +func stripSubResources(root *bundleconfig.Root, ctx context.Context) error { + if err := root.MarkMutatorEntry(ctx); err != nil { + return err + } + err := forEachResource(root, func(_ string, resource any) error { + typ := reflect.TypeOf(resource) + for typ.Kind() == reflect.Pointer { + typ = typ.Elem() + } + for _, kind := range subResourceKinds { + if _, _, ok := structaccess.FindStructFieldByKeyType(typ, kind); !ok { + continue + } + if err := structaccess.SetByString(resource, kind, nil); err != nil { + return err + } + } + return nil + }) + if exitErr := root.MarkMutatorExit(ctx); exitErr != nil { + return exitErr + } + return err +} + +// forEachResource visits every declared resource with its config node key. +func forEachResource(root *bundleconfig.Root, fn func(node string, resource any) error) error { + for _, group := range root.Resources.AllResources() { + for key, resource := range group.Resources { + node := "resources." + group.Description.PluralName + "." + key + if err := fn(node, resource); err != nil { + return fmt.Errorf("%s: %w", node, err) + } + } + } + return nil +} + +// uniqueName keeps cloud resources from colliding between runs and between the +// parallel subtests of one run. +func uniqueName() string { + return "f" + strings.ToLower(strings.ReplaceAll(uuid.NewString(), "-", ""))[:20] +} + +// converged deploys the current config and reports whether a following plan is clean. +func (h *bundleHarness) converged() bool { + if _, diags := h.deploy(); diags.HasError() { + return false + } + plan, diags := h.readPlan() + return !diags.HasError() && plan != nil && !hasDrift(plan) +} + +// tolerantT wraps a *testing.T for the fake server. This suite deliberately drives the +// engine with odd field values, and some of them make it issue a request the fake +// server has no route for -- an empty name lands on "GET /api/2.0/serving-endpoints/". +// The fake server reports that with Errorf, which would fail the whole run; here it is +// just another observation, so errors are downgraded to log lines. +type tolerantT struct { + *testing.T +} + +func (tolerantT) Error(args ...any) {} +func (tolerantT) Errorf(format string, args ...any) {} diff --git a/bundle/direct/autotest/output/alerts.txt b/bundle/direct/autotest/output/alerts.txt new file mode 100644 index 00000000000..3f4391b2425 --- /dev/null +++ b/bundle/direct/autotest/output/alerts.txt @@ -0,0 +1,15 @@ +# field | from | to | verdict | detail +# +# How the direct engine handles every field of this resource type: each row is one move of one field +# between two values, "absent" included. An outcome the engine got wrong is recorded here rather than +# failed, so this file is a description of current behaviour -- what fails the test is a verdict +# changing. Passing rows are counted in the summary and listed in logs/.full.txt, along with +# the evidence behind each finding and what could not be covered. +# +# Regenerate with ./task test-update-fields. See bundle/direct/autotest/README.md for the verdicts. + + +=== summary +OK 26 +NOT_OBSERVABLE 6 +SKIPPED 17 diff --git a/bundle/direct/autotest/output/apps.txt b/bundle/direct/autotest/output/apps.txt new file mode 100644 index 00000000000..fc6e31dfed8 --- /dev/null +++ b/bundle/direct/autotest/output/apps.txt @@ -0,0 +1,109 @@ +# field | from | to | verdict | detail +# +# How the direct engine handles every field of this resource type: each row is one move of one field +# between two values, "absent" included. An outcome the engine got wrong is recorded here rather than +# failed, so this file is a description of current behaviour -- what fails the test is a verdict +# changing. Passing rows are counted in the summary and listed in logs/.full.txt, along with +# the evidence behind each finding and what could not be covered. +# +# Regenerate with ./task test-update-fields. See bundle/direct/autotest/README.md for the verdicts. + +config.command absent len1 START_NOT_REACHED no active deployment +config.command absent len2 START_NOT_REACHED no active deployment +config.command len1 absent START_NOT_REACHED no active deployment +config.command len1 len2 START_NOT_REACHED no active deployment +config.command len2 absent START_NOT_REACHED no active deployment +config.command len2 len1 START_NOT_REACHED no active deployment +config.command[0] absent x UNSETTABLE cannot make config.command[0] absent: config.command[1] would shift into it +config.command[0] absent y UNSETTABLE cannot make config.command[0] absent: config.command[1] would shift into it +config.command[0] x absent START_NOT_REACHED no active deployment +config.command[0] x y START_NOT_REACHED no active deployment +config.command[0] y absent START_NOT_REACHED no active deployment +config.command[0] y x START_NOT_REACHED no active deployment +config.command[1] absent x START_NOT_REACHED no active deployment +config.command[1] absent y START_NOT_REACHED no active deployment +config.command[1] x absent START_NOT_REACHED no active deployment +config.command[1] x y START_NOT_REACHED no active deployment +config.command[1] y absent START_NOT_REACHED no active deployment +config.command[1] y x START_NOT_REACHED no active deployment +config.env absent len0 START_NOT_REACHED no active deployment +config.env absent len1 START_NOT_REACHED no active deployment +config.env len0 absent START_NOT_REACHED no active deployment +config.env len0 len1 START_NOT_REACHED no active deployment +config.env len1 absent START_NOT_REACHED no active deployment +config.env len1 len0 START_NOT_REACHED no active deployment +config.env[0].name x y START_NOT_REACHED no active deployment +config.env[0].name y x START_NOT_REACHED no active deployment +config.env[0].value absent x START_NOT_REACHED no active deployment +config.env[0].value absent y START_NOT_REACHED no active deployment +config.env[0].value x absent START_NOT_REACHED no active deployment +config.env[0].value x y START_NOT_REACHED no active deployment +config.env[0].value y absent START_NOT_REACHED no active deployment +config.env[0].value y x START_NOT_REACHED no active deployment +config.env[0].value_from absent x START_NOT_REACHED no active deployment +config.env[0].value_from absent y START_NOT_REACHED no active deployment +config.env[0].value_from x absent START_NOT_REACHED no active deployment +config.env[0].value_from x y START_NOT_REACHED no active deployment +config.env[0].value_from y absent START_NOT_REACHED no active deployment +config.env[0].value_from y x START_NOT_REACHED no active deployment +description x absent UPDATE_IGNORED +description y absent UPDATE_IGNORED +git_source.branch absent x SUPPRESSED no active deployment +git_source.branch absent y SUPPRESSED no active deployment +git_source.branch x absent START_NOT_REACHED no active deployment +git_source.branch x y START_NOT_REACHED no active deployment +git_source.branch y absent START_NOT_REACHED no active deployment +git_source.branch y x START_NOT_REACHED no active deployment +git_source.commit absent x SUPPRESSED no active deployment +git_source.commit absent y SUPPRESSED no active deployment +git_source.commit x absent START_NOT_REACHED no active deployment +git_source.commit x y START_NOT_REACHED no active deployment +git_source.commit y absent START_NOT_REACHED no active deployment +git_source.commit y x START_NOT_REACHED no active deployment +git_source.git_repository.auto_deploy absent false SUPPRESSED no active deployment +git_source.git_repository.auto_deploy absent true SUPPRESSED no active deployment +git_source.git_repository.auto_deploy false absent START_NOT_REACHED no active deployment +git_source.git_repository.auto_deploy false true START_NOT_REACHED no active deployment +git_source.git_repository.auto_deploy true absent START_NOT_REACHED no active deployment +git_source.git_repository.auto_deploy true false START_NOT_REACHED no active deployment +git_source.git_repository.caller_credential_id 1 2 START_NOT_REACHED no active deployment +git_source.git_repository.caller_credential_id 1 absent START_NOT_REACHED no active deployment +git_source.git_repository.caller_credential_id 2 1 START_NOT_REACHED no active deployment +git_source.git_repository.caller_credential_id 2 absent START_NOT_REACHED no active deployment +git_source.git_repository.caller_credential_id absent 1 SUPPRESSED no active deployment +git_source.git_repository.caller_credential_id absent 2 SUPPRESSED no active deployment +git_source.resolved_commit absent x SUPPRESSED no active deployment +git_source.resolved_commit absent y SUPPRESSED no active deployment +git_source.resolved_commit x absent START_NOT_REACHED no active deployment +git_source.resolved_commit x y START_NOT_REACHED no active deployment +git_source.resolved_commit y absent START_NOT_REACHED no active deployment +git_source.resolved_commit y x START_NOT_REACHED no active deployment +git_source.source_code_path absent x SUPPRESSED no active deployment +git_source.source_code_path absent y SUPPRESSED no active deployment +git_source.source_code_path x absent START_NOT_REACHED no active deployment +git_source.source_code_path x y START_NOT_REACHED no active deployment +git_source.source_code_path y absent START_NOT_REACHED no active deployment +git_source.source_code_path y x START_NOT_REACHED no active deployment +git_source.tag absent x SUPPRESSED no active deployment +git_source.tag absent y SUPPRESSED no active deployment +git_source.tag x absent START_NOT_REACHED no active deployment +git_source.tag x y START_NOT_REACHED no active deployment +git_source.tag y absent START_NOT_REACHED no active deployment +git_source.tag y x START_NOT_REACHED no active deployment +source_code_path absent x-UNIQUE SUPPRESSED no active deployment +source_code_path absent y-UNIQUE SUPPRESSED no active deployment +source_code_path x-UNIQUE absent START_NOT_REACHED no active deployment +source_code_path x-UNIQUE y-UNIQUE START_NOT_REACHED no active deployment +source_code_path y-UNIQUE absent START_NOT_REACHED no active deployment +source_code_path y-UNIQUE x-UNIQUE START_NOT_REACHED no active deployment +user_api_scopes len1 absent UPDATE_IGNORED +user_api_scopes len2 absent UPDATE_IGNORED + +=== summary +OK 20 +OK_RECREATE 2 +SUPPRESSED 16 +UPDATE_IGNORED 4 +UNSETTABLE 2 +SKIPPED 94 +START_NOT_REACHED 68 diff --git a/bundle/direct/autotest/output/catalogs.txt b/bundle/direct/autotest/output/catalogs.txt new file mode 100644 index 00000000000..df334748455 --- /dev/null +++ b/bundle/direct/autotest/output/catalogs.txt @@ -0,0 +1,23 @@ +# field | from | to | verdict | detail +# +# How the direct engine handles every field of this resource type: each row is one move of one field +# between two values, "absent" included. An outcome the engine got wrong is recorded here rather than +# failed, so this file is a description of current behaviour -- what fails the test is a verdict +# changing. Passing rows are counted in the summary and listed in logs/.full.txt, along with +# the evidence behind each finding and what could not be covered. +# +# Regenerate with ./task test-update-fields. See bundle/direct/autotest/README.md for the verdicts. + +custom_max_retention_hours 168 absent UPDATE_IGNORED +custom_max_retention_hours 720 absent UPDATE_IGNORED +properties keys1 absent UPDATE_IGNORED +properties keys1 keys0 UPDATE_IGNORED +properties['team'] x absent UPDATE_IGNORED +properties['team'] y absent UPDATE_IGNORED + +=== summary +OK 18 +SUPPRESSED 1 +NOT_OBSERVABLE 1 +UPDATE_IGNORED 6 +SKIPPED 8 diff --git a/bundle/direct/autotest/output/cluster_policies.txt b/bundle/direct/autotest/output/cluster_policies.txt new file mode 100644 index 00000000000..62baeca361d --- /dev/null +++ b/bundle/direct/autotest/output/cluster_policies.txt @@ -0,0 +1,30 @@ +# field | from | to | verdict | detail +# +# How the direct engine handles every field of this resource type: each row is one move of one field +# between two values, "absent" included. An outcome the engine got wrong is recorded here rather than +# failed, so this file is a description of current behaviour -- what fails the test is a verdict +# changing. Passing rows are counted in the summary and listed in logs/.full.txt, along with +# the evidence behind each finding and what could not be covered. +# +# Regenerate with ./task test-update-fields. See bundle/direct/autotest/README.md for the verdicts. + +definition absent v1 BASE_ERROR cannot update resources.cluster_policies.foo: updating id=[ID]: 'definition' must be supplied. (400 INVALID_PARAMETER_VALUE) +definition absent v2 BASE_ERROR cannot update resources.cluster_policies.foo: updating id=[ID]: 'definition' must be supplied. (400 INVALID_PARAMETER_VALUE) +definition v1 absent BACKEND_ERROR cannot update resources.cluster_policies.foo: updating id=[ID]: 'definition' must be supplied. (400 INVALID_PARAMETER_VALUE) +definition v2 absent BACKEND_ERROR cannot update resources.cluster_policies.foo: updating id=[ID]: 'definition' must be supplied. (400 INVALID_PARAMETER_VALUE) +libraries[0].jar absent dbfs:/FileStore/a.jar BASE_ERROR cannot update resources.cluster_policies.foo: updating id=[ID]: Unknown library (400 INVALID_PARAMETER_VALUE) +libraries[0].jar absent dbfs:/FileStore/b.jar BASE_ERROR cannot update resources.cluster_policies.foo: updating id=[ID]: Unknown library (400 INVALID_PARAMETER_VALUE) +libraries[0].jar dbfs:/FileStore/a.jar absent BACKEND_ERROR cannot update resources.cluster_policies.foo: updating id=[ID]: Unknown library (400 INVALID_PARAMETER_VALUE) +libraries[0].jar dbfs:/FileStore/b.jar absent BACKEND_ERROR cannot update resources.cluster_policies.foo: updating id=[ID]: Unknown library (400 INVALID_PARAMETER_VALUE) +name absent x-UNIQUE BASE_ERROR cannot update resources.cluster_policies.foo: updating id=[ID]: 'name' must be supplied. (400 INVALID_PARAMETER_VALUE) +name absent y-UNIQUE BASE_ERROR cannot update resources.cluster_policies.foo: updating id=[ID]: 'name' must be supplied. (400 INVALID_PARAMETER_VALUE) +name x-UNIQUE absent BACKEND_ERROR cannot update resources.cluster_policies.foo: updating id=[ID]: 'name' must be supplied. (400 INVALID_PARAMETER_VALUE) +name y-UNIQUE absent BACKEND_ERROR cannot update resources.cluster_policies.foo: updating id=[ID]: 'name' must be supplied. (400 INVALID_PARAMETER_VALUE) + +=== summary +OK 22 +SUPPRESSED 1 +NOT_OBSERVABLE 1 +BACKEND_ERROR 6 +SKIPPED 10 +BASE_ERROR 6 diff --git a/bundle/direct/autotest/output/clusters.txt b/bundle/direct/autotest/output/clusters.txt new file mode 100644 index 00000000000..60924c8e2a2 --- /dev/null +++ b/bundle/direct/autotest/output/clusters.txt @@ -0,0 +1,16 @@ +# field | from | to | verdict | detail +# +# How the direct engine handles every field of this resource type: each row is one move of one field +# between two values, "absent" included. An outcome the engine got wrong is recorded here rather than +# failed, so this file is a description of current behaviour -- what fails the test is a verdict +# changing. Passing rows are counted in the summary and listed in logs/.full.txt, along with +# the evidence behind each finding and what could not be covered. +# +# Regenerate with ./task test-update-fields. See bundle/direct/autotest/README.md for the verdicts. + + +=== summary +OK 367 +SUPPRESSED 18 +NOT_OBSERVABLE 7 +SKIPPED 13 diff --git a/bundle/direct/autotest/output/dashboards.txt b/bundle/direct/autotest/output/dashboards.txt new file mode 100644 index 00000000000..7c041108d98 --- /dev/null +++ b/bundle/direct/autotest/output/dashboards.txt @@ -0,0 +1,22 @@ +# field | from | to | verdict | detail +# +# How the direct engine handles every field of this resource type: each row is one move of one field +# between two values, "absent" included. An outcome the engine got wrong is recorded here rather than +# failed, so this file is a description of current behaviour -- what fails the test is a verdict +# changing. Passing rows are counted in the summary and listed in logs/.full.txt, along with +# the evidence behind each finding and what could not be covered. +# +# Regenerate with ./task test-update-fields. See bundle/direct/autotest/README.md for the verdicts. + +display_name absent x-UNIQUE START_NOT_REACHED cannot create the resource with this starting value: cannot create resources.dashboards.foo: [Request Validation] display name cannot be empty (400 INVALID_PARAMETER_VALUE) +display_name absent y-UNIQUE START_NOT_REACHED cannot create the resource with this starting value: cannot create resources.dashboards.foo: [Request Validation] display name cannot be empty (400 INVALID_PARAMETER_VALUE) +display_name x-UNIQUE absent UPDATE_IGNORED +display_name y-UNIQUE absent UPDATE_IGNORED + +=== summary +OK 6 +SUPPRESSED 2 +NOT_OBSERVABLE 6 +UPDATE_IGNORED 2 +SKIPPED 11 +START_NOT_REACHED 2 diff --git a/bundle/direct/autotest/output/database_catalogs.txt b/bundle/direct/autotest/output/database_catalogs.txt new file mode 100644 index 00000000000..b40b27a0b9b --- /dev/null +++ b/bundle/direct/autotest/output/database_catalogs.txt @@ -0,0 +1,19 @@ +# field | from | to | verdict | detail +# +# How the direct engine handles every field of this resource type: each row is one move of one field +# between two values, "absent" included. An outcome the engine got wrong is recorded here rather than +# failed, so this file is a description of current behaviour -- what fails the test is a verdict +# changing. Passing rows are counted in the summary and listed in logs/.full.txt, along with +# the evidence behind each finding and what could not be covered. +# +# Regenerate with ./task test-update-fields. See bundle/direct/autotest/README.md for the verdicts. + +database_instance_name x y BASE_ERROR cannot recreate resources.database_catalogs.foo: database instance with name 'x' not found (404 UNKNOWN) +database_instance_name y x BASE_ERROR cannot recreate resources.database_catalogs.foo: database instance with name 'y' not found (404 UNKNOWN) + +=== summary +OK_RECREATE 8 +SUPPRESSED 1 +NOT_OBSERVABLE 1 +SKIPPED 2 +BASE_ERROR 2 diff --git a/bundle/direct/autotest/output/database_instances.txt b/bundle/direct/autotest/output/database_instances.txt new file mode 100644 index 00000000000..b072a60c536 --- /dev/null +++ b/bundle/direct/autotest/output/database_instances.txt @@ -0,0 +1,70 @@ +# field | from | to | verdict | detail +# +# How the direct engine handles every field of this resource type: each row is one move of one field +# between two values, "absent" included. An outcome the engine got wrong is recorded here rather than +# failed, so this file is a description of current behaviour -- what fails the test is a verdict +# changing. Passing rows are counted in the summary and listed in logs/.full.txt, along with +# the evidence behind each finding and what could not be covered. +# +# Regenerate with ./task test-update-fields. See bundle/direct/autotest/README.md for the verdicts. + +capacity absent x UPDATE_IGNORED +capacity absent y UPDATE_IGNORED +capacity x y POST_DEPLOY_DRIFT capacity +capacity y x POST_DEPLOY_DRIFT capacity +custom_tags absent len1 COLLATERAL_DRIFT capacity +custom_tags len0 len1 COLLATERAL_DRIFT capacity +custom_tags len1 absent COLLATERAL_DRIFT capacity +custom_tags len1 len0 COLLATERAL_DRIFT capacity +custom_tags[0].key absent x COLLATERAL_DRIFT capacity +custom_tags[0].key absent y COLLATERAL_DRIFT capacity +custom_tags[0].key x absent COLLATERAL_DRIFT capacity +custom_tags[0].key x y COLLATERAL_DRIFT capacity +custom_tags[0].key y absent COLLATERAL_DRIFT capacity +custom_tags[0].key y x COLLATERAL_DRIFT capacity +custom_tags[0].value absent x COLLATERAL_DRIFT capacity +custom_tags[0].value absent y COLLATERAL_DRIFT capacity +custom_tags[0].value x absent COLLATERAL_DRIFT capacity +custom_tags[0].value x y COLLATERAL_DRIFT capacity +custom_tags[0].value y absent COLLATERAL_DRIFT capacity +custom_tags[0].value y x COLLATERAL_DRIFT capacity +enable_pg_native_login absent true COLLATERAL_DRIFT capacity +enable_pg_native_login false true COLLATERAL_DRIFT capacity +enable_pg_native_login true absent COLLATERAL_DRIFT capacity +enable_pg_native_login true false COLLATERAL_DRIFT capacity +enable_readable_secondaries absent true COLLATERAL_DRIFT capacity +enable_readable_secondaries false true COLLATERAL_DRIFT capacity +enable_readable_secondaries true absent COLLATERAL_DRIFT capacity +enable_readable_secondaries true false COLLATERAL_DRIFT capacity +node_count 1 2 COLLATERAL_DRIFT capacity +node_count 1 absent COLLATERAL_DRIFT capacity +node_count 2 1 COLLATERAL_DRIFT capacity +node_count 2 absent COLLATERAL_DRIFT capacity +node_count absent 1 COLLATERAL_DRIFT capacity +node_count absent 2 COLLATERAL_DRIFT capacity +retention_window_in_days 1 2 COLLATERAL_DRIFT capacity +retention_window_in_days 1 absent COLLATERAL_DRIFT capacity +retention_window_in_days 2 1 COLLATERAL_DRIFT capacity +retention_window_in_days 2 absent COLLATERAL_DRIFT capacity +retention_window_in_days absent 1 COLLATERAL_DRIFT capacity +retention_window_in_days absent 2 COLLATERAL_DRIFT capacity +stopped absent true COLLATERAL_DRIFT capacity +stopped false true COLLATERAL_DRIFT capacity +stopped true absent COLLATERAL_DRIFT capacity +stopped true false COLLATERAL_DRIFT capacity +usage_policy_id absent x COLLATERAL_DRIFT capacity +usage_policy_id absent y COLLATERAL_DRIFT capacity +usage_policy_id x absent COLLATERAL_DRIFT capacity +usage_policy_id x y COLLATERAL_DRIFT capacity +usage_policy_id y absent COLLATERAL_DRIFT capacity +usage_policy_id y x COLLATERAL_DRIFT capacity + +=== summary +OK 2 +OK_RECREATE 32 +SUPPRESSED 6 +NOT_OBSERVABLE 2 +UPDATE_IGNORED 2 +POST_DEPLOY_DRIFT 2 +COLLATERAL_DRIFT 46 +SKIPPED 24 diff --git a/bundle/direct/autotest/output/experiments.txt b/bundle/direct/autotest/output/experiments.txt new file mode 100644 index 00000000000..0d42a8773df --- /dev/null +++ b/bundle/direct/autotest/output/experiments.txt @@ -0,0 +1,16 @@ +# field | from | to | verdict | detail +# +# How the direct engine handles every field of this resource type: each row is one move of one field +# between two values, "absent" included. An outcome the engine got wrong is recorded here rather than +# failed, so this file is a description of current behaviour -- what fails the test is a verdict +# changing. Passing rows are counted in the summary and listed in logs/.full.txt, along with +# the evidence behind each finding and what could not be covered. +# +# Regenerate with ./task test-update-fields. See bundle/direct/autotest/README.md for the verdicts. + + +=== summary +OK 2 +OK_RECREATE 12 +OK_INERT 18 +SKIPPED 2 diff --git a/bundle/direct/autotest/output/external_locations.txt b/bundle/direct/autotest/output/external_locations.txt new file mode 100644 index 00000000000..99e0d260f16 --- /dev/null +++ b/bundle/direct/autotest/output/external_locations.txt @@ -0,0 +1,17 @@ +# field | from | to | verdict | detail +# +# How the direct engine handles every field of this resource type: each row is one move of one field +# between two values, "absent" included. An outcome the engine got wrong is recorded here rather than +# failed, so this file is a description of current behaviour -- what fails the test is a verdict +# changing. Passing rows are counted in the summary and listed in logs/.full.txt, along with +# the evidence behind each finding and what could not be covered. +# +# Regenerate with ./task test-update-fields. See bundle/direct/autotest/README.md for the verdicts. + + +=== summary +OK 26 +OK_RECREATE 110 +SUPPRESSED 5 +NOT_OBSERVABLE 3 +SKIPPED 18 diff --git a/bundle/direct/autotest/output/genie_spaces.txt b/bundle/direct/autotest/output/genie_spaces.txt new file mode 100644 index 00000000000..d0ace3bf0f8 --- /dev/null +++ b/bundle/direct/autotest/output/genie_spaces.txt @@ -0,0 +1,20 @@ +# field | from | to | verdict | detail +# +# How the direct engine handles every field of this resource type: each row is one move of one field +# between two values, "absent" included. An outcome the engine got wrong is recorded here rather than +# failed, so this file is a description of current behaviour -- what fails the test is a verdict +# changing. Passing rows are counted in the summary and listed in logs/.full.txt, along with +# the evidence behind each finding and what could not be covered. +# +# Regenerate with ./task test-update-fields. See bundle/direct/autotest/README.md for the verdicts. + +description x absent UPDATE_IGNORED +description y absent UPDATE_IGNORED +title x-UNIQUE absent UPDATE_IGNORED +title y-UNIQUE absent UPDATE_IGNORED + +=== summary +OK 8 +NOT_OBSERVABLE 6 +UPDATE_IGNORED 4 +SKIPPED 4 diff --git a/bundle/direct/autotest/output/instance_pools.txt b/bundle/direct/autotest/output/instance_pools.txt new file mode 100644 index 00000000000..8ad82343ace --- /dev/null +++ b/bundle/direct/autotest/output/instance_pools.txt @@ -0,0 +1,20 @@ +# field | from | to | verdict | detail +# +# How the direct engine handles every field of this resource type: each row is one move of one field +# between two values, "absent" included. An outcome the engine got wrong is recorded here rather than +# failed, so this file is a description of current behaviour -- what fails the test is a verdict +# changing. Passing rows are counted in the summary and listed in logs/.full.txt, along with +# the evidence behind each finding and what could not be covered. +# +# Regenerate with ./task test-update-fields. See bundle/direct/autotest/README.md for the verdicts. + +enable_elastic_disk false true UPDATE_IGNORED +enable_elastic_disk true false UPDATE_IGNORED + +=== summary +OK 45 +OK_RECREATE 46 +SUPPRESSED 5 +NOT_OBSERVABLE 2 +UPDATE_IGNORED 2 +SKIPPED 6 diff --git a/bundle/direct/autotest/output/job_runs.txt b/bundle/direct/autotest/output/job_runs.txt new file mode 100644 index 00000000000..f7528a050df --- /dev/null +++ b/bundle/direct/autotest/output/job_runs.txt @@ -0,0 +1,16 @@ +# field | from | to | verdict | detail +# +# How the direct engine handles every field of this resource type: each row is one move of one field +# between two values, "absent" included. An outcome the engine got wrong is recorded here rather than +# failed, so this file is a description of current behaviour -- what fails the test is a verdict +# changing. Passing rows are counted in the summary and listed in logs/.full.txt, along with +# the evidence behind each finding and what could not be covered. +# +# Regenerate with ./task test-update-fields. See bundle/direct/autotest/README.md for the verdicts. + + +=== summary +OK_RECREATE 109 +SUPPRESSED 9 +NOT_OBSERVABLE 12 +SKIPPED 1 diff --git a/bundle/direct/autotest/output/jobs.txt b/bundle/direct/autotest/output/jobs.txt new file mode 100644 index 00000000000..18487b2ee84 --- /dev/null +++ b/bundle/direct/autotest/output/jobs.txt @@ -0,0 +1,17 @@ +# field | from | to | verdict | detail +# +# How the direct engine handles every field of this resource type: each row is one move of one field +# between two values, "absent" included. An outcome the engine got wrong is recorded here rather than +# failed, so this file is a description of current behaviour -- what fails the test is a verdict +# changing. Passing rows are counted in the summary and listed in logs/.full.txt, along with +# the evidence behind each finding and what could not be covered. +# +# Regenerate with ./task test-update-fields. See bundle/direct/autotest/README.md for the verdicts. + + +=== summary +OK 88 +OK_INERT 6 +SUPPRESSED 8 +NOT_OBSERVABLE 6 +SKIPPED 88 diff --git a/bundle/direct/autotest/output/model_serving_endpoints.txt b/bundle/direct/autotest/output/model_serving_endpoints.txt new file mode 100644 index 00000000000..c98c8725ae7 --- /dev/null +++ b/bundle/direct/autotest/output/model_serving_endpoints.txt @@ -0,0 +1,18 @@ +# field | from | to | verdict | detail +# +# How the direct engine handles every field of this resource type: each row is one move of one field +# between two values, "absent" included. An outcome the engine got wrong is recorded here rather than +# failed, so this file is a description of current behaviour -- what fails the test is a verdict +# changing. Passing rows are counted in the summary and listed in logs/.full.txt, along with +# the evidence behind each finding and what could not be covered. +# +# Regenerate with ./task test-update-fields. See bundle/direct/autotest/README.md for the verdicts. + + +=== summary +OK 24 +OK_RECREATE 12 +OK_INERT 6 +SUPPRESSED 3 +NOT_OBSERVABLE 1 +SKIPPED 4 diff --git a/bundle/direct/autotest/output/models.txt b/bundle/direct/autotest/output/models.txt new file mode 100644 index 00000000000..5a705b5557e --- /dev/null +++ b/bundle/direct/autotest/output/models.txt @@ -0,0 +1,19 @@ +# field | from | to | verdict | detail +# +# How the direct engine handles every field of this resource type: each row is one move of one field +# between two values, "absent" included. An outcome the engine got wrong is recorded here rather than +# failed, so this file is a description of current behaviour -- what fails the test is a verdict +# changing. Passing rows are counted in the summary and listed in logs/.full.txt, along with +# the evidence behind each finding and what could not be covered. +# +# Regenerate with ./task test-update-fields. See bundle/direct/autotest/README.md for the verdicts. + +description x absent BACKEND_ERROR cannot update resources.models.foo: updating id=test-model-[UNIQUE_NAME]: Description cannot be empty. (400 INVALID_PARAMETER_VALUE) +description y absent BACKEND_ERROR cannot update resources.models.foo: updating id=test-model-[UNIQUE_NAME]: Description cannot be empty. (400 INVALID_PARAMETER_VALUE) + +=== summary +OK 4 +OK_RECREATE 2 +OK_INERT 18 +BACKEND_ERROR 2 +SKIPPED 1 diff --git a/bundle/direct/autotest/output/pipelines.txt b/bundle/direct/autotest/output/pipelines.txt new file mode 100644 index 00000000000..6d3b42b17d4 --- /dev/null +++ b/bundle/direct/autotest/output/pipelines.txt @@ -0,0 +1,17 @@ +# field | from | to | verdict | detail +# +# How the direct engine handles every field of this resource type: each row is one move of one field +# between two values, "absent" included. An outcome the engine got wrong is recorded here rather than +# failed, so this file is a description of current behaviour -- what fails the test is a verdict +# changing. Passing rows are counted in the summary and listed in logs/.full.txt, along with +# the evidence behind each finding and what could not be covered. +# +# Regenerate with ./task test-update-fields. See bundle/direct/autotest/README.md for the verdicts. + + +=== summary +OK 531 +OK_INERT 12 +SUPPRESSED 32 +NOT_OBSERVABLE 23 +SKIPPED 41 diff --git a/bundle/direct/autotest/output/postgres_branches.txt b/bundle/direct/autotest/output/postgres_branches.txt new file mode 100644 index 00000000000..23076acc9df --- /dev/null +++ b/bundle/direct/autotest/output/postgres_branches.txt @@ -0,0 +1,25 @@ +# field | from | to | verdict | detail +# +# How the direct engine handles every field of this resource type: each row is one move of one field +# between two values, "absent" included. An outcome the engine got wrong is recorded here rather than +# failed, so this file is a description of current behaviour -- what fails the test is a verdict +# changing. Passing rows are counted in the summary and listed in logs/.full.txt, along with +# the evidence behind each finding and what could not be covered. +# +# Regenerate with ./task test-update-fields. See bundle/direct/autotest/README.md for the verdicts. + +is_protected true absent BACKEND_ERROR cannot update resources.postgres_branches.foo: updating id=projects/test-pg-project-[UNIQU...ovided in request (400 INVALID_PARAMETER_VALUE) +no_expiry absent false BASE_ERROR cannot update resources.postgres_branches.foo: updating id=projects/test-pg-project-[UNIQU...ovided in request (400 INVALID_PARAMETER_VALUE) +no_expiry true absent BACKEND_ERROR cannot update resources.postgres_branches.foo: updating id=projects/test-pg-project-[UNIQU...ovided in request (400 INVALID_PARAMETER_VALUE) +parent x y BASE_ERROR cannot recreate resources.postgres_branches.foo: No stub found for pattern: POST /api/2.0/postgres/x/branches (501 ) +parent y x BASE_ERROR cannot recreate resources.postgres_branches.foo: No stub found for pattern: POST /api/2.0/postgres/y/branches (501 ) + +=== summary +OK 10 +OK_RECREATE 14 +OK_INERT 6 +SUPPRESSED 4 +NOT_OBSERVABLE 1 +BACKEND_ERROR 2 +SKIPPED 1 +BASE_ERROR 3 diff --git a/bundle/direct/autotest/output/postgres_catalogs.txt b/bundle/direct/autotest/output/postgres_catalogs.txt new file mode 100644 index 00000000000..68c2f021a47 --- /dev/null +++ b/bundle/direct/autotest/output/postgres_catalogs.txt @@ -0,0 +1,16 @@ +# field | from | to | verdict | detail +# +# How the direct engine handles every field of this resource type: each row is one move of one field +# between two values, "absent" included. An outcome the engine got wrong is recorded here rather than +# failed, so this file is a description of current behaviour -- what fails the test is a verdict +# changing. Passing rows are counted in the summary and listed in logs/.full.txt, along with +# the evidence behind each finding and what could not be covered. +# +# Regenerate with ./task test-update-fields. See bundle/direct/autotest/README.md for the verdicts. + + +=== summary +OK_RECREATE 14 +SUPPRESSED 1 +NOT_OBSERVABLE 1 +SKIPPED 1 diff --git a/bundle/direct/autotest/output/postgres_databases.txt b/bundle/direct/autotest/output/postgres_databases.txt new file mode 100644 index 00000000000..10af6e91a9b --- /dev/null +++ b/bundle/direct/autotest/output/postgres_databases.txt @@ -0,0 +1,19 @@ +# field | from | to | verdict | detail +# +# How the direct engine handles every field of this resource type: each row is one move of one field +# between two values, "absent" included. An outcome the engine got wrong is recorded here rather than +# failed, so this file is a description of current behaviour -- what fails the test is a verdict +# changing. Passing rows are counted in the summary and listed in logs/.full.txt, along with +# the evidence behind each finding and what could not be covered. +# +# Regenerate with ./task test-update-fields. See bundle/direct/autotest/README.md for the verdicts. + +parent x y BASE_ERROR cannot recreate resources.postgres_databases.foo: No stub found for pattern: POST /api/2.0/postgres/x/databases (501 ) +parent y x BASE_ERROR cannot recreate resources.postgres_databases.foo: No stub found for pattern: POST /api/2.0/postgres/y/databases (501 ) + +=== summary +OK 8 +OK_RECREATE 2 +OK_INERT 6 +SKIPPED 1 +BASE_ERROR 2 diff --git a/bundle/direct/autotest/output/postgres_endpoints.txt b/bundle/direct/autotest/output/postgres_endpoints.txt new file mode 100644 index 00000000000..f502e1216d9 --- /dev/null +++ b/bundle/direct/autotest/output/postgres_endpoints.txt @@ -0,0 +1,37 @@ +# field | from | to | verdict | detail +# +# How the direct engine handles every field of this resource type: each row is one move of one field +# between two values, "absent" included. An outcome the engine got wrong is recorded here rather than +# failed, so this file is a description of current behaviour -- what fails the test is a verdict +# changing. Passing rows are counted in the summary and listed in logs/.full.txt, along with +# the evidence behind each finding and what could not be covered. +# +# Regenerate with ./task test-update-fields. See bundle/direct/autotest/README.md for the verdicts. + +autoscaling_limit_max_cu 1 absent BACKEND_ERROR cannot update resources.postgres_endpoints.foo: updating id=projects/test-pg-project-[UNIQ...ovided in request (400 INVALID_PARAMETER_VALUE) +autoscaling_limit_max_cu 2 absent BACKEND_ERROR cannot update resources.postgres_endpoints.foo: updating id=projects/test-pg-project-[UNIQ...ovided in request (400 INVALID_PARAMETER_VALUE) +autoscaling_limit_min_cu 1 absent BACKEND_ERROR cannot update resources.postgres_endpoints.foo: updating id=projects/test-pg-project-[UNIQ...ovided in request (400 INVALID_PARAMETER_VALUE) +autoscaling_limit_min_cu 2 absent BACKEND_ERROR cannot update resources.postgres_endpoints.foo: updating id=projects/test-pg-project-[UNIQ...ovided in request (400 INVALID_PARAMETER_VALUE) +disabled true absent BACKEND_ERROR cannot update resources.postgres_endpoints.foo: updating id=projects/test-pg-project-[UNIQ...ovided in request (400 INVALID_PARAMETER_VALUE) +group.enable_readable_secondaries absent false BACKEND_ERROR cannot update resources.postgres_endpoints.foo: updating id=projects/test-pg-project-[UNIQ...ask: 'spec.group' (400 INVALID_PARAMETER_VALUE) +group.enable_readable_secondaries absent true BACKEND_ERROR cannot update resources.postgres_endpoints.foo: updating id=projects/test-pg-project-[UNIQ...ask: 'spec.group' (400 INVALID_PARAMETER_VALUE) +group.enable_readable_secondaries false absent BASE_ERROR cannot update resources.postgres_endpoints.foo: updating id=projects/test-pg-project-[UNIQ...ask: 'spec.group' (400 INVALID_PARAMETER_VALUE) +group.enable_readable_secondaries false true BASE_ERROR cannot update resources.postgres_endpoints.foo: updating id=projects/test-pg-project-[UNIQ...ask: 'spec.group' (400 INVALID_PARAMETER_VALUE) +group.enable_readable_secondaries true absent BASE_ERROR cannot update resources.postgres_endpoints.foo: updating id=projects/test-pg-project-[UNIQ...ask: 'spec.group' (400 INVALID_PARAMETER_VALUE) +group.enable_readable_secondaries true false BASE_ERROR cannot update resources.postgres_endpoints.foo: updating id=projects/test-pg-project-[UNIQ...ask: 'spec.group' (400 INVALID_PARAMETER_VALUE) +group.max 1 2 BASE_ERROR cannot update resources.postgres_endpoints.foo: updating id=projects/test-pg-project-[UNIQ...ask: 'spec.group' (400 INVALID_PARAMETER_VALUE) +group.max 2 1 BASE_ERROR cannot update resources.postgres_endpoints.foo: updating id=projects/test-pg-project-[UNIQ...ask: 'spec.group' (400 INVALID_PARAMETER_VALUE) +group.min 1 2 BASE_ERROR cannot update resources.postgres_endpoints.foo: updating id=projects/test-pg-project-[UNIQ...ask: 'spec.group' (400 INVALID_PARAMETER_VALUE) +group.min 2 1 BASE_ERROR cannot update resources.postgres_endpoints.foo: updating id=projects/test-pg-project-[UNIQ...ask: 'spec.group' (400 INVALID_PARAMETER_VALUE) +parent x y BASE_ERROR cannot recreate resources.postgres_endpoints.foo: No stub found for pattern: POST /api/2.0/postgres/x/endpoints (501 ) +parent y x BASE_ERROR cannot recreate resources.postgres_endpoints.foo: No stub found for pattern: POST /api/2.0/postgres/y/endpoints (501 ) + +=== summary +OK 15 +OK_RECREATE 4 +OK_INERT 6 +SUPPRESSED 2 +NOT_OBSERVABLE 2 +BACKEND_ERROR 7 +SKIPPED 1 +BASE_ERROR 10 diff --git a/bundle/direct/autotest/output/postgres_projects.txt b/bundle/direct/autotest/output/postgres_projects.txt new file mode 100644 index 00000000000..67b2678566e --- /dev/null +++ b/bundle/direct/autotest/output/postgres_projects.txt @@ -0,0 +1,45 @@ +# field | from | to | verdict | detail +# +# How the direct engine handles every field of this resource type: each row is one move of one field +# between two values, "absent" included. An outcome the engine got wrong is recorded here rather than +# failed, so this file is a description of current behaviour -- what fails the test is a verdict +# changing. Passing rows are counted in the summary and listed in logs/.full.txt, along with +# the evidence behind each finding and what could not be covered. +# +# Regenerate with ./task test-update-fields. See bundle/direct/autotest/README.md for the verdicts. + +budget_policy_id x absent BACKEND_ERROR cannot update resources.postgres_projects.foo: updating id=projects/test-pg-project-[UNIQU...ovided in request (400 INVALID_PARAMETER_VALUE) +budget_policy_id y absent BACKEND_ERROR cannot update resources.postgres_projects.foo: updating id=projects/test-pg-project-[UNIQU...ovided in request (400 INVALID_PARAMETER_VALUE) +custom_tags absent len0 BASE_ERROR cannot update resources.postgres_projects.foo: updating id=projects/test-pg-project-[UNIQU...ovided in request (400 INVALID_PARAMETER_VALUE) +custom_tags absent len1 BASE_ERROR cannot update resources.postgres_projects.foo: updating id=projects/test-pg-project-[UNIQU...ovided in request (400 INVALID_PARAMETER_VALUE) +custom_tags len0 absent BASE_ERROR cannot update resources.postgres_projects.foo: updating id=projects/test-pg-project-[UNIQU...ovided in request (400 INVALID_PARAMETER_VALUE) +custom_tags len0 len1 BASE_ERROR cannot update resources.postgres_projects.foo: updating id=projects/test-pg-project-[UNIQU...ovided in request (400 INVALID_PARAMETER_VALUE) +custom_tags len1 absent BACKEND_ERROR cannot update resources.postgres_projects.foo: updating id=projects/test-pg-project-[UNIQU...ovided in request (400 INVALID_PARAMETER_VALUE) +custom_tags len1 len0 BACKEND_ERROR cannot update resources.postgres_projects.foo: updating id=projects/test-pg-project-[UNIQU...ovided in request (400 INVALID_PARAMETER_VALUE) +default_endpoint_settings.autoscaling_limit_max_cu 1 2 BASE_ERROR cannot update resources.postgres_projects.foo: updating id=projects/test-pg-project-[UNIQU...ovided in request (400 INVALID_PARAMETER_VALUE) +default_endpoint_settings.autoscaling_limit_max_cu 1 absent BASE_ERROR cannot update resources.postgres_projects.foo: updating id=projects/test-pg-project-[UNIQU...ovided in request (400 INVALID_PARAMETER_VALUE) +default_endpoint_settings.autoscaling_limit_max_cu 2 1 BASE_ERROR cannot update resources.postgres_projects.foo: updating id=projects/test-pg-project-[UNIQU...ovided in request (400 INVALID_PARAMETER_VALUE) +default_endpoint_settings.autoscaling_limit_max_cu 2 absent BASE_ERROR cannot update resources.postgres_projects.foo: updating id=projects/test-pg-project-[UNIQU...ovided in request (400 INVALID_PARAMETER_VALUE) +default_endpoint_settings.autoscaling_limit_max_cu absent 1 BACKEND_ERROR cannot update resources.postgres_projects.foo: updating id=projects/test-pg-project-[UNIQU...ovided in request (400 INVALID_PARAMETER_VALUE) +default_endpoint_settings.autoscaling_limit_max_cu absent 2 BACKEND_ERROR cannot update resources.postgres_projects.foo: updating id=projects/test-pg-project-[UNIQU...ovided in request (400 INVALID_PARAMETER_VALUE) +default_endpoint_settings.autoscaling_limit_min_cu 1 2 BASE_ERROR cannot update resources.postgres_projects.foo: updating id=projects/test-pg-project-[UNIQU...ovided in request (400 INVALID_PARAMETER_VALUE) +default_endpoint_settings.autoscaling_limit_min_cu 1 absent BASE_ERROR cannot update resources.postgres_projects.foo: updating id=projects/test-pg-project-[UNIQU...ovided in request (400 INVALID_PARAMETER_VALUE) +default_endpoint_settings.autoscaling_limit_min_cu 2 1 BASE_ERROR cannot update resources.postgres_projects.foo: updating id=projects/test-pg-project-[UNIQU...ovided in request (400 INVALID_PARAMETER_VALUE) +default_endpoint_settings.autoscaling_limit_min_cu 2 absent BASE_ERROR cannot update resources.postgres_projects.foo: updating id=projects/test-pg-project-[UNIQU...ovided in request (400 INVALID_PARAMETER_VALUE) +default_endpoint_settings.autoscaling_limit_min_cu absent 1 BACKEND_ERROR cannot update resources.postgres_projects.foo: updating id=projects/test-pg-project-[UNIQU...ovided in request (400 INVALID_PARAMETER_VALUE) +default_endpoint_settings.autoscaling_limit_min_cu absent 2 BACKEND_ERROR cannot update resources.postgres_projects.foo: updating id=projects/test-pg-project-[UNIQU...ovided in request (400 INVALID_PARAMETER_VALUE) +default_endpoint_settings.no_suspension false absent BACKEND_ERROR cannot update resources.postgres_projects.foo: updating id=projects/test-pg-project-[UNIQU...ovided in request (400 INVALID_PARAMETER_VALUE) +default_endpoint_settings.no_suspension true absent BACKEND_ERROR cannot update resources.postgres_projects.foo: updating id=projects/test-pg-project-[UNIQU...ovided in request (400 INVALID_PARAMETER_VALUE) +display_name absent x BASE_ERROR cannot update resources.postgres_projects.foo: updating id=projects/test-pg-project-[UNIQU...ovided in request (400 INVALID_PARAMETER_VALUE) +display_name absent y BASE_ERROR cannot update resources.postgres_projects.foo: updating id=projects/test-pg-project-[UNIQU...ovided in request (400 INVALID_PARAMETER_VALUE) +display_name x absent BACKEND_ERROR cannot update resources.postgres_projects.foo: updating id=projects/test-pg-project-[UNIQU...ovided in request (400 INVALID_PARAMETER_VALUE) +display_name y absent BACKEND_ERROR cannot update resources.postgres_projects.foo: updating id=projects/test-pg-project-[UNIQU...ovided in request (400 INVALID_PARAMETER_VALUE) +enable_pg_native_login true absent BACKEND_ERROR cannot update resources.postgres_projects.foo: updating id=projects/test-pg-project-[UNIQU...ovided in request (400 INVALID_PARAMETER_VALUE) + +=== summary +OK 29 +OK_RECREATE 8 +SUPPRESSED 4 +BACKEND_ERROR 13 +SKIPPED 3 +BASE_ERROR 14 diff --git a/bundle/direct/autotest/output/postgres_roles.txt b/bundle/direct/autotest/output/postgres_roles.txt new file mode 100644 index 00000000000..5cab59b561a --- /dev/null +++ b/bundle/direct/autotest/output/postgres_roles.txt @@ -0,0 +1,21 @@ +# field | from | to | verdict | detail +# +# How the direct engine handles every field of this resource type: each row is one move of one field +# between two values, "absent" included. An outcome the engine got wrong is recorded here rather than +# failed, so this file is a description of current behaviour -- what fails the test is a verdict +# changing. Passing rows are counted in the summary and listed in logs/.full.txt, along with +# the evidence behind each finding and what could not be covered. +# +# Regenerate with ./task test-update-fields. See bundle/direct/autotest/README.md for the verdicts. + +parent x y BASE_ERROR cannot recreate resources.postgres_roles.foo: No stub found for pattern: POST /api/2.0/postgres/x/roles (501 ) +parent y x BASE_ERROR cannot recreate resources.postgres_roles.foo: No stub found for pattern: POST /api/2.0/postgres/y/roles (501 ) + +=== summary +OK 22 +OK_RECREATE 20 +OK_INERT 6 +SUPPRESSED 1 +NOT_OBSERVABLE 1 +SKIPPED 1 +BASE_ERROR 2 diff --git a/bundle/direct/autotest/output/postgres_synced_tables.txt b/bundle/direct/autotest/output/postgres_synced_tables.txt new file mode 100644 index 00000000000..787c4566dfd --- /dev/null +++ b/bundle/direct/autotest/output/postgres_synced_tables.txt @@ -0,0 +1,16 @@ +# field | from | to | verdict | detail +# +# How the direct engine handles every field of this resource type: each row is one move of one field +# between two values, "absent" included. An outcome the engine got wrong is recorded here rather than +# failed, so this file is a description of current behaviour -- what fails the test is a verdict +# changing. Passing rows are counted in the summary and listed in logs/.full.txt, along with +# the evidence behind each finding and what could not be covered. +# +# Regenerate with ./task test-update-fields. See bundle/direct/autotest/README.md for the verdicts. + + +=== summary +OK_RECREATE 110 +SUPPRESSED 6 +NOT_OBSERVABLE 4 +SKIPPED 1 diff --git a/bundle/direct/autotest/output/quality_monitors.txt b/bundle/direct/autotest/output/quality_monitors.txt new file mode 100644 index 00000000000..182e92b6d11 --- /dev/null +++ b/bundle/direct/autotest/output/quality_monitors.txt @@ -0,0 +1,22 @@ +# field | from | to | verdict | detail +# +# How the direct engine handles every field of this resource type: each row is one move of one field +# between two values, "absent" included. An outcome the engine got wrong is recorded here rather than +# failed, so this file is a description of current behaviour -- what fails the test is a verdict +# changing. Passing rows are counted in the summary and listed in logs/.full.txt, along with +# the evidence behind each finding and what could not be covered. +# +# Regenerate with ./task test-update-fields. See bundle/direct/autotest/README.md for the verdicts. + +latest_monitor_failure_msg absent x POST_DEPLOY_DRIFT latest_monitor_failure_msg +latest_monitor_failure_msg absent y POST_DEPLOY_DRIFT latest_monitor_failure_msg +latest_monitor_failure_msg x y POST_DEPLOY_DRIFT latest_monitor_failure_msg +latest_monitor_failure_msg y x POST_DEPLOY_DRIFT latest_monitor_failure_msg + +=== summary +OK 100 +OK_RECREATE 4 +SUPPRESSED 6 +NOT_OBSERVABLE 4 +POST_DEPLOY_DRIFT 4 +SKIPPED 1 diff --git a/bundle/direct/autotest/output/registered_models.txt b/bundle/direct/autotest/output/registered_models.txt new file mode 100644 index 00000000000..297d4329fa0 --- /dev/null +++ b/bundle/direct/autotest/output/registered_models.txt @@ -0,0 +1,27 @@ +# field | from | to | verdict | detail +# +# How the direct engine handles every field of this resource type: each row is one move of one field +# between two values, "absent" included. An outcome the engine got wrong is recorded here rather than +# failed, so this file is a description of current behaviour -- what fails the test is a verdict +# changing. Passing rows are counted in the summary and listed in logs/.full.txt, along with +# the evidence behind each finding and what could not be covered. +# +# Regenerate with ./task test-update-fields. See bundle/direct/autotest/README.md for the verdicts. + +full_name absent x UPDATE_IGNORED +full_name absent y UPDATE_IGNORED +full_name x y UPDATE_IGNORED +full_name y x UPDATE_IGNORED +owner absent x UPDATE_IGNORED +owner absent y UPDATE_IGNORED +owner x y UPDATE_IGNORED +owner y x UPDATE_IGNORED + +=== summary +OK 44 +OK_RECREATE 2 +OK_ID_FIELD_REQUIRED 8 +SUPPRESSED 1 +NOT_OBSERVABLE 1 +UPDATE_IGNORED 8 +SKIPPED 9 diff --git a/bundle/direct/autotest/output/schemas.txt b/bundle/direct/autotest/output/schemas.txt new file mode 100644 index 00000000000..348784ab2e6 --- /dev/null +++ b/bundle/direct/autotest/output/schemas.txt @@ -0,0 +1,24 @@ +# field | from | to | verdict | detail +# +# How the direct engine handles every field of this resource type: each row is one move of one field +# between two values, "absent" included. An outcome the engine got wrong is recorded here rather than +# failed, so this file is a description of current behaviour -- what fails the test is a verdict +# changing. Passing rows are counted in the summary and listed in logs/.full.txt, along with +# the evidence behind each finding and what could not be covered. +# +# Regenerate with ./task test-update-fields. See bundle/direct/autotest/README.md for the verdicts. + +custom_max_retention_hours 168 absent UPDATE_IGNORED +custom_max_retention_hours 720 absent UPDATE_IGNORED +properties keys1 absent UPDATE_IGNORED +properties keys1 keys0 UPDATE_IGNORED +properties['team'] x absent UPDATE_IGNORED +properties['team'] y absent UPDATE_IGNORED + +=== summary +OK 16 +OK_RECREATE 2 +SUPPRESSED 1 +NOT_OBSERVABLE 1 +UPDATE_IGNORED 6 +SKIPPED 2 diff --git a/bundle/direct/autotest/output/secret_scopes.txt b/bundle/direct/autotest/output/secret_scopes.txt new file mode 100644 index 00000000000..c2011a38c39 --- /dev/null +++ b/bundle/direct/autotest/output/secret_scopes.txt @@ -0,0 +1,17 @@ +# field | from | to | verdict | detail +# +# How the direct engine handles every field of this resource type: each row is one move of one field +# between two values, "absent" included. An outcome the engine got wrong is recorded here rather than +# failed, so this file is a description of current behaviour -- what fails the test is a verdict +# changing. Passing rows are counted in the summary and listed in logs/.full.txt, along with +# the evidence behind each finding and what could not be covered. +# +# Regenerate with ./task test-update-fields. See bundle/direct/autotest/README.md for the verdicts. + +backend_type AZURE_KEYVAULT DATABRICKS BASE_ERROR cannot recreate resources.secret_scopes.foo: Scope with Azure KeyVault must have AzureKeyV...Metadata defined! (400 INVALID_PARAMETER_VALUE) +backend_type AZURE_KEYVAULT absent BASE_ERROR cannot recreate resources.secret_scopes.foo: Scope with Azure KeyVault must have AzureKeyV...Metadata defined! (400 INVALID_PARAMETER_VALUE) + +=== summary +NOT_OBSERVABLE 10 +SKIPPED 1 +BASE_ERROR 2 diff --git a/bundle/direct/autotest/output/secrets.txt b/bundle/direct/autotest/output/secrets.txt new file mode 100644 index 00000000000..aeb3aca552c --- /dev/null +++ b/bundle/direct/autotest/output/secrets.txt @@ -0,0 +1,25 @@ +# field | from | to | verdict | detail +# +# How the direct engine handles every field of this resource type: each row is one move of one field +# between two values, "absent" included. An outcome the engine got wrong is recorded here rather than +# failed, so this file is a description of current behaviour -- what fails the test is a verdict +# changing. Passing rows are counted in the summary and listed in logs/.full.txt, along with +# the evidence behind each finding and what could not be covered. +# +# Regenerate with ./task test-update-fields. See bundle/direct/autotest/README.md for the verdicts. + +comment x absent UPDATE_IGNORED +comment y absent UPDATE_IGNORED +owner absent x SUPPRESSED spec:input_only +owner absent y SUPPRESSED spec:input_only +owner x absent SUPPRESSED spec:input_only +owner x y SUPPRESSED spec:input_only +owner y absent SUPPRESSED spec:input_only +owner y x SUPPRESSED spec:input_only + +=== summary +OK 4 +OK_RECREATE 2 +SUPPRESSED 6 +UPDATE_IGNORED 2 +SKIPPED 10 diff --git a/bundle/direct/autotest/output/sql_warehouses.txt b/bundle/direct/autotest/output/sql_warehouses.txt new file mode 100644 index 00000000000..e82c629dc14 --- /dev/null +++ b/bundle/direct/autotest/output/sql_warehouses.txt @@ -0,0 +1,34 @@ +# field | from | to | verdict | detail +# +# How the direct engine handles every field of this resource type: each row is one move of one field +# between two values, "absent" included. An outcome the engine got wrong is recorded here rather than +# failed, so this file is a description of current behaviour -- what fails the test is a verdict +# changing. Passing rows are counted in the summary and listed in logs/.full.txt, along with +# the evidence behind each finding and what could not be covered. +# +# Regenerate with ./task test-update-fields. See bundle/direct/autotest/README.md for the verdicts. + +auto_stop_mins 10 absent UPDATE_IGNORED +auto_stop_mins 20 absent UPDATE_IGNORED +cluster_size 2X-Small absent UPDATE_IGNORED +cluster_size X-Small absent UPDATE_IGNORED +cluster_size absent 2X-Small START_NOT_REACHED cannot create the resource with this starting value: cannot create resources.sql_warehouses.foo: Required field 'cluster_size' is missing. (400 INVALID_PARAMETER_VALUE) +cluster_size absent X-Small START_NOT_REACHED cannot create the resource with this starting value: cannot create resources.sql_warehouses.foo: Required field 'cluster_size' is missing. (400 INVALID_PARAMETER_VALUE) +enable_photon true absent UPDATE_IGNORED +max_num_clusters 2 absent UPDATE_IGNORED +max_num_clusters 3 absent UPDATE_IGNORED +max_num_clusters absent 2 START_NOT_REACHED cannot create the resource with this starting value: cannot create resources.sql_warehouses.foo: 0 is not a valid value for max_num_clusters. T...n or equal to 40. (400 INVALID_PARAMETER_VALUE) +max_num_clusters absent 3 START_NOT_REACHED cannot create the resource with this starting value: cannot create resources.sql_warehouses.foo: 0 is not a valid value for max_num_clusters. T...n or equal to 40. (400 INVALID_PARAMETER_VALUE) +name absent x-UNIQUE START_NOT_REACHED cannot create the resource with this starting value: cannot create resources.sql_warehouses.foo: Invalid value for SQL Endpoint name, it cannot be empty. (400 INVALID_PARAMETER_VALUE) +name absent y-UNIQUE START_NOT_REACHED cannot create the resource with this starting value: cannot create resources.sql_warehouses.foo: Invalid value for SQL Endpoint name, it cannot be empty. (400 INVALID_PARAMETER_VALUE) +name x-UNIQUE absent UPDATE_IGNORED +name y-UNIQUE absent UPDATE_IGNORED +spot_instance_policy COST_OPTIMIZED absent UPDATE_IGNORED +spot_instance_policy RELIABILITY_OPTIMIZED absent UPDATE_IGNORED + +=== summary +OK 31 +SUPPRESSED 2 +UPDATE_IGNORED 11 +SKIPPED 6 +START_NOT_REACHED 6 diff --git a/bundle/direct/autotest/output/synced_database_tables.txt b/bundle/direct/autotest/output/synced_database_tables.txt new file mode 100644 index 00000000000..e1d9a827c5f --- /dev/null +++ b/bundle/direct/autotest/output/synced_database_tables.txt @@ -0,0 +1,24 @@ +# field | from | to | verdict | detail +# +# How the direct engine handles every field of this resource type: each row is one move of one field +# between two values, "absent" included. An outcome the engine got wrong is recorded here rather than +# failed, so this file is a description of current behaviour -- what fails the test is a verdict +# changing. Passing rows are counted in the summary and listed in logs/.full.txt, along with +# the evidence behind each finding and what could not be covered. +# +# Regenerate with ./task test-update-fields. See bundle/direct/autotest/README.md for the verdicts. + +database_instance_name absent x DEPLOY_ERROR cannot recreate resources.synced_database_tables.foo: database instance with name 'x' not found (404 UNKNOWN) +database_instance_name absent y DEPLOY_ERROR cannot recreate resources.synced_database_tables.foo: database instance with name 'y' not found (404 UNKNOWN) +database_instance_name x absent BASE_ERROR cannot recreate resources.synced_database_tables.foo: database instance with name 'x' not found (404 UNKNOWN) +database_instance_name x y BASE_ERROR cannot recreate resources.synced_database_tables.foo: database instance with name 'x' not found (404 UNKNOWN) +database_instance_name y absent BASE_ERROR cannot recreate resources.synced_database_tables.foo: database instance with name 'y' not found (404 UNKNOWN) +database_instance_name y x BASE_ERROR cannot recreate resources.synced_database_tables.foo: database instance with name 'y' not found (404 UNKNOWN) + +=== summary +OK_RECREATE 68 +SUPPRESSED 4 +NOT_OBSERVABLE 2 +DEPLOY_ERROR 2 +SKIPPED 35 +BASE_ERROR 4 diff --git a/bundle/direct/autotest/output/vector_search_endpoints.txt b/bundle/direct/autotest/output/vector_search_endpoints.txt new file mode 100644 index 00000000000..abfbe8fe2f3 --- /dev/null +++ b/bundle/direct/autotest/output/vector_search_endpoints.txt @@ -0,0 +1,18 @@ +# field | from | to | verdict | detail +# +# How the direct engine handles every field of this resource type: each row is one move of one field +# between two values, "absent" included. An outcome the engine got wrong is recorded here rather than +# failed, so this file is a description of current behaviour -- what fails the test is a verdict +# changing. Passing rows are counted in the summary and listed in logs/.full.txt, along with +# the evidence behind each finding and what could not be covered. +# +# Regenerate with ./task test-update-fields. See bundle/direct/autotest/README.md for the verdicts. + +target_qps 1 absent UPDATE_IGNORED +target_qps 2 absent UPDATE_IGNORED + +=== summary +OK 10 +OK_RECREATE 4 +UPDATE_IGNORED 2 +SKIPPED 2 diff --git a/bundle/direct/autotest/output/vector_search_indexes.txt b/bundle/direct/autotest/output/vector_search_indexes.txt new file mode 100644 index 00000000000..623f223c5ba --- /dev/null +++ b/bundle/direct/autotest/output/vector_search_indexes.txt @@ -0,0 +1,14 @@ +# field | from | to | verdict | detail +# +# How the direct engine handles every field of this resource type: each row is one move of one field +# between two values, "absent" included. An outcome the engine got wrong is recorded here rather than +# failed, so this file is a description of current behaviour -- what fails the test is a verdict +# changing. Passing rows are counted in the summary and listed in logs/.full.txt, along with +# the evidence behind each finding and what could not be covered. +# +# Regenerate with ./task test-update-fields. See bundle/direct/autotest/README.md for the verdicts. + + +=== summary +OK_RECREATE 2 +SKIPPED 7 diff --git a/bundle/direct/autotest/output/volumes.txt b/bundle/direct/autotest/output/volumes.txt new file mode 100644 index 00000000000..70459278683 --- /dev/null +++ b/bundle/direct/autotest/output/volumes.txt @@ -0,0 +1,14 @@ +# field | from | to | verdict | detail +# +# How the direct engine handles every field of this resource type: each row is one move of one field +# between two values, "absent" included. An outcome the engine got wrong is recorded here rather than +# failed, so this file is a description of current behaviour -- what fails the test is a verdict +# changing. Passing rows are counted in the summary and listed in logs/.full.txt, along with +# the evidence behind each finding and what could not be covered. +# +# Regenerate with ./task test-update-fields. See bundle/direct/autotest/README.md for the verdicts. + + +=== summary +OK 8 +SKIPPED 2 diff --git a/bundle/direct/autotest/report_test.go b/bundle/direct/autotest/report_test.go new file mode 100644 index 00000000000..2cfe83946da --- /dev/null +++ b/bundle/direct/autotest/report_test.go @@ -0,0 +1,582 @@ +package autotest + +import ( + "cmp" + "encoding/json" + "fmt" + "os" + "path/filepath" + "slices" + "strings" + "time" + + "github.com/databricks/cli/internal/testutil" + "github.com/databricks/cli/libs/testdiff" +) + +// verdict classifies what happened to one field transition. The suite never fails a +// subtest on a bad verdict: the point is to catalog which fields the engine handles, +// so every outcome is recorded and the golden file is the report. +type verdict string + +const ( + // The engine planned a change, applied it, and the next plan was clean. + verdictOK verdict = "OK" + // Same, but the engine replaced the resource instead of updating it. + verdictRecreate verdict = "OK_RECREATE" + + // The planner saw the field change and deliberately dropped it. The detail carries + // the engine's own reason, which is the interesting part. + verdictSuppressed verdict = "SUPPRESSED" + // The plan has no entry for the field at all: the two values are indistinguishable + // in the state the engine sends, so there is nothing to observe. An unset bool and + // an explicit false are the common case. A field the engine consumes before it plans + // anything also lands here -- an alert's file_path is read during initialize and its + // contents become other fields, so the path itself is not in the state at all. + verdictNotObservable verdict = "NOT_OBSERVABLE" + // The config changed, the field diff exists, but the plan produced no action at all. + verdictNoPlan verdict = "NO_PLAN" + // The field drifts with no config change at all: right after the base deploy, the plan + // already wants to update it. Usually the read does not echo the field, so state and + // remote never agree. Recorded once per config against the field itself -- otherwise it + // makes every other field's post-deploy plan dirty and gets blamed on all of them. + verdictBaselineDrift verdict = "BASELINE_DRIFT" + // The apply succeeded and the engine sent the write, but the field's remote value is + // exactly what it was beforehand, on two consecutive reads: the backend accepted the + // request and ignored this field. The most common shape is a field being cleared, where + // the update request type is omitempty so the field never makes it into the body. + // + // Two reads, not one, is what separates this from STALE_READ. + verdictUpdateIgnored verdict = "UPDATE_IGNORED" + // The write did land, but the read straight after the apply did not show it and the one + // after that did. Not a field-support gap, yet worth seeing: a user planning right + // after a deploy is shown a change that does not exist. + verdictStaleRead verdict = "STALE_READ" + // The apply succeeded, but the plan taken straight afterwards still proposes a change + // to this field and the remote moved to something else again, so deploying never + // converges for a reason other than a plainly ignored write. + verdictDrift verdict = "POST_DEPLOY_DRIFT" + // The resource declares that it ignores local changes to this field, and it does: the + // change came back suppressed with exactly the declared reason. Recorded so the claim is + // verified rather than assumed. + verdictInertConfirmed verdict = "OK_INERT" + // The resource declares that it ignores local changes to this field, and it does not. + // Either the declaration is wrong or the engine stopped honouring it. + verdictInertViolated verdict = "INERT_NOT_HONOURED" + // The field under test converged, but updating it left some *other* field of the same + // resource drifting. The detail names that other field, which is where the fix belongs: + // one such field makes every deploy of the resource dirty regardless of what changed. + verdictCollateral verdict = "COLLATERAL_DRIFT" + // The field itself converged, but some other node of the resource is still drifted + // post-deploy. Sub-resources are stripped before planning, so nothing should reach + // this: it guards against blaming a field for drift that is not its own. + verdictDriftChild verdict = "POST_DEPLOY_DRIFT_CHILD" + + // The backend rejected the value. Usually the value library needs a valid + // value for this field rather than the generic per-kind default. + verdictBackendError verdict = "BACKEND_ERROR" + // Apply failed for a reason that is not an API rejection. + verdictDeployError verdict = "DEPLOY_ERROR" + // The operation did not finish inside the per-operation deadline. Recorded rather + // than waited out: an app rename, for one, blocks on the old name leaving DELETING. + verdictTimeout verdict = "TIMEOUT" + // Planning itself failed. + verdictPlanError verdict = "PLAN_ERROR" + + // The value could not be written into the config at all: the parent object is + // absent from this base config, or the type does not accept the value. + // The field composes the resource's ID, and the value under test is absent. There is no + // resource to create without its name, so the backend refusing the create states the API's + // contract rather than a defect. Read off the resource's own provided_id_fields declaration, + // never off the error: which field the message happens to name is not evidence, since a + // recreate carries the whole resource. + verdictIDFieldRequired verdict = "OK_ID_FIELD_REQUIRED" + + verdictUnsettable verdict = "UNSETTABLE" + // Left out on purpose by the resource's value library, with a reason. + verdictSkipped verdict = "SKIPPED" + // The transition's starting value could not be deployed, so the transition + // itself was never observed. + verdictBaseError verdict = "BASE_ERROR" + // The starting value deployed without error but the field did not end up holding it, so + // the transition under test could not be set up. A field the API refuses to clear is the + // usual case: nothing can start from absent once the field has ever been set, and a fresh + // resource whose base config declares the field cannot start from absent either. Reported + // rather than tested from the wrong starting point, which would label the result a move + // that never happened. + verdictStartNotReached verdict = "START_NOT_REACHED" +) + +// leavesResourceUsable reports whether the deployed resource is still in a known-good +// state after this outcome. When it is not, the next transition must start from a fresh +// resource, or one real failure cascades into a run of unrelated ones. +func (v verdict) leavesResourceUsable() bool { + switch v { + case verdictOK, verdictRecreate, verdictSuppressed, verdictNotObservable, verdictUnsettable, + verdictSkipped, verdictInertConfirmed, verdictInertViolated: + return true + case verdictDrift, verdictCollateral, verdictDriftChild, verdictUpdateIgnored, + verdictStaleRead, verdictBaselineDrift, verdictStartNotReached: + // The resource exists and still matches what the engine last sent; only the one + // field did not stick. The next transition re-deploys the field anyway. + // + // START_NOT_REACHED is here for a second reason: the caller has already retried it on + // a fresh resource, so rebuilding again would buy nothing. + return true + default: + return false + } +} + +// benignSuppressions are the planner's own reasons for dropping a change where nothing the +// user asked for is lost, so the outcome reads like OK and belongs in the full report only. +// +// - empty: old and new are both empty -- an unset bool against an explicit false +// - remote_already_set: the remote already holds the value the config asks for +// - backend_default: the config asks for nothing and the backend chose, by design +// +// Every other reason means a change the user expressed was dropped, which is a finding +// however defensible it is -- terraform_compat and no_update_api being the clearest. +var benignSuppressions = map[string]bool{ + "empty": true, + "remote_already_set": true, + "backend_default": true, +} + +// isProblem reports whether a result is something a person should look at. The committed +// report lists only these; the logs/.full.txt companion lists everything. +func (r result) isProblem() bool { + switch r.verdict { + case verdictOK, verdictRecreate, verdictNotObservable, verdictSkipped, verdictInertConfirmed, + verdictIDFieldRequired: + return false + case verdictSuppressed: + return !benignSuppressions[r.detail] + default: + return true + } +} + +// result is one line of the report. +type result struct { + field string + from, to string + verdict verdict + detail string + + // evidence is the raw material behind the verdict -- the post-deploy plan for drift, + // the whole API error for a rejection. Printed only into the full report, indented + // under its line: it is what you would otherwise re-run the case to see, and it is + // far too long and too full of generated ids to belong in a committed golden. + evidence string +} + +// report accumulates results for one resource type. Resource types run in parallel, but each +// owns its report and everything within a type runs in sequence, so nothing here synchronizes. +// Parallelizing the field subtests would need that to change. +type report struct { + resourceType string + + // started is when this type's run began, so the full report can say how long it took. Only in + // the full report: the committed golden has to be the same whatever the machine's speed. + started time.Time + + results []result + wildcard map[string]bool + covered map[string]bool + + // legend maps a size label back to the value it stands for, keyed by field: "keys1" means + // a different map for properties than it does for options, so the field is part of the key. + legend map[legendKey]string + + // sampled holds the fields a -sample run tested, and is nil when every field was tested. + // The report is then compared against the same subset of its golden rather than whole. + sampled map[string]bool +} + +// addSampled records that this field is one of the sampled ones. +func (r *report) addSampled(path string) { + if r.sampled == nil { + r.sampled = map[string]bool{} + } + r.sampled[path] = true +} + +// reportHeader introduces a committed report to a reader who has only the diff. +const reportHeader = `# field | from | to | verdict | detail +# +# How the direct engine handles every field of this resource type: each row is one move of one field +# between two values, "absent" included. An outcome the engine got wrong is recorded here rather than +# failed, so this file is a description of current behaviour -- what fails the test is a verdict +# changing. Passing rows are counted in the summary and listed in logs/.full.txt, along with +# the evidence behind each finding and what could not be covered. +# +# Regenerate with ./task test-update-fields. See bundle/direct/autotest/README.md for the verdicts. + +` + +// fieldColumnWidth is how wide the field column is in a rendered row. Named because sampledRows +// reads the field back out of the text by it. +const fieldColumnWidth = 44 + +// legendKey identifies one size label of one field. +type legendKey struct { + field, label string +} + +// addLegend records what a size label stands for. Only containers get one: a scalar's label is +// the value, so there is nothing to explain. +func (r *report) addLegend(field, label string, value any) { + rendered, err := json.Marshal(value) + if err != nil { + return + } + // Redacted like everything else in the report: a seeded value can name the workspace user or + // carry this run's unique suffix, both of which differ between a fake server and a real one. + if r.legend == nil { + r.legend = map[legendKey]string{} + } + r.legend[legendKey{field, label}] = redactIDs(string(rendered)) +} + +func (r *report) add(res result) { + r.results = append(r.results, res) +} + +// addCoverage records what one config reached and what it could not. A field is only +// reported as not covered when *no* config of the type reached it: several configs of one +// resource declare different blocks, so a union of the per-config gaps would list fields +// that are in fact tested -- jobs has eight configs and only some declare tasks. +func (r *report) addCoverage(covered []field, uncovered []string) { + if r.wildcard == nil { + r.wildcard = map[string]bool{} + } + if r.covered == nil { + r.covered = map[string]bool{} + } + for _, p := range uncovered { + r.wildcard[p] = true + } + for _, f := range covered { + r.covered[patternOf(f.path)] = true + } +} + +// patternOf turns a concrete path back into the pattern it came from, so a field reached +// through one config's slice cancels the same pattern reported by a config without it. +func patternOf(path string) string { + var sb strings.Builder + for i := 0; i < len(path); i++ { + if path[i] != '[' { + sb.WriteByte(path[i]) + continue + } + end := strings.IndexByte(path[i:], ']') + if end < 0 { + sb.WriteByte(path[i]) + continue + } + inner := path[i+1 : i+end] + if strings.HasPrefix(inner, "'") { + // A map key: the walk pattern for a map is ".*", not "[*]". + sb.WriteString(".*") + } else { + sb.WriteString("[*]") + } + i += end + } + return sb.String() +} + +// uncoveredPaths returns the patterns no config of this type reached. Caller holds the lock. +func (r *report) uncoveredPaths() []string { + var out []string + for path := range r.wildcard { + if !r.covered[path] { + out = append(out, path) + } + } + slices.Sort(out) + return out +} + +// render produces a report body. Problems-only is the committed form: one line per finding, +// plus the count of every verdict -- including the passing ones, which no line names. Those +// counts are what makes a change in *passing* behaviour visible: a field that starts being +// recreated instead of updated moves one OK to OK_RECREATE and nothing else would show it, +// on either the fake server or a real workspace. The full form adds every row, the evidence +// behind each finding, and what was not covered. +func (r *report) render(problemsOnly bool) string { + var sb strings.Builder + if problemsOnly { + // The committed report is read on its own in a diff, with nothing around it to say what + // the columns are. Deliberately free of counts, times and dates: a header that moved when + // something unrelated changed would be noise in every review. + sb.WriteString(reportHeader) + } + + slices.SortStableFunc(r.results, func(a, b result) int { + return cmp.Or( + strings.Compare(a.field, b.field), + strings.Compare(a.from, b.from), + strings.Compare(a.to, b.to), + ) + }) + + counts := map[verdict]int{} + // The labels the rendered rows actually use, so the legend explains those and no others. + used := map[legendKey]bool{} + for _, res := range r.results { + counts[res.verdict]++ + if problemsOnly && !res.isProblem() { + continue + } + used[legendKey{res.field, res.from}] = true + used[legendKey{res.field, res.to}] = true + line := fmt.Sprintf("%-*s %-10s %-10s %s", fieldColumnWidth, res.field, res.from, res.to, res.verdict) + if res.detail != "" { + line += " " + res.detail + } + sb.WriteString(strings.TrimRight(line, " ") + "\n") + + if !problemsOnly && res.evidence != "" { + // Indented, but with nothing prefixed: leading whitespace is insignificant to + // JSON, so the block can be copied straight out and parsed. + for evidenceLine := range strings.SplitSeq(strings.TrimRight(res.evidence, "\n"), "\n") { + sb.WriteString(" " + evidenceLine + "\n") + } + } + } + + sb.WriteString("\n=== summary\n") + for _, v := range []verdict{ + verdictOK, verdictRecreate, verdictIDFieldRequired, verdictInertConfirmed, verdictSuppressed, + verdictNotObservable, verdictNoPlan, verdictInertViolated, + verdictBaselineDrift, verdictStaleRead, verdictUpdateIgnored, verdictDrift, + verdictCollateral, verdictDriftChild, + verdictBackendError, verdictDeployError, verdictTimeout, verdictPlanError, + verdictUnsettable, verdictSkipped, verdictBaseError, + verdictStartNotReached, + } { + if counts[v] > 0 { + fmt.Fprintf(&sb, "%-18s %d\n", v, counts[v]) + } + } + + if problemsOnly { + return sb.String() + } + + fmt.Fprintf(&sb, "\n=== elapsed\n%s\n", r.elapsed().Round(time.Millisecond)) + + // The legend goes here and not into the committed report: it is JSON, so it would churn + // whenever an SDK type gains a field, and a reader decoding a label is already looking at + // this file for the evidence behind the row. + if legend := r.renderLegend(used); legend != "" { + sb.WriteString("\n=== values\n") + sb.WriteString(legend) + } + + if gaps := r.uncoveredPaths(); len(gaps) > 0 { + fmt.Fprintf(&sb, "\n=== not covered: %d fields with nothing to test\n", len(gaps)) + fmt.Fprintf(&sb, "# either no config of this type declares the slice or map they sit in,\n") + fmt.Fprintf(&sb, "# or their type has no generic value and the value library gives them none\n") + for _, p := range gaps { + fmt.Fprintf(&sb, "%s\n", p) + } + } + + return sb.String() +} + +// renderLegend spells out the size labels the rows used. Caller holds the lock. +func (r *report) renderLegend(used map[legendKey]bool) string { + keys := make([]legendKey, 0, len(used)) + for key := range used { + if _, ok := r.legend[key]; ok { + keys = append(keys, key) + } + } + slices.SortFunc(keys, func(a, b legendKey) int { + return cmp.Or(strings.Compare(a.field, b.field), strings.Compare(a.label, b.label)) + }) + + var sb strings.Builder + for _, key := range keys { + fmt.Fprintf(&sb, "%-44s %-10s %s\n", key.field, key.label, r.legend[key]) + } + return sb.String() +} + +// write compares the findings against the committed golden. There is one golden, and both +// the fake server and a real workspace are held to it: a divergence means the fake server +// does not model the API faithfully, which is worth failing over rather than filing away in +// a per-cloud file nobody diffs. +// +// -update rewrites it, so a cloud run can be used to correct a golden the fake server got +// wrong. The full form is never compared -- it is an aid for reading one run, and it moves +// whenever a passing row moves. +func (r *report) write(t testutil.TestingT) { + name := reportPath(r.resourceType + ".txt") + body := r.render(true) + + fullName := r.resourceType + ".full.txt" + if isCloud() { + // Keep both around when comparing a cloud run against the fake server. + fullName = r.resourceType + "." + cloudName() + ".full.txt" + } + writeReport(t, logPath(fullName), r.render(false)) + + if testdiff.OverwriteMode { + writeReport(t, name, body) + return + } + + // A sampled run knows about a few of the type's fields, so its report is held to those + // fields' rows and nothing else. The rows this run produced are filtered structurally, and + // the golden -- which is only text -- by the field column. The summary counts every field + // and has no meaningful subset, so both sides drop it. + if r.sampled != nil { + expected, err := os.ReadFile(name) + if err != nil { + t.Errorf("reading %s: %s (run ./task test-update-fields)", name, err) + return + } + want := sampledRows(testdiff.NormalizeNewlines(string(expected)), r.sampled) + testdiff.AssertEqualTexts(t, name, name, want, sampledRows(body, r.sampled)) + return + } + + // Not tolerated as missing: a report with no golden would silently pass, so adding a + // resource type without generating its report, or deleting one, would go unnoticed. + expected, err := os.ReadFile(name) + if err != nil { + t.Errorf("reading %s: %s (run ./task test-update-fields)", name, err) + return + } + testdiff.AssertEqualTexts(t, name, name, testdiff.NormalizeNewlines(string(expected)), body) +} + +// collateralCause returns the field a COLLATERAL_DRIFT row blames, which the renderer puts in the +// detail column. Reports false for any other row. +func collateralCause(line string) (string, bool) { + marker := " " + string(verdictCollateral) + " " + _, after, ok := strings.Cut(line, marker) + if !ok { + return "", false + } + // The detail is a comma-separated list of drifting paths; the first is enough to decide + // whether this run had any business reproducing the row. + detail := strings.TrimSpace(after) + cause, _, _ := strings.Cut(detail, ",") + return cause, true +} + +// rowField reports whether a rendered row belongs to a sampled field, or to the harness itself. +// +// Matched by prefix rather than by reading a fixed-width column: a field path can be far longer +// than the column (a pipeline's ingestion_definition reaches 130 characters), and the column then +// holds a truncated path that matches nothing -- silently dropping the row from both sides, which +// is the one thing this comparison must not do. +func rowField(line string, sampled map[string]bool) bool { + // A row the harness records against itself -- "(base config)", "(rebuild)", "(baseline)" -- + // says the run for that type did not happen, so no sample may hide it. + if strings.HasPrefix(line, "(") { + return true + } + for field := range sampled { + // The separator matters: without it "tags" would also claim "tags_extra". + if strings.HasPrefix(line, field+" ") { + return true + } + } + return false +} + +// sampledRows keeps the report rows belonging to the sampled fields, dropping the summary and +// blank lines. Rows the harness records against itself rather than a field -- "(base config)", +// "(rebuild)", "(baseline)" -- are always kept: they mean the run for that type did not happen, +// which no sample should be allowed to hide. +// +// The field is read off by column width rather than by cutting at the first space: a map key can +// contain one ("tags['my team']"), and cutting there would compare half a path. +func sampledRows(body string, sampled map[string]bool) string { + var sb strings.Builder + for line := range strings.SplitSeq(body, "\n") { + if strings.HasPrefix(line, "=== summary") { + break + } + if line == "" { + continue + } + // COLLATERAL_DRIFT is drift the plan attributes to another field, which its detail names. + // A run that did not sample that other field cannot reproduce the row, so it is dropped -- + // but only then. Dropping every one of them would hide a sampled field turning from OK + // into COLLATERAL_DRIFT, which is a regression like any other. + if cause, ok := collateralCause(line); ok && !sampled[cause] { + continue + } + if rowField(line, sampled) { + sb.WriteString(line + "\n") + } + } + return sb.String() +} + +func writeReport(t testutil.TestingT, name, body string) { + if err := os.MkdirAll(filepath.Dir(name), 0o755); err != nil { + t.Errorf("creating %s: %s", filepath.Dir(name), err) + return + } + // The previous copy of a log is kept next to the new one: reading what a change did to a full + // report means diffing the two, and it is not in git to diff against. Not for a committed + // golden -- git is its history, and a .backup beside it would be an untracked file in the one + // directory that holds only tracked ones. + if strings.HasPrefix(name, logsDir+string(os.PathSeparator)) { + if previous, err := os.ReadFile(name); err == nil { + if err := os.WriteFile(name+".backup", previous, 0o644); err != nil { + t.Errorf("backing up %s: %s", name, err) + return + } + } + } + if err := os.WriteFile(name, []byte(body), 0o644); err != nil { + t.Errorf("writing %s: %s", name, err) + } +} + +// outputDir holds the committed reports and nothing else, so a change to one is visible in a diff. +// Kept out of the package directory so the Go files stay readable next to three dozen goldens. +const outputDir = "output" + +// logsDir holds what a run produces for a reader rather than for review: the full report, and the +// cloud run's copy of it. Ignored wholesale, so nothing here can be mistaken for a golden. +const logsDir = "logs" + +func reportPath(name string) string { + return filepath.Join(outputDir, name) +} + +// elapsed is how long this type's run took, or zero when the report was built without a start time +// (the unit tests construct one directly). +func (r *report) elapsed() time.Duration { + if r.started.IsZero() { + return 0 + } + return time.Since(r.started) +} + +func logPath(name string) string { + return filepath.Join(logsDir, name) +} + +func cloudName() string { + switch env := os.Getenv("CLOUD_ENV"); env { + case "ucws": + return "aws" + case "gcp-ucws": + return "gcp" + default: + return env + } +} diff --git a/bundle/direct/autotest/testdata/fields/alerts.yml b/bundle/direct/autotest/testdata/fields/alerts.yml new file mode 100644 index 00000000000..94e403cf6be --- /dev/null +++ b/bundle/direct/autotest/testdata/fields/alerts.yml @@ -0,0 +1,47 @@ +# Value library for alerts. See ../../README.md. + +# The invariant config declares neither block, so their fields would only ever be reported +# as not covered. Seeding one entry each makes both testable: the containers themselves and +# the fields inside an entry. +base: + warehouse_id: $TEST_DEFAULT_WAREHOUSE_ID + display_name: test-alert-$UNIQUE_NAME + file_path: ./alert.dbalert.json + +skip: + # An alert built from a .dbalert.json takes everything but warehouse_id, display_name and + # file_path from that file: load_dbalert_files rejects any other field in the bundle YAML + # ("field X is not allowed in the bundle configuration"). So these are not bundle fields at + # all, and the values the suite used to seed reached the resource only by being patched in + # after that mutator had run. + evaluation: settable only in the .dbalert.json, not in the bundle + parameters: settable only in the .dbalert.json, not in the bundle + + # Must name an existing SQL warehouse, and alerts require one, so there is no absent + # transition either -- a value-to-value move would need a second warehouse to exist. + warehouse_id: needs a second SQL warehouse to exist + # Must name an existing workspace folder, and the backend refuses to move an alert + # between folders at all ("Updating parent path is not supported"). + parent_path: needs a second workspace folder; the backend refuses to move an alert + # Must be the id of a notification destination registered with the workspace. + evaluation.notification.subscriptions[*].destination_id: needs a notification destination + + # A threshold compares against either a column or a value, and the alert file this config + # loads (data/alert.dbalert.json) sets the value -- so the column is the other arm of that + # choice and the backend drops it, leaving it drifting forever. The value's own three + # members (bool, double, string) are a oneof in the same way. + evaluation.threshold: two arms of one choice, and the alert file already picks the value + # Also a oneof (user_name or service_principal_name), and which member is valid depends on + # whether the identity running the suite is a user or a service principal. run_as_user_name + # is the deprecated flat form of the same thing: setting it has no effect when the caller is + # a service principal, which is what a cloud run is. + run_as: a oneof, and which member is valid depends on the run identity + run_as_user_name: only settable when the caller is a user, not a service principal + # An alert has to be able to notify somebody, so the backend will not leave a subscription + # without a recipient -- but what it keeps instead is not something this suite can predict. + evaluation.notification.subscriptions[*].user_email: the backend refuses to leave a subscription without a recipient + +fields: + # A real zone, and a real cron expression -- neither is free text. + schedule.timezone_id: [UTC, America/New_York] + schedule.quartz_cron_schedule: ["0 0 9 10 1 ?", "0 0 10 10 1 ?"] diff --git a/bundle/direct/autotest/testdata/fields/apps.yml b/bundle/direct/autotest/testdata/fields/apps.yml new file mode 100644 index 00000000000..bd23620e245 --- /dev/null +++ b/bundle/direct/autotest/testdata/fields/apps.yml @@ -0,0 +1,73 @@ +# Value library for apps. See ../../README.md. +# +# resources is deliberately not seeded. An entry is a union (secret, job, sql_warehouse, ...) +# and the API rejects one carrying no member at all -- "Invalid resource provided" -- while +# every member names a workspace object this suite does not provision: a secret scope, a job, a +# warehouse. So the block's fields stay uncovered, which the report says. + +# base declares neither block. config carries the app's command and env; the +# resources list is a union where an entry names one referenced object. +base: + name: app-$UNIQUE_NAME + source_code_path: ./app + config: + command: + - python + - app.py + env: + - name: MY_VAR + value: value + +fields: + # Real OAuth scopes, verified against the API: it refuses anything else outright ("The specified + # scope x is not a valid scope"), and the mock server takes any string. Declared for the list as + # well as the element, so growing the list uses scopes the backend accepts. + user_api_scopes: + - [sql] + - [dashboards.genie, sql] + user_api_scopes[*]: [sql, dashboards.genie] + + compute_size: [MEDIUM, LARGE] + +skip: + # Starting an app deploys its source, and "Invalid source code path: .../files/app. Path does not + # exist": this suite uploads no bundle files -- a sync per permutation would dominate the run -- so + # the directory the app points at is never there. The mock server does not check, which is the only + # reason starting one ever appeared to work. + lifecycle.started: starting the app deploys source this suite does not upload + + # "Setting forward_user_access_token on CreateApp is not enabled in this workspace." A + # workspace-level feature, so the local rows for it record what the mock server allows and not + # what a workspace does. + forward_user_access_token: needs the feature enabled on the workspace + + # "git_repository.caller_credential_id can only be set on CreateApp, not on UpdateApp." Every + # transition of it is an update, so none is a request the API accepts; reaching it would need a + # fixture that varies the field across creates. + git_repository.caller_credential_id: settable only on create, and every transition is an update + + # "Git repository requires both URL and provider to be defined" -- the block is validated as a set, + # and this leaf cannot move without the other two beside it. Same shape as the url and provider + # skips below. + git_repository.auto_deploy: validated as a set with the url and provider, which base does not declare + + + # Both name an account-level policy: "Invalid resource: accounts//budgetPolicies/x". A + # test workspace has none to reference, and creating one is an account-level operation. + budget_policy_id: needs an account-level budget policy to exist + usage_policy_id: needs an account-level usage policy to exist + + # The API validates the block as a set: "Git repository requires both URL and provider to be + # defined." Moving either leaf on its own leaves the other unset, so neither is testable until + # the fixture declares a coherent block for them to vary inside -- the same shape as a job's + # git_source, which base seeds for exactly that reason. + git_source.git_repository.url: validated as a set with the provider, which the fixture does not declare + git_source.git_repository.provider: validated as a set with the url, which the fixture does not declare + git_repository.url: validated as a set with the provider, which the fixture does not declare + git_repository.provider: validated as a set with the url, which the fixture does not declare + + # "Manual instance count configuration is not enabled in this workspace." A workspace-level + # feature, so the local report for these two records what the fake server allows, not what a + # workspace does. + compute_max_instances: needs manual instance count enabled on the workspace + compute_min_instances: needs manual instance count enabled on the workspace diff --git a/bundle/direct/autotest/testdata/fields/catalogs.yml b/bundle/direct/autotest/testdata/fields/catalogs.yml new file mode 100644 index 00000000000..600e387d4eb --- /dev/null +++ b/bundle/direct/autotest/testdata/fields/catalogs.yml @@ -0,0 +1,33 @@ +# Value library for catalogs. See ../../README.md. + +# properties is a map the config does not declare, so its entries would be unreachable. +base: + name: test-catalog-$UNIQUE_NAME + comment: This is a test catalog + properties: + team: eng + +skip: + # options only applies to a foreign catalog, which needs a real connection, and + # UpdateCatalog rejects the field outright ("UpdateCatalog options can not be + # provided") -- so seeding it made every *other* field of the catalog fail its update + # too. Worth a look on its own: the engine sends options on update whenever the config + # declares them, which a foreign catalog cannot survive. + options: rejected by UpdateCatalog; only settable at create on a foreign catalog + # A Delta Sharing catalog needs both of these and a share that exists. + provider_name: needs a Delta Sharing provider and share + share_name: needs a Delta Sharing provider and share + # A foreign catalog needs a connection that exists. + connection_name: needs a connection + # Must point at a real external location. + storage_root: needs an external location + # Needs a customer-managed key registered with the workspace; every field of the block + # is a UUID or key id the backend validates against that registration. + managed_encryption_settings: needs a customer-managed key registered with the workspace + # A UUID naming the workspace's metastore, which the backend fills in. + metastore_id: assigned by the backend + +fields: + # The backend takes hours but validates them as days: the retention period must be 0 or + # between 7 and 30 days, so the generic 1 and 2 are rejected. + custom_max_retention_hours: [168, 720] diff --git a/bundle/direct/autotest/testdata/fields/cluster_policies.yml b/bundle/direct/autotest/testdata/fields/cluster_policies.yml new file mode 100644 index 00000000000..2e34a58519a --- /dev/null +++ b/bundle/direct/autotest/testdata/fields/cluster_policies.yml @@ -0,0 +1,41 @@ +# Value library for cluster policies. See ../../README.md. + +# libraries is a union: an entry carries exactly one of jar, egg, whl, pypi, maven or cran. +# One entry, so the container and that member are testable. The other members stay uncovered +# for this config, which the report says: they can only be reached by putting a second member +# on this entry, which is not something a user can write -- the backend keeps whichever it +# prefers and the whole block then reads as drifting. +base: + name: test-cluster-policy-$UNIQUE_NAME + definition: '{"spark_version":{"type":"fixed","value":"13.3.x-scala2.12"}}' + libraries: + - jar: dbfs:/FileStore/test.jar + +skip: + # The other members of the union. Reaching one means putting a second artifact on the entry + # that already holds the jar, which is not a valid write: the backend keeps one of them and + # the whole block then reads as drifting. Covering them needs a config whose libraries entry + # holds that member instead. + libraries[*].egg: another member of the same union as the seeded jar + libraries[*].whl: another member of the same union as the seeded jar + libraries[*].requirements: another member of the same union as the seeded jar + libraries[*].pypi: another member of the same union as the seeded jar + libraries[*].maven: another member of the same union as the seeded jar + libraries[*].cran: another member of the same union as the seeded jar + + # A policy takes its rules from either its own definition or a policy family, never both, + # and this config sets a definition. A family also has to be one the workspace offers. + policy_family_id: mutually exclusive with the definition this config sets + policy_family_definition_overrides: only meaningful with a policy family + policy_family_version: only meaningful with a policy family + +fields: + # A library path, not free text. Clearing it is still tried, and leaves the entry with no + # artifact at all, which the API rejects. + libraries[*].jar: [dbfs:/FileStore/a.jar, dbfs:/FileStore/b.jar] + + # A policy definition is a JSON document the backend parses, and it is required: the API + # rejects an update that does not carry one. + definition: + - '{"spark_version":{"type":"fixed","value":"13.3.x-scala2.12"}}' + - '{"spark_version":{"type":"fixed","value":"14.3.x-scala2.12"}}' diff --git a/bundle/direct/autotest/testdata/fields/clusters.yml b/bundle/direct/autotest/testdata/fields/clusters.yml new file mode 100644 index 00000000000..212fef219ff --- /dev/null +++ b/bundle/direct/autotest/testdata/fields/clusters.yml @@ -0,0 +1,59 @@ +# Value library for clusters. See ../../README.md. + +# init_scripts is a union: an entry carries exactly one of workspace, volumes, s3, abfss, +# gcs, dbfs or file. Only the workspace member is seeded -- every other one names +# cloud-specific storage, and the suite runs against all three clouds -- so the rest stay +# uncovered for this config, which the report says. +base: + cluster_name: test-cluster-$UNIQUE_NAME + spark_version: 13.3.x-scala2.12 + node_type_id: $NODE_TYPE_ID + instance_pool_id: $TEST_INSTANCE_POOL_ID + num_workers: 1 + custom_tags: + team: eng + spark_conf: + spark.speculation: "true" + spark_env_vars: + MY_VAR: value + ssh_public_keys: + - ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQtest + +fields: + # The API refuses anything between 1 and 9: "The cluster autotermination time cannot be less than + # 10 minutes." 0 disables it and is accepted too, but the CLI defaults the field to 60, so 0 is + # what a user writes to turn it off rather than a second ordinary value. + autotermination_minutes: [10, 20] + +skip: + # An init script has to name a file that exists: a real workspace fails the cluster with "Init + # scripts failed ... Tree node with path ... does not exist". This suite deliberately uploads no + # bundle files -- a sync per permutation would dominate the run -- so no destination it could name + # is there, and the mock server's acceptance of any path is what made the field look testable. + init_scripts: an init script has to name a file that exists, and this suite uploads none + # "is_single_node is not allowed with unspecified kind." The field is only meaningful on a cluster + # whose kind says it is one, and this fixture's cluster has no kind, so no value of it is valid + # here. The mock server stores it either way, which is what made the field look testable. + is_single_node: only allowed on a cluster with a kind, which this fixture does not set + + # A cluster policy has to exist: "'x' is not a valid cluster policy ID." The fake server takes any + # string, so these rows only ever measured the fake. + policy_id: needs a cluster policy to exist + # An init script entry carries exactly one of workspace, volumes, s3, abfss, gcs, dbfs or file, + # and base seeds the workspace member. Setting a leaf inside another arm leaves that arm without + # its own destination, which the backend accepts and then ignores -- every such row was recording + # the fake server storing a value the API drops. Covering an arm needs a fixture that seeds it, + # not a value for a leaf inside one that is not there. + init_scripts[*].abfss: an init script member the seeded workspace member excludes + init_scripts[*].dbfs: an init script member the seeded workspace member excludes + init_scripts[*].file: an init script member the seeded workspace member excludes + init_scripts[*].gcs: an init script member the seeded workspace member excludes + init_scripts[*].s3: an init script member the seeded workspace member excludes + init_scripts[*].volumes: an init script member the seeded workspace member excludes + + # "The field 'worker_node_type_flexibility' cannot be supplied when an instance pool ID is + # provided", and base provides one. The alternates would also have to name a + # second node type, which differs per cloud. + worker_node_type_flexibility: incompatible with the instance pool the config sets + driver_node_type_flexibility: incompatible with the instance pool the config sets + node_type_flexibility: incompatible with the instance pool the config sets diff --git a/bundle/direct/autotest/testdata/fields/dashboards.yml b/bundle/direct/autotest/testdata/fields/dashboards.yml new file mode 100644 index 00000000000..f36565046cd --- /dev/null +++ b/bundle/direct/autotest/testdata/fields/dashboards.yml @@ -0,0 +1,26 @@ +# Value library for dashboards. See ../../README.md. + +base: + warehouse_id: $TEST_DEFAULT_WAREHOUSE_ID + display_name: test-dashboard-$UNIQUE_NAME + file_path: ./dashboard.lvdash.json + +skip: + # Must name a workspace folder that exists, and this suite provisions only the bundle's own. + # A dashboard also cannot be moved between folders without being recreated, and the recreate + # then fails against a folder that is not there. + parent_path: needs a second workspace folder to exist + # Must name an existing SQL warehouse, and dashboards need one, so a value-to-value move + # would need a second warehouse. + warehouse_id: needs a second SQL warehouse to exist + + # A concurrency token the backend assigns: it can be sent on an update to say which version + # was last read, but its value is never the caller's. genie_spaces declares the same field + # spec:output_only in resources.generated.yml and dashboards does not, which is an asymmetry + # in the generated annotations rather than anything this suite can settle. + etag: a concurrency token whose value the backend assigns + + # Query parameters on the create call (json:"-", url:"dataset_catalog"), not part of the + # dashboard, so the backend never echoes them and nothing can be observed about them here. + dataset_catalog: a query parameter on create, never returned by a read + dataset_schema: a query parameter on create, never returned by a read diff --git a/bundle/direct/autotest/testdata/fields/database_catalogs.yml b/bundle/direct/autotest/testdata/fields/database_catalogs.yml new file mode 100644 index 00000000000..96d8af042f6 --- /dev/null +++ b/bundle/direct/autotest/testdata/fields/database_catalogs.yml @@ -0,0 +1,15 @@ +# Value library for database catalogs. See ../../README.md. + +# Lakebase v1 is not available in every cloud, same as the instance this catalog needs. +local_only: Lakebase v1 is not available in every cloud + +deps: + database_instances: + name: test-db-instance-$UNIQUE_NAME + capacity: CU_1 + +base: + database_instance_name: ${resources.database_instances.database_instances.name} + database_name: test_db + name: test-catalog-$UNIQUE_NAME + create_database_if_not_exists: true diff --git a/bundle/direct/autotest/testdata/fields/database_instances.yml b/bundle/direct/autotest/testdata/fields/database_instances.yml new file mode 100644 index 00000000000..0932f25ff0f --- /dev/null +++ b/bundle/direct/autotest/testdata/fields/database_instances.yml @@ -0,0 +1,11 @@ +# Value library for database instances. See ../../README.md. + +# Lakebase v1 (database instances, catalogs and synced tables) is not available +# on every cloud, and the invariant suite excludes it from every cloud run for that reason. +local_only: Lakebase v1 is not available in every cloud +base: + name: test-db-instance-$UNIQUE_NAME + capacity: CU_1 + custom_tags: + - key: team + value: eng diff --git a/bundle/direct/autotest/testdata/fields/experiments.yml b/bundle/direct/autotest/testdata/fields/experiments.yml new file mode 100644 index 00000000000..b2f3490820c --- /dev/null +++ b/bundle/direct/autotest/testdata/fields/experiments.yml @@ -0,0 +1,26 @@ +# Value library for experiments. See ../../README.md. + +# The invariant config declares no tags, so tags[*] would expand to nothing and its fields +# would only be reported as not covered. Seeding one entry makes the whole block testable: +# the container itself (add and remove an entry) and the fields inside the entry. +# +# The UC trace location is the same case as a job's git_source: the backend requires the +# catalog and the schema together, so neither can be reached one field at a time. +base: + name: /Users/$CURRENT_USER_NAME/test-experiment-$UNIQUE_NAME + tags: + - key: team + value: eng + trace_location: + uc_trace_location: + catalog: main + schema: default + +fields: + # An experiment name is an absolute workspace path, so the generic "x" is rejected. + name: [/Shared/test-experiment-a, /Shared/test-experiment-b] + # The artifact location needs a scheme. + artifact_location: [dbfs:/tmp/mlflow-a, dbfs:/tmp/mlflow-b] + # See the note in volumes.yml: a reference has to name something that exists. + trace_location.uc_trace_location.catalog: [main] + trace_location.uc_trace_location.schema: [default] diff --git a/bundle/direct/autotest/testdata/fields/external_locations.yml b/bundle/direct/autotest/testdata/fields/external_locations.yml new file mode 100644 index 00000000000..c8e9cba360c --- /dev/null +++ b/bundle/direct/autotest/testdata/fields/external_locations.yml @@ -0,0 +1,13 @@ +# Value library for external locations. See ../../README.md. + +# An external location points at cloud storage through a storage credential, which needs +# cloud IAM set up for the workspace. base names a credential the mock server +# invents, so the whole type is local-only -- the same reason the invariant suite excludes +# this config from its cloud run. +base: + name: test_location_$UNIQUE_NAME + url: s3://test-bucket/path + credential_name: test_storage_credential + comment: "Test external location from DABs" + +local_only: needs a storage credential with cloud IAM behind it diff --git a/bundle/direct/autotest/testdata/fields/genie_spaces.yml b/bundle/direct/autotest/testdata/fields/genie_spaces.yml new file mode 100644 index 00000000000..98e7863fe1d --- /dev/null +++ b/bundle/direct/autotest/testdata/fields/genie_spaces.yml @@ -0,0 +1,18 @@ +# Value library for Genie spaces. See ../../README.md. + +base: + warehouse_id: $TEST_DEFAULT_WAREHOUSE_ID + title: test-genie-space-$UNIQUE_NAME + # Structured (inline) serialized_space is marshalled to a JSON string by + # ConfigureGenieSpaceSerializedSpace; this config doubles as a regression + # guard that the normalization produces a drift-free deploy. Kept minimal + # ({version: 1}) so the real backend accepts it on cloud: it rejects + # unknown fields such as display_name with INVALID_PARAMETER_VALUE. + serialized_space: + version: 1 + +skip: + # Same two as dashboards: both name workspace objects this suite does not provision a second + # of. See dashboards.yml. + parent_path: needs a second workspace folder to exist + warehouse_id: needs a second SQL warehouse to exist diff --git a/bundle/direct/autotest/testdata/fields/instance_pools.yml b/bundle/direct/autotest/testdata/fields/instance_pools.yml new file mode 100644 index 00000000000..337b27d4d1c --- /dev/null +++ b/bundle/direct/autotest/testdata/fields/instance_pools.yml @@ -0,0 +1,26 @@ +# Value library for instance pools. See ../../README.md. + +# Most of a pool's spec is immutable, so most transitions recreate it -- and a real workspace +# refuses the delete half ("Can't delete pool with id: ..."), which this suite hits on every +# one of them. A plain deploy and destroy works, which is what the invariant suite does; a +# thousand recreates in a row does not, so the catalog for this type is the local one. +local_only: a real workspace refuses to delete a pool, and most fields recreate +base: + instance_pool_name: test-instance-pool-$UNIQUE_NAME + node_type_id: $NODE_TYPE_ID + custom_tags: + team: eng + preloaded_spark_versions: + - $DEFAULT_SPARK_VERSION + +skip: + # A preloaded image needs custom containers enabled for the workspace, which a test + # workspace does not have ("Custom containers is turned off for your deployment"). + preloaded_docker_images: needs custom containers enabled for the workspace + # Naming a second node type, which differs per cloud and the suite has no variable for. + node_type_flexibility: needs a second node type, which differs per cloud + # Only one of aws_/azure_/gcp_attributes applies, decided by the workspace's cloud, and the + # backend fills in defaults for the one that does -- so a value set here cannot be cleared. + aws_attributes: cloud-specific; only one of the three attribute blocks applies + azure_attributes: cloud-specific; only one of the three attribute blocks applies + gcp_attributes: cloud-specific; only one of the three attribute blocks applies diff --git a/bundle/direct/autotest/testdata/fields/job_runs.yml b/bundle/direct/autotest/testdata/fields/job_runs.yml new file mode 100644 index 00000000000..32214831823 --- /dev/null +++ b/bundle/direct/autotest/testdata/fields/job_runs.yml @@ -0,0 +1,22 @@ +# Value library for job runs. See ../../README.md. + +# Deploying a job run actually runs the job, and the wait needs a task a workspace will execute -- +# serverless keeps that to about a minute. The catalog moves a field hundreds of times, so this type +# is driven against the fake server only; a real workspace would spend a run per transition. +local_only: every transition would start a real job run + +deps: + jobs: + name: test-job-$UNIQUE_NAME + tasks: + - task_key: only_task + spark_python_task: + python_file: ./job_run.py + environment_key: default + environments: + - environment_key: default + spec: + environment_version: "2" + +base: + job_id: ${resources.jobs.jobs.id} diff --git a/bundle/direct/autotest/testdata/fields/jobs.yml b/bundle/direct/autotest/testdata/fields/jobs.yml new file mode 100644 index 00000000000..b05f3707743 --- /dev/null +++ b/bundle/direct/autotest/testdata/fields/jobs.yml @@ -0,0 +1,251 @@ +# Value library for jobs. See ../../README.md. + +# git_source only validates as a coherent set: a provider, a url, and exactly one of +# branch/tag/commit. Seeding one makes its fields reachable, so changing the commit or the +# provider is tested rather than skipped. +# +# job_clusters, triggers and environments would otherwise be absent entirely, so their +# fields would go untested; seeding one entry each makes them reachable. +base: + name: test-job-$UNIQUE_NAME + git_source: + git_provider: gitHub + git_url: https://github.com/databricks/cli.test + git_commit: abc123 + tasks: + - task_key: seeded + notebook_task: + # Relative, because the seeded git_source makes the task read from the repository: + # "Only relative paths are currently supported for remote repositories." + notebook_path: notebook + job_cluster_key: seeded + job_clusters: + - job_cluster_key: seeded + new_cluster: + spark_version: 13.3.x-scala2.12 + node_type_id: $NODE_TYPE_ID + num_workers: 1 + custom_tags: + team: eng + spark_conf: + spark.speculation: "true" + environments: + - environment_key: seeded + spec: + client: "1" + dependencies: + - requests + parameters: + - name: table + default: users + tags: + team: eng + health: + rules: + - metric: RUN_DURATION_SECONDS + op: GREATER_THAN + value: 3600 + webhook_notifications: + on_failure: + - id: 00000000-0000-0000-0000-000000000000 + trigger: + pause_status: UNPAUSED + periodic: + interval: 1 + unit: HOURS + +skip: + # Clearing the name does not converge on a real workspace. The CLI defaults a nameless job to + # Untitled (resourcemutator's defaults table), the remote ends up holding that, and the plan keeps + # proposing an update afterwards -- so the transition to absent never settles. The mock server + # stores the cleared value and converges, which is the divergence. The other five transitions of + # the field are fine on both, so only absent is out of reach. + name: clearing it leaves the remote at the CLI default Untitled, and the plan never converges + + # --- Fields a real workspace rejects. The mock server accepts every one of them, so each row below + # had only ever measured the mock server. The API's own words are quoted in each comment. + + # Invalid trigger settings. One of schedule, trigger or continuous can be set -- and base seeds a + # trigger, so no field of continuous is reachable without removing that first. + continuous: one of schedule, trigger or continuous, and base declares a trigger + + # Metadata file path must begin with a slash, so no generic value is one. + deployment.metadata_file_path: must be an absolute path, which no generic value is + + # An environment is validated as a set -- either base environment or version must be specified, and + # specifying both is refused. No one of the three can move on its own. + environments[*].spec.base_environment: validated as a set with client and environment_version + environments[*].spec.client: validated as a set with base_environment and environment_version + environments[*].spec.environment_version: validated as a set with base_environment and client + + # Git branch, tag, or commit must be specified -- clearing the commit leaves no ref at all, and the + # other two members of that oneof are already skipped. + git_source.git_commit: clearing it leaves git_source with no ref, which the API refuses + + # Job cluster seeded is not defined in the field job_clusters -- the task names the key, so the list + # cannot move unless the task moves with it. + job_clusters: the task names a key in it, which cannot change at the same time + + # Cluster-level constraints. A job cluster is created with no kind and no size of its own, so + # several of these can never hold on this fixture. + job_clusters[*].new_cluster.autoscale.min_workers: the maximum must be greater, and max does not move with it + job_clusters[*].new_cluster.autotermination_minutes: automated clusters do not support autotermination + job_clusters[*].new_cluster.cluster_log_conf.dbfs.destination: needs a dbfs path, which no generic value is + job_clusters[*].new_cluster.cluster_log_conf.volumes.destination: needs an access mode this cluster does not have + job_clusters[*].new_cluster.is_single_node: not allowed with unspecified kind, and a job cluster has none + job_clusters[*].new_cluster.use_ml_runtime: not allowed with unspecified kind, and a job cluster has none + job_clusters[*].new_cluster.num_workers: clearing it leaves the cluster with no size, which the API refuses + job_clusters[*].new_cluster.spark_version: required, so clearing it is refused + job_clusters[*].new_cluster.remote_disk_throughput: an unrecognized field on this workspace + job_clusters[*].new_cluster.total_initial_remote_disk_size: an unrecognized field on this workspace + + # A shared job cluster is only supported for multi-task jobs, so the list cannot shrink to one task + # while a job_cluster_key is in play. + tasks: a shared job cluster needs more than one task, and base seeds one of each + + # compute in task settings cannot be used with job_cluster_key, which base seeds. + tasks[*].compute.hardware_accelerator: cannot be set on a task that names a job_cluster_key + + # Only an absolute notebook path is accepted, so no generic value is one. + tasks[*].notebook_task.source: needs an absolute notebook path, which no generic value is + + # A Python operator task does not need a command, and the API refuses one. + tasks[*].python_operator_task.main: the API refuses a command on this task type + + # The duration warning threshold must not be greater than the timeout, and base seeds a rule whose + # threshold is above every timeout the suite would try. + timeout_seconds: must exceed the seeded health rule warning threshold + + # A webhook id has to be a valid UUID naming a destination that exists. + webhook_notifications.on_failure[*].id: needs a notification destination to exist + + # --- Fields the API accepts and then does not apply. Recorded here rather than as rows, because the + # mock server does apply them: a local row would say OK where a workspace says the write had no + # effect. Modelling each in libs/testserver is the real fix; until then the reason is the finding. + edit_mode: the API accepts the write and does not apply it + format: the API accepts the write and does not apply it + health.rules: the API accepts the write and does not apply it + max_concurrent_runs: the API accepts the write and does not apply it + parent_path: the API accepts the write and does not apply it + trigger.pause_status: the API accepts the write and does not apply it + # Not seeded in base either: the create does not apply it, so the job drifts from the moment it + # exists and every other field is then measured against a resource already pending. + email_notifications: the API accepts the write and does not apply it + tasks[*].email_notifications: the API accepts the write and does not apply it + tasks[*].disable_auto_optimization: the API accepts the write and does not apply it + # An init script has to name a file that exists: a real workspace fails the cluster with "Init + # scripts failed ... Tree node with path ... does not exist". This suite deliberately uploads no + # bundle files -- a sync per permutation would dominate the run -- so no destination it could name + # is there, and the mock server's acceptance of any path is what made the field look testable. + job_clusters[*].new_cluster.init_scripts: an init script has to name a file that exists, and this suite uploads none + # git_branch and git_tag are the other two sides of the same oneof as git_commit, which + # base sets. Adding one alongside it is rejected for being ambiguous, not for anything + # this suite measures. Covered by acceptance/bundle/resources/jobs. + git_source.git_branch: mutually exclusive with the seeded git_commit + git_source.git_tag: mutually exclusive with the seeded git_commit + + # A task's type is a union and base seeds notebook_task, so reaching any other member means + # putting two types on one task -- which the API rejects, usually by complaining about the + # cluster the seeded task carries ("For each does not need a cluster"). Locally the fake server + # accepted it, so ~2200 transitions were exercising configurations no user can write. + # + # Covering a member needs a fixture whose task is of that type. Some exist in the acceptance corpus -- + # job_apply_policy_default_values_for_each_task.yml.tmpl is one -- but this suite drives the + # simplest config per resource type, so they are not the one picked. Their fields are listed as + # uncovered rather than tested against a task that cannot hold them. + tasks[*].ai_runtime_task: another member of the same union as the seeded notebook_task + tasks[*].alert_task: another member of the same union as the seeded notebook_task + tasks[*].clean_rooms_notebook_task: another member of the same union as the seeded notebook_task + tasks[*].condition_task: another member of the same union as the seeded notebook_task + tasks[*].dashboard_task: another member of the same union as the seeded notebook_task + tasks[*].dbt_cloud_task: another member of the same union as the seeded notebook_task + tasks[*].dbt_platform_task: another member of the same union as the seeded notebook_task + tasks[*].dbt_task: another member of the same union as the seeded notebook_task + tasks[*].for_each_task: another member of the same union as the seeded notebook_task + tasks[*].gen_ai_compute_task: another member of the same union as the seeded notebook_task + tasks[*].pipeline_task: another member of the same union as the seeded notebook_task + tasks[*].power_bi_task: another member of the same union as the seeded notebook_task + tasks[*].python_wheel_task: another member of the same union as the seeded notebook_task + tasks[*].run_job_task: another member of the same union as the seeded notebook_task + tasks[*].spark_jar_task: another member of the same union as the seeded notebook_task + tasks[*].spark_python_task: another member of the same union as the seeded notebook_task + tasks[*].spark_submit_task: another member of the same union as the seeded notebook_task + tasks[*].sql_task: another member of the same union as the seeded notebook_task + + # A task's compute is a union too, and base seeds job_cluster_key. + tasks[*].existing_cluster_id: another way to name the task's compute than the seeded job_cluster_key + tasks[*].new_cluster: another way to name the task's compute than the seeded job_cluster_key + tasks[*].environment_key: only for a task type that runs in a serverless environment + + # Each names a workspace object this suite does not provision. + job_clusters[*].serverless_compute_id: needs a serverless compute to exist + schedule.sql_condition: needs a saved query and a warehouse to exist + trigger.sql_condition: needs a saved query and a warehouse to exist + trigger.model.securable_name: needs a registered model to exist + git_source.job_source: needs a job whose configuration lives in the repository + # run_as must name a principal allowed to run the job, which depends on the identity the + # suite runs as -- a service principal on cloud. + run_as: depends on the identity the suite runs as + # A job cluster key has to name an entry in job_clusters, and this suite drives one field at a + # time -- so renaming the key on the task and on the definition cannot happen together. + tasks[*].job_cluster_key: has to name a job_clusters entry, which cannot change with it + job_clusters[*].job_cluster_key: has to match the key the task names, which cannot change with it + # A node type is cloud-specific and the suite has one variable for it, so there is no second + # value to move to: "Node type x is not supported. Supported node types: r3.xlarge, ...". + job_clusters[*].new_cluster.node_type_id: cloud-specific; the suite has one node type + job_clusters[*].new_cluster.driver_node_type_id: cloud-specific; the suite has one node type + # An account-level policy, as with apps. + usage_policy_id: needs an account-level usage policy to exist + # The backend assigns a job cluster its name: "Cluster name should not be provided". + job_clusters[*].new_cluster.cluster_name: assigned by the backend for a job cluster + # Each names a workspace object this suite does not provision. + job_clusters[*].new_cluster.policy_id: needs a cluster policy to exist + job_clusters[*].new_cluster.instance_pool_id: needs an instance pool to exist + job_clusters[*].new_cluster.driver_instance_pool_id: needs an instance pool to exist + job_clusters[*].new_cluster.init_scripts[*].s3.kms_key: needs a KMS key registered for the workspace + job_clusters[*].new_cluster.cluster_log_conf.s3.kms_key: needs a KMS key registered for the workspace + tasks[*].notebook_task.warehouse_id: needs a second SQL warehouse to exist + # A trigger, a schedule and a continuous block are three arms of one choice, so a schedule + # field cannot move on a job base gives no schedule. + schedule: one of schedule, trigger or continuous, and the config declares none + # trigger is a choice between file_arrival, table_update, model and periodic, and base seeds + # periodic. Each other member is rejected as a set, in its own words: + # "Invalid trigger settings. One of 'schedule', 'trigger' or 'continuous' can be set." + # "Field 'new_settings.trigger.table_update.table_names' is required, expected non-empty collection!" + # "Missing required field: new_settings.trigger.model.condition" + # Covering them needs a second fixture per member rather than a value for a leaf inside one. + trigger.file_arrival: a trigger member the seeded periodic trigger excludes + trigger.table_update: a trigger member the seeded periodic trigger excludes + trigger.model: a trigger member the seeded periodic trigger excludes + + # "dependencies and java_dependencies cannot be provided at the same time for environment key + # seeded", and base seeds dependencies. + environments[*].spec.java_dependencies: cannot be set alongside the seeded dependencies + + # "Sparse checkout pattern must a valid file path and cannot exceed 2048 characters" -- the + # generic x and y are not paths. + git_source.sparse_checkout: needs a valid file path, which no generic string value is + + # Both hold node type ids, and a node type is cloud-specific: "Node type x is not supported." + # The suite has one node type variable, so there is no second value to move to. + job_clusters[*].new_cluster.worker_node_type_flexibility: cloud-specific node type ids + job_clusters[*].new_cluster.driver_node_type_flexibility: cloud-specific node type ids + + # "S3 cluster log destination is provided without region" -- the block is validated as a set, + # and a region is cloud-specific. + job_clusters[*].new_cluster.cluster_log_conf.s3: validated as a set with a cloud-specific region + + # Only one of the three cloud attribute blocks applies, decided by the workspace's cloud. + job_clusters[*].new_cluster.azure_attributes: cloud-specific; only one attribute block applies + job_clusters[*].new_cluster.gcp_attributes: cloud-specific; only one attribute block applies + job_clusters[*].new_cluster.aws_attributes: cloud-specific; only one attribute block applies + +fields: + format: [SINGLE_TASK, MULTI_TASK] + performance_target: [PERFORMANCE_OPTIMIZED, STANDARD] + max_concurrent_runs: [1, 5] + timeout_seconds: [600, 1200] + git_source.git_provider: [gitHub, gitLab] + # A repository URL and a TZ database zone id, neither free text. + git_source.git_url: [https://github.com/databricks/cli.test, https://github.com/databricks/dabs.test] + git_source.git_commit: [abc123, def456] diff --git a/bundle/direct/autotest/testdata/fields/model_serving_endpoints.yml b/bundle/direct/autotest/testdata/fields/model_serving_endpoints.yml new file mode 100644 index 00000000000..9818a6d8c64 --- /dev/null +++ b/bundle/direct/autotest/testdata/fields/model_serving_endpoints.yml @@ -0,0 +1,58 @@ +# Value library for model serving endpoints. See ../../README.md. +# +# tags is declared by no config, so it is seeded here. +# +# ai_gateway and the top-level rate_limits are deliberately not seeded: the API rejects both on +# an endpoint created without a config block ("Cannot specify ai_gateway when creating endpoints +# without a config"), and base declares no config block. Covering them needs a +# config that declares a served entity, which needs a model in the workspace -- so the report +# lists their fields as uncovered. +base: + name: test-endpoint-$UNIQUE_NAME + tags: + - key: team + value: eng + +skip: + # Not seeded in base on purpose: the create does not apply email_notifications (the plan right + # after it already wants to update them), so declaring one would leave the endpoint drifting for + # its whole life and every other field measured against that. The leaves are reachable anyway -- + # the suite grows the container itself -- so seeding bought a BASELINE_DRIFT row and nothing else. + # The behaviour is recorded where it belongs, in + # acceptance/bundle/resources/model_serving_endpoints/update/email-notifications. + + # An AI gateway only applies to an endpoint serving an external model, provisioned + # throughput, or a custom or agent model -- "AI Gateway is currently only supported for + # External Models, Provisioned Throughput and Custom or Agent Model". base serves none of + # those, and the API also refuses the whole block on an endpoint created without a config. + # Covering it needs a fixture that declares such a served entity, which needs a model in the + # workspace. + ai_gateway: needs an endpoint serving an external, provisioned-throughput or custom model + # Same shape: "Config needs to be set" -- there is no config block on this endpoint to + # attach an auto-capture destination to. + config: needs a served entity, which needs a model in the workspace + + # Seeding a coherent telemetry block is rejected on every later update ("failed to update + # telemetry config"), and without one the fields inside it are unreachable. So this block is + # not covered here; acceptance/bundle/resources/model_serving_endpoints covers it end to end. + telemetry_config: rejected on update once seeded, and unreachable unless seeded + +fields: + # A recipient is validated by format, so the generic "x" is rejected. Reserved + # non-resolving addresses, per RFC 2606. + # Both the list and the address inside it need real values: the API validates every entry + # ("Invalid email format: x"), and the generic one- and two-element lists a slice gets by default + # hold plain strings. Declared for the container as well as the element, so growing the list is + # tested with addresses the backend accepts. + # + # In alphabetical order, because the backend returns the list sorted: declared the other way round, + # a two-address list read back reordered and the endpoint drifted on the ordering alone, for good. + # That is a property worth knowing, and it makes the field untestable if the fixture fights it. + email_notifications.on_update_failure: + - [alerts@databricks.invalid] + - [alerts@databricks.invalid, notify@databricks.invalid] + email_notifications.on_update_success: + - [alerts@databricks.invalid] + - [alerts@databricks.invalid, notify@databricks.invalid] + email_notifications.on_update_failure[*]: [notify@databricks.invalid, alerts@databricks.invalid] + email_notifications.on_update_success[*]: [notify@databricks.invalid, alerts@databricks.invalid] diff --git a/bundle/direct/autotest/testdata/fields/models.yml b/bundle/direct/autotest/testdata/fields/models.yml new file mode 100644 index 00000000000..a31a03a546d --- /dev/null +++ b/bundle/direct/autotest/testdata/fields/models.yml @@ -0,0 +1,6 @@ +# Value library for MLflow models. See ../../README.md. +base: + name: test-model-$UNIQUE_NAME + tags: + - key: team + value: eng diff --git a/bundle/direct/autotest/testdata/fields/pipelines.yml b/bundle/direct/autotest/testdata/fields/pipelines.yml new file mode 100644 index 00000000000..1f6c1124341 --- /dev/null +++ b/bundle/direct/autotest/testdata/fields/pipelines.yml @@ -0,0 +1,155 @@ +# Value library for pipelines. See ../../README.md. + +# base has to declare a clusters block, or its whole spec -- 346 +# fields, including the init_scripts union and the tag maps -- would go untested. Values +# that differ per cloud come from the same variables an acceptance config would get. +base: + name: test-pipeline-$UNIQUE_NAME + clusters: + - label: default + node_type_id: $NODE_TYPE_ID + num_workers: 1 + custom_tags: + team: eng + spark_conf: + spark.speculation: "true" + spark_env_vars: + MY_VAR: value + ssh_public_keys: + - ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQtest + configuration: + my.key: value + environment: + dependencies: + - requests + filters: + include: + - included + exclude: + - excluded + libraries: + - notebook: + path: /Shared/notebook + notifications: + - alerts: + - on-update-failure + email_recipients: + - notify@example.com + restart_window: + start_hour: 1 + time_zone_id: UTC + tags: + team: eng + +skip: + + # The schema field is refused on a storage-backed pipeline, which base declares: the API points at + # its migrate-to-dpm documentation. Clearing it is the half that fails, so the field is not testable + # on this fixture. + schema: refused on a storage-backed pipeline, which base declares + + # Both autoscale bounds are reported against num_workers, which base sets and no transition moves + # with them -- the same reason autoscale.mode is skipped just above. + clusters[*].autoscale.max_workers: drift is attributed to num_workers, which cannot move with it + clusters[*].autoscale.min_workers: drift is attributed to num_workers, which cannot move with it + + # An alert name and a recipient are both validated -- Invalid ... notifications: x -- so neither the + # list nor the entry inside it can hold a generic value. A recipient has to be an address and an + # alert has to be one the API knows. + notifications[*].alerts: needs alert names the API knows, and generic values are refused + notifications[*].email_recipients: needs real addresses, and generic values are refused + + # You cannot provide clusters on a serverless pipeline, and base declares one. + serverless: cannot be set on a pipeline that declares clusters, which base does + + # name must be set, so clearing it is refused, and this fixture has one name to give. + name: cannot be cleared, and the suite has one name for it + + # A pipeline parameter is accepted and not applied. + # Not seeded in base either: the create does not apply it, so the pipeline drifts from the moment it + # exists and every other field is then measured against a resource already pending. + parameters: the API accepts the write and does not apply it + + # Changing the autoscale mode is reported against num_workers, which base sets and the suite does not + # move at the same time -- the same reason the other autoscale fields are already skipped. + clusters[*].autoscale.mode: drift is attributed to num_workers, which cannot move with it + + # --- Fields a real workspace rejects, with the API's own reason. The mock server accepts them all. + + # Cannot add catalog to an existing pipeline with a storage location, and this fixture's pipeline has + # one -- the backend assigns it when neither storage nor catalog is given. That the engine sends the + # update anyway rather than recreating is a finding in its own right; see the branch notes. + catalog: cannot be added to a pipeline that already has a storage location + + # A cluster label has to be default or maintenance. + clusters[*].label: only default or maintenance are accepted + + # The environment version applies to Unity Catalog pipelines, and this one is storage-backed. + environment.environment_version: only for Unity Catalog pipelines, and base is storage-backed + + # An event log can only be configured on a UC pipeline, so the whole block is unreachable here. + event_log: only for Unity Catalog pipelines, and base is storage-backed + + # libraries must contain at least one element, so the list cannot be emptied. + libraries: cannot be emptied, and the API refuses a pipeline with no library + + # Only an absolute file path is accepted. + libraries[*].file.path: needs an absolute path, which no generic value is + + # Storage has to be an absolute path. + storage: needs an absolute path, which no generic value is + + # --- Fields the API accepts and then does not apply. The mock server applies them, so a local row + # would say OK where a workspace says the write had no effect. Modelling each in libs/testserver is + # the real fix; until then the reason is the finding. + dry_run: the API accepts the write and does not apply it + libraries[*].jar: the API accepts the write and does not apply it + libraries[*].maven: the API accepts the write and does not apply it + clusters[*].azure_attributes.capacity_reservation_group: the API accepts the write and does not apply it + clusters[*].gcp_attributes.confidential_compute_type: the API accepts the write and does not apply it + clusters[*].gcp_attributes.first_on_demand: the API accepts the write and does not apply it + ingestion_definition.ingest_from_uc_foreign_catalog: the API accepts the write and does not apply it + # An init script has to name a file that exists: a real workspace fails the cluster with "Init + # scripts failed ... Tree node with path ... does not exist". This suite deliberately uploads no + # bundle files -- a sync per permutation would dominate the run -- so no destination it could name + # is there, and the mock server's acceptance of any path is what made the field look testable. + clusters[*].init_scripts: an init script has to name a file that exists, and this suite uploads none + # Each names a workspace or account object this suite does not provision: an account-level + # budget policy, an instance pool, a cluster policy, a KMS key, a UC connection, an ingestion + # gateway pipeline. + budget_policy_id: needs an account-level budget policy to exist + clusters[*].instance_pool_id: needs a second instance pool to exist + clusters[*].driver_instance_pool_id: needs a second instance pool to exist + clusters[*].policy_id: needs a cluster policy to exist + gateway_definition: needs a UC connection to a source database + ingestion_definition.connection_name: needs a UC connection to a source database + ingestion_definition.ingestion_gateway_id: needs an ingestion gateway pipeline to exist + # run_as must name a principal allowed to run the pipeline, which depends on the identity the + # suite runs as -- a service principal on cloud. + run_as: depends on the identity the suite runs as + # A cron trigger is validated as a pair: "Invalid cron expression '' or timezone ...", so the + # timezone cannot move without an expression beside it. The config declares no trigger, so + # there is none to move it with. + trigger.cron: validated as a pair with the cron expression, which the config does not declare + + # An init script entry carries exactly one of workspace, volumes, s3, abfss, gcs, dbfs or file, + # and base seeds the workspace member. Setting a leaf inside another arm leaves that arm without + # its own destination, which the backend accepts and then ignores -- every such row was recording + # the fake server storing a value the API drops. Covering an arm needs a fixture that seeds it, + # not a value for a leaf inside one that is not there. + clusters[*].init_scripts[*].abfss: an init script member the seeded workspace member excludes + clusters[*].init_scripts[*].dbfs: an init script member the seeded workspace member excludes + clusters[*].init_scripts[*].file: an init script member the seeded workspace member excludes + clusters[*].init_scripts[*].gcs: an init script member the seeded workspace member excludes + clusters[*].init_scripts[*].s3: an init script member the seeded workspace member excludes + clusters[*].init_scripts[*].volumes: an init script member the seeded workspace member excludes + +fields: + # Constrained by the backend but typed as a plain string in the SDK, so nothing derives their + # values: the generic "x" is rejected on a real workspace. + channel: [CURRENT, PREVIEW] + + # A root path is a workspace path, and a time zone is a TZ database id, not free text. + root_path: [/Shared/pipeline-a, /Shared/pipeline-b] + restart_window.time_zone_id: [UTC, America/New_York] + edition: [CORE, PRO] diff --git a/bundle/direct/autotest/testdata/fields/postgres_branches.yml b/bundle/direct/autotest/testdata/fields/postgres_branches.yml new file mode 100644 index 00000000000..20d45e6676a --- /dev/null +++ b/bundle/direct/autotest/testdata/fields/postgres_branches.yml @@ -0,0 +1,15 @@ +# Value library for postgres branches. See ../../README.md. + +# The parent project holds its name after deletion until the soft delete is purged, so the chain +# cannot be rebuilt on a real workspace. See postgres_projects.yml. +local_only: a recreate cannot reuse the project name until the soft delete is purged + +deps: + postgres_projects: + project_id: test-pg-project-$UNIQUE_NAME + display_name: Test Postgres Project + +base: + parent: ${resources.postgres_projects.postgres_projects.name} + branch_id: test-branch-$UNIQUE_NAME + no_expiry: true diff --git a/bundle/direct/autotest/testdata/fields/postgres_catalogs.yml b/bundle/direct/autotest/testdata/fields/postgres_catalogs.yml new file mode 100644 index 00000000000..bbb8fcc8fb4 --- /dev/null +++ b/bundle/direct/autotest/testdata/fields/postgres_catalogs.yml @@ -0,0 +1,14 @@ +# Value library for postgres catalogs. See ../../README.md. + +local_only: a recreate cannot reuse the project name until the soft delete is purged + +deps: + postgres_projects: + project_id: test-pg-project-$UNIQUE_NAME + display_name: Test Postgres Project + +base: + catalog_id: test_pg_catalog_$UNIQUE_NAME + branch: ${resources.postgres_projects.postgres_projects.name}/branches/production + postgres_database: appdb + create_database_if_missing: true diff --git a/bundle/direct/autotest/testdata/fields/postgres_databases.yml b/bundle/direct/autotest/testdata/fields/postgres_databases.yml new file mode 100644 index 00000000000..ebc2576d0ce --- /dev/null +++ b/bundle/direct/autotest/testdata/fields/postgres_databases.yml @@ -0,0 +1,22 @@ +# Value library for postgres databases. See ../../README.md. + +local_only: a recreate cannot reuse the project name until the soft delete is purged + +deps: + postgres_projects: + project_id: test-pg-project-$UNIQUE_NAME + display_name: Test Postgres Project + postgres_branches: + parent: ${resources.postgres_projects.postgres_projects.name} + branch_id: test-branch-$UNIQUE_NAME + no_expiry: true + postgres_roles: + parent: ${resources.postgres_branches.postgres_branches.name} + role_id: test-role-$UNIQUE_NAME + postgres_role: app_role + +base: + parent: ${resources.postgres_branches.postgres_branches.name} + database_id: test-database-$UNIQUE_NAME + postgres_database: app_db + role: ${resources.postgres_roles.postgres_roles.name} diff --git a/bundle/direct/autotest/testdata/fields/postgres_endpoints.yml b/bundle/direct/autotest/testdata/fields/postgres_endpoints.yml new file mode 100644 index 00000000000..26392bccaa8 --- /dev/null +++ b/bundle/direct/autotest/testdata/fields/postgres_endpoints.yml @@ -0,0 +1,18 @@ +# Value library for postgres endpoints. See ../../README.md. + +local_only: a recreate cannot reuse the project name until the soft delete is purged + +deps: + postgres_projects: + project_id: test-pg-project-$UNIQUE_NAME + display_name: Test Postgres Project + postgres_branches: + parent: ${resources.postgres_projects.postgres_projects.name} + branch_id: test-branch-$UNIQUE_NAME + no_expiry: true + +base: + parent: ${resources.postgres_branches.postgres_branches.name} + endpoint_id: test-endpoint-$UNIQUE_NAME + endpoint_type: ENDPOINT_TYPE_READ_WRITE + suspend_timeout_duration: 300s diff --git a/bundle/direct/autotest/testdata/fields/postgres_projects.yml b/bundle/direct/autotest/testdata/fields/postgres_projects.yml new file mode 100644 index 00000000000..bbdac04e78d --- /dev/null +++ b/bundle/direct/autotest/testdata/fields/postgres_projects.yml @@ -0,0 +1,32 @@ +# Value library for postgres projects. See ../../README.md. + +# A recreate reuses the project name, and a deleted project holds its name until the soft delete +# is purged: "A soft-deleted project currently has this resource name." So on a real workspace +# every recreating field fails the create half, the same shape as instance_pools. A plain deploy +# and destroy works -- the invariant suite does that on aws -- but a run of recreates does not, +# so the catalog for this type is the local one. +local_only: a recreate cannot reuse the project name until the soft delete is purged +base: + project_id: test-pg-project-$UNIQUE_NAME + display_name: Test Postgres Project + enable_pg_native_login: false + custom_tags: + - key: team + value: eng + +skip: + # A branch reference, not a name: the API answers a bare string with "Field 'default_branch' + # expects 'projects//branches/'". Naming one needs a branch that exists, and + # this fixture creates only the project. + default_branch: names a branch resource the fixture does not create + + # The real API refuses a pg_setting on the create it is sent with: "setting cannot be changed; + # setting: work_mem" (aws, 2026-08). Seeding one made the whole type undeployable there, and the + # 12 rows it bought were all engine errors already -- every transition of this map was rejected + # for naming spec.default_endpoint_settings.pg_settings in update_mask without providing it. + default_endpoint_settings.pg_settings: the API rejects a setting on the create that carries it + +fields: + # project_id is part of the resource's hierarchical name, so a change recreates it. + project_id: [pgproj1, pgproj2] + pg_version: [16, 17] diff --git a/bundle/direct/autotest/testdata/fields/postgres_roles.yml b/bundle/direct/autotest/testdata/fields/postgres_roles.yml new file mode 100644 index 00000000000..5e9460447fc --- /dev/null +++ b/bundle/direct/autotest/testdata/fields/postgres_roles.yml @@ -0,0 +1,19 @@ +# Value library for postgres roles. See ../../README.md. + +# A recreate reuses the project name, and a deleted project holds it until the soft delete is +# purged, so the parent chain cannot be rebuilt on a real workspace. See postgres_projects.yml. +local_only: a recreate cannot reuse the project name until the soft delete is purged + +deps: + postgres_projects: + project_id: test-pg-project-$UNIQUE_NAME + display_name: Test Postgres Project + postgres_branches: + parent: ${resources.postgres_projects.postgres_projects.name} + branch_id: test-branch-$UNIQUE_NAME + no_expiry: true + +base: + parent: ${resources.postgres_branches.postgres_branches.name} + role_id: test-role-$UNIQUE_NAME + postgres_role: app_role diff --git a/bundle/direct/autotest/testdata/fields/postgres_synced_tables.yml b/bundle/direct/autotest/testdata/fields/postgres_synced_tables.yml new file mode 100644 index 00000000000..8008258a837 --- /dev/null +++ b/bundle/direct/autotest/testdata/fields/postgres_synced_tables.yml @@ -0,0 +1,27 @@ +# Value library for postgres synced tables. See ../../README.md. + +# The branch below names a project this fixture does not create -- one resource per bundle -- so +# on a real workspace there is no parent to attach to and the create fails with "Project +# test-pg-project-... not found". The fake server does not require the parent to exist, so the +# local report stands on its own; confirming it needs a project and branch provisioned first. +local_only: needs a postgres project and branch the suite does not provision +base: + synced_table_id: lakebase_$UNIQUE_NAME.public.trips_synced + source_table_full_name: main.raw.trips + primary_key_columns: [id] + scheduling_policy: SNAPSHOT + postgres_database: appdb + branch: projects/test-pg-project-$UNIQUE_NAME/branches/production + create_database_objects_if_missing: true + new_pipeline_spec: + storage_catalog: main + storage_schema: pipelines + extra_columns: + - column_name: ingested_at + column_type: TIMESTAMP + type_overrides: + # pg_type is an enum, not a Postgres type name: only PG_SPECIFIC_TYPE_HALFVEC, _VARCHAR and + # _VECTOR. "bigint" parsed to the unspecified zero value, which the real API rejects as a + # missing required field while the fake server stored it verbatim. + - column_name: id + pg_type: PG_SPECIFIC_TYPE_VARCHAR diff --git a/bundle/direct/autotest/testdata/fields/quality_monitors.yml b/bundle/direct/autotest/testdata/fields/quality_monitors.yml new file mode 100644 index 00000000000..720521c56da --- /dev/null +++ b/bundle/direct/autotest/testdata/fields/quality_monitors.yml @@ -0,0 +1,16 @@ +# Value library for quality monitors. See ../../README.md. + +# A monitor attaches to a table that already holds data, and its own name is that table's -- the +# suite provisions no table, so a real workspace has nothing to monitor. +local_only: needs an existing table with data to monitor + +base: + table_name: main.default.test_monitor_$UNIQUE_NAME + assets_dir: /Shared/databricks_monitoring/main.default.test_monitor_$UNIQUE_NAME + output_schema_name: main.default + inference_log: + granularities: ["1 day"] + timestamp_col: timestamp + prediction_col: prediction + model_id_col: model_id + problem_type: PROBLEM_TYPE_REGRESSION diff --git a/bundle/direct/autotest/testdata/fields/registered_models.yml b/bundle/direct/autotest/testdata/fields/registered_models.yml new file mode 100644 index 00000000000..1e3637e3092 --- /dev/null +++ b/bundle/direct/autotest/testdata/fields/registered_models.yml @@ -0,0 +1,26 @@ +# Value library for registered models. See ../../README.md. +# +# aliases is output-only in practice -- an alias is created through its own API, not by +# writing the model -- but the OpenAPI spec does not annotate it, so it is not skipped +# automatically. Seeding one lets the suite record what actually happens. +base: + name: test-model-$UNIQUE_NAME + catalog_name: main + schema_name: default + aliases: + - alias_name: champion + version_num: 1 + +skip: + # The backend assigns an alias its id; a value of ours is rejected as a bad UUID. + aliases[*].id: assigned by the backend + # Same shape: a UUID naming the workspace's metastore, which the backend fills in. + metastore_id: assigned by the backend + # Accepted by the API and then not honoured, so it drifts forever. Testing it properly + # needs a real external location, as with volumes. + storage_location: needs an external location + +fields: + # See the note in volumes.yml: a reference has to name something that exists. + catalog_name: [main] + schema_name: [default] diff --git a/bundle/direct/autotest/testdata/fields/schemas.yml b/bundle/direct/autotest/testdata/fields/schemas.yml new file mode 100644 index 00000000000..873dd7bb155 --- /dev/null +++ b/bundle/direct/autotest/testdata/fields/schemas.yml @@ -0,0 +1,21 @@ +# Value library for schemas. See ../../README.md. + +# properties is a map the config does not declare, so its entries would be unreachable. +base: + catalog_name: main + name: test-schema-$UNIQUE_NAME + comment: base comment + properties: + team: eng + +skip: + # storage_root must point at a real external location. + storage_root: needs an external location + +fields: + # See the note in volumes.yml: a reference has to name something that exists. + catalog_name: [main] + + # The backend takes hours but validates them as days: the retention period must be 0 or + # between 7 and 30 days, so the generic 1 and 2 are rejected. + custom_max_retention_hours: [168, 720] diff --git a/bundle/direct/autotest/testdata/fields/secret_scopes.yml b/bundle/direct/autotest/testdata/fields/secret_scopes.yml new file mode 100644 index 00000000000..44d4b03612b --- /dev/null +++ b/bundle/direct/autotest/testdata/fields/secret_scopes.yml @@ -0,0 +1,6 @@ + +base: + + + name: test-scope-$UNIQUE_NAME + backend_type: DATABRICKS diff --git a/bundle/direct/autotest/testdata/fields/secrets.yml b/bundle/direct/autotest/testdata/fields/secrets.yml new file mode 100644 index 00000000000..d1f52786af4 --- /dev/null +++ b/bundle/direct/autotest/testdata/fields/secrets.yml @@ -0,0 +1,31 @@ +# Value library for secrets. See ../../README.md. + +# The secret goes in a schema this fixture creates, not in main.default: the limit is 100 secrets per +# schema, and a shared workspace's default schema fills up with other runs' leftovers -- "Cannot +# create 1 Secret(s) in Schema ... (estimated count: 100, limit: 100)". A fresh schema is empty. +deps: + schemas: + catalog_name: main + name: test-secret-schema-$UNIQUE_NAME + +variables: + secret_value: seeded-value + +base: + catalog_name: main + schema_name: ${resources.schemas.schemas.name} + name: test-secret-$UNIQUE_NAME + value: ${var.secret_value} + +skip: + # The config format refuses a literal here, to keep secrets out of config files: "Secret value + # must be a variable reference". No value this suite could move the field to is a legal bundle, + # so the only reachable shape is the reference base declares. + value: the config format requires a variable reference, which no literal value is + + # Both name objects that have to exist: "Catalog 'x' does not exist", "Schema 'main.x' does not + # exist". The fake server takes any string, so these rows only ever measured the fake. A second + # catalog and schema would make them testable; the fixture creates one schema, and moving the + # secret between two is a different test than moving a field. + catalog_name: names a catalog that has to exist, and the suite provisions one + schema_name: names the schema the fixture creates, and there is no second one to move to diff --git a/bundle/direct/autotest/testdata/fields/sql_warehouses.yml b/bundle/direct/autotest/testdata/fields/sql_warehouses.yml new file mode 100644 index 00000000000..3dc9abe4f47 --- /dev/null +++ b/bundle/direct/autotest/testdata/fields/sql_warehouses.yml @@ -0,0 +1,46 @@ +# Value library for SQL warehouses. See ../../README.md. + +base: + name: test-warehouse-$UNIQUE_NAME + cluster_size: 2X-Small + auto_stop_mins: 10 + max_num_clusters: 1 + min_num_clusters: 1 + warehouse_type: CLASSIC + +skip: + # A warehouse is STARTING for a while after it is created, not RUNNING, so on a real workspace a + # config asking for started: false is already satisfied and the change is suppressed as + # remote_already_set -- where the mock server, which reports RUNNING at once, plans and applies a + # stop. One row differs, absent to false. Modelling it needs the mock server to move a warehouse + # from STARTING to RUNNING across reads, which the engine's waiter depends on, so the state this + # field is in depends on timing rather than on the engine and it is not worth a row either way. + lifecycle.started: a new warehouse is STARTING, so stopping it is already satisfied on cloud + # Must be an ARN of an instance profile registered with the workspace. + instance_profile_arn: needs an instance profile registered with the workspace + # Only accepted on the CUSTOM channel, so the version and the channel name have to move + # together; either alone is rejected. + channel.dbsql_version: only valid on the CUSTOM channel, so it cannot move alone + # Serverless is only offered on a PRO warehouse, and base is CLASSIC, so + # this flag cannot be moved on its own -- the pair would have to change together. + enable_serverless_compute: only valid on a PRO warehouse, which the config is not + +fields: + # The backend validates these as ranges rather than accepting any integer: + # auto_stop_mins is 0 (never) or at least 10, and max_num_clusters is at least 2 once it + # is being changed at all. min_num_clusters cannot exceed max_num_clusters, which the + # base sets to 1, so 1 is its only deployable value -- with absent in the set + # that is still an add and a remove. + auto_stop_mins: [10, 20] + min_num_clusters: [1] + max_num_clusters: [2, 3] + + # A t-shirt size, not free text. + cluster_size: [2X-Small, X-Small] + + # CHANNEL_NAME_CUSTOM is only accepted together with a dbsql_version, so the two would + # have to move together; the other names stand alone. + channel.name: [CHANNEL_NAME_CURRENT, CHANNEL_NAME_PREVIEW] + + # A tag key has to be non-empty, so "" is not one of the values to try. + tags.custom_tags[*].key: [team, owner] diff --git a/bundle/direct/autotest/testdata/fields/synced_database_tables.yml b/bundle/direct/autotest/testdata/fields/synced_database_tables.yml new file mode 100644 index 00000000000..8489565c6bb --- /dev/null +++ b/bundle/direct/autotest/testdata/fields/synced_database_tables.yml @@ -0,0 +1,24 @@ +# Value library for synced database tables. See ../../README.md. + +# Lakebase v1 is not available in every cloud, and the source table below is not provisioned. +local_only: Lakebase v1 is not available in every cloud + +deps: + database_instances: + name: test-db-instance-$UNIQUE_NAME + capacity: CU_1 + database_catalogs: + database_instance_name: ${resources.database_instances.database_instances.name} + database_name: test_db + name: test-catalog-$UNIQUE_NAME + create_database_if_not_exists: true + +base: + name: ${resources.database_catalogs.database_catalogs.name}.${resources.database_catalogs.database_catalogs.database_name}.test_table + database_instance_name: ${resources.database_instances.database_instances.name} + logical_database_name: ${resources.database_catalogs.database_catalogs.database_name} + spec: + source_table_full_name: main.test_synced_$UNIQUE_NAME.trips_source + scheduling_policy: SNAPSHOT + primary_key_columns: + - tpep_pickup_datetime diff --git a/bundle/direct/autotest/testdata/fields/vector_search_endpoints.yml b/bundle/direct/autotest/testdata/fields/vector_search_endpoints.yml new file mode 100644 index 00000000000..743a37d56c0 --- /dev/null +++ b/bundle/direct/autotest/testdata/fields/vector_search_endpoints.yml @@ -0,0 +1,11 @@ +# Value library for vector search endpoints. See ../../README.md. + +base: + # Endpoint names must be under 50 characters. + name: test-vse-$UNIQUE_NAME + endpoint_type: STANDARD + +skip: + # An account-level policy, as with apps: "Failed to validate budget policy x." A test workspace has + # none to reference, and creating one is an account-level operation. + budget_policy_id: needs an account-level budget policy to exist diff --git a/bundle/direct/autotest/testdata/fields/vector_search_indexes.yml b/bundle/direct/autotest/testdata/fields/vector_search_indexes.yml new file mode 100644 index 00000000000..67d6da176d0 --- /dev/null +++ b/bundle/direct/autotest/testdata/fields/vector_search_indexes.yml @@ -0,0 +1,34 @@ +# Value library for vector search indexes. See ../../README.md. + +deps: + vector_search_endpoints: + name: test-vse-$UNIQUE_NAME + endpoint_type: STANDARD + +base: + name: main.default.test_index_$UNIQUE_NAME + endpoint_name: ${resources.vector_search_endpoints.vector_search_endpoints.name} + primary_key: id + index_type: DIRECT_ACCESS + direct_access_index_spec: + schema_json: '{"id":"integer","vector":"array"}' + embedding_vector_columns: + - name: vector + embedding_dimension: 768 + +skip: + # Every field of an index is immutable, so each transition deletes and recreates it, and the whole + # type took three hours on a real workspace before hitting the test timeout -- 102 rows, each + # waiting for an index to come online behind an endpoint that has to be provisioned first. That + # cost buys little: the fields are the index spec, and what the catalog checks about them is the + # same recreate path every time. + # + # So one field is driven and the rest are skipped. primary_key is the cheapest that still means + # something: it is required, so it has two transitions rather than six, and both go through + # create, recreate and converge -- which is what could break for this type. + delta_sync_index_spec: one field of this type is driven; see primary_key + direct_access_index_spec: one field of this type is driven; see primary_key + endpoint_name: one field of this type is driven; see primary_key + index_type: one field of this type is driven; see primary_key + index_subtype: one field of this type is driven; see primary_key + name: one field of this type is driven; see primary_key diff --git a/bundle/direct/autotest/testdata/fields/volumes.yml b/bundle/direct/autotest/testdata/fields/volumes.yml new file mode 100644 index 00000000000..dd1e53874a6 --- /dev/null +++ b/bundle/direct/autotest/testdata/fields/volumes.yml @@ -0,0 +1,23 @@ +# Value library for volumes. See ../../README.md. +base: + name: test-volume-$UNIQUE_NAME + catalog_name: main + schema_name: default + +skip: + # An external volume needs a storage location backed by a real external location and + # cloud IAM, which this suite does not provision. + storage_location: needs an external location + +fields: + # EXTERNAL is only valid together with storage_location, so only MANAGED is + # deployable on its own. + volume_type: [MANAGED] + + # A name that refers to another object has to name one that exists: a real workspace + # rejects the generic "x" outright, so the field would report nothing but BACKEND_ERROR. + # main.default exists in every UC workspace the suite runs against. One value, not two: + # a second catalog and schema would have to be provisioned, and the pairs with absent + # already cover what a single value can show. + catalog_name: [main] + schema_name: [default] diff --git a/bundle/direct/autotest/values_test.go b/bundle/direct/autotest/values_test.go new file mode 100644 index 00000000000..d9f0055c3b8 --- /dev/null +++ b/bundle/direct/autotest/values_test.go @@ -0,0 +1,902 @@ +package autotest + +import ( + "encoding/json" + "errors" + "fmt" + "hash/fnv" + "io/fs" + "maps" + "math/rand/v2" + "os" + "path/filepath" + "reflect" + "slices" + "strconv" + "strings" + + "github.com/databricks/cli/bundle/direct/dresources" + "github.com/databricks/cli/bundle/internal/validation/generated" + "github.com/databricks/cli/libs/structs/structaccess" + "github.com/databricks/cli/libs/structs/structpath" + "github.com/databricks/cli/libs/structs/structwalk" + "go.yaml.in/yaml/v3" +) + +// absent is the "field not present in the config" value. Every transition is a move +// between two values drawn from a field's set, and add/remove are just the moves +// with absent on one side. A Go struct has no absent, so nil stands for it: writing it +// means the zero value with the field dropped from ForceSendFields. +var absent any = nil + +// fieldValues is the per-resource-type value library, e.g. testdata/fields/schemas.yml. +// +// A field with no entry falls back to defaultValues for its Go kind. Fields the backend +// constrains (enums, ids, cross-referenced names) need an explicit entry, otherwise +// every value is rejected and the field reports only BACKEND_ERROR. +type fieldValues struct { + // skip lists field paths to leave out entirely, with a reason. A key ending in ".*" + // skips that whole subtree. + skip map[string]string + + // fields maps a field path to the values to try, as the YAML gave them. They are + // converted to the field's own Go type when the field is enumerated. + fields map[string][]any + + // base is the resource itself, rendered into a one-resource databricks.yml and deployed + // before anything is measured. What it declares is what can be tested: a block it omits has + // no entry for its fields to live in, and a block the API only accepts whole -- a coherent + // git_source -- cannot be built up one field at a time. Once declared, each field varies + // normally. + // + base any + + // localOnly is the reason this resource type cannot be driven against a real workspace: + // it needs workspace or cloud state the suite does not provision (a storage credential + // with IAM behind it), or the service is not available in every cloud. Mirrors the + // per-config cloud exclusions in acceptance/bundle/invariant/test.toml. + // + // Such a type is skipped on cloud rather than reported: the local golden stands, and a + // cloud run neither confirms nor contradicts it. + localOnly string + + // variables are bundle variables the fixture's own values reference, with the value to give + // each. A secret's value has to be a ${var...} reference -- the config format refuses a + // literal, to keep secrets out of config files -- so the fixture needs somewhere to declare + // one. The value is passed the way a user would, through BUNDLE_VAR_: a default in the + // config would be resolved before that validation runs, and then fail it. + variables map[string]string + + // deps are resources the one under test needs in order to exist, keyed by resource type: a + // postgres branch needs a project, a secret needs nothing but a catalog and schema that + // already exist. They are deployed in the same bundle and referenced from base with + // ${resources...}, and no field of one is ever tested -- only the resource under test is. + deps map[string]any + + // clouds names the clouds whose workspaces can host this resource type, for a service that + // exists on some and not others. Empty means every cloud. Unlike localOnly this does not + // give up on the cloud run: the type is verified where the service exists and skipped + // where it does not, which mirrors CloudEnvs in acceptance/bundle/resources/*/test.toml. + clouds []string +} + +// fieldsDir holds the per-resource-type value libraries. +const fieldsDir = "testdata/fields" + +// declaredUnsettable collects the fields the resource declares as backend outputs. A user +// cannot meaningfully set one at all -- they are in the input schema by accident -- so they +// are skipped and the resource's own reason recorded instead of spending six deploys per +// field rediscovering it. +// +// This and declaredIgnoredLocally are the only places the suite reads resources.yml. +func declaredUnsettable(adapter *dresources.Adapter) []dresources.FieldRule { + var rules []dresources.FieldRule + for _, cfg := range lifecycleConfigs(adapter) { + // Only outputs are unsettable. An input_only or managed field is one the backend + // owns on read but the user may still legitimately write, so those stay in. + for _, rule := range cfg.IgnoreRemoteChanges { + if strings.HasSuffix(rule.Reason, "output_only") { + rules = append(rules, rule) + } + } + } + return rules +} + +// declaredIgnoredLocally collects the fields a resource says it drops local changes to. +// These are *not* skipped: the declaration is a claim about behaviour, and the suite is in a +// position to check it. A transition of one should come back suppressed with that same +// reason; anything else means the field is not actually inert and the declaration is wrong. +func declaredIgnoredLocally(adapter *dresources.Adapter) []dresources.FieldRule { + var rules []dresources.FieldRule + for _, cfg := range lifecycleConfigs(adapter) { + rules = append(rules, cfg.IgnoreLocalChanges...) + } + return rules +} + +// declaredDeliberate collects every field rule the resource declares about not acting: the ones +// it ignores local changes to, and the ones whose remote value it does not compare. A plan that +// skips a field for one of these reasons has done what the resource says it does, so the config's +// value is as reached as it is ever going to be. +func declaredDeliberate(adapter *dresources.Adapter) []dresources.FieldRule { + var rules []dresources.FieldRule + for _, cfg := range lifecycleConfigs(adapter) { + rules = append(rules, cfg.IgnoreLocalChanges...) + rules = append(rules, cfg.IgnoreRemoteChanges...) + } + return rules +} + +// declaredIDFields collects the fields the resource declares as composing its ID -- its name, +// as opposed to a server-generated id. A resource cannot exist without one: the engine fetches +// it by that ID, and a local change to one recreates rather than updates (see adapter.go's +// recreateOnChange). So an absent ID field has no resource to create, and the backend refusing +// to create it is the API contract, not a finding. +func declaredIDFields(adapter *dresources.Adapter) []dresources.FieldRule { + var rules []dresources.FieldRule + for _, cfg := range lifecycleConfigs(adapter) { + rules = append(rules, cfg.ProvidedIDFields...) + } + return rules +} + +func lifecycleConfigs(adapter *dresources.Adapter) []*dresources.ResourceLifecycleConfig { + var out []*dresources.ResourceLifecycleConfig + for _, cfg := range []*dresources.ResourceLifecycleConfig{adapter.ResourceConfig(), adapter.GeneratedResourceConfig()} { + if cfg != nil { + out = append(out, cfg) + } + } + return out +} + +// ruleReason reports the reason a field matches one of the given rules. Accepts both a +// concrete path ("tags[0].key") and a pattern ("tags[*].key"): the second form is needed +// for a field whose container the config does not declare, where no concrete path exists. +func ruleReason(rules []dresources.FieldRule, path string) (string, bool) { + if concrete, err := structpath.ParsePath(path); err == nil { + for _, rule := range rules { + // The same match the planner makes, so the suite and the engine agree on scope. + if concrete.HasPatternPrefix(rule.Field) { + return rule.Reason, true + } + } + return "", false + } + + // A pattern: compare textually, since one pattern cannot be matched against another. + for _, rule := range rules { + declared := rule.Field.String() + if path == declared || strings.HasPrefix(path, declared+".") || strings.HasPrefix(path, declared+"[") { + return rule.Reason, true + } + } + return "", false +} + +// cliManagedFields are fields the bundle acts on itself and never sends to any API, so no +// transition of one can appear in a plan. Only fields with no API counterpart at all belong +// here: a field the CLI merely *overwrites* is still worth testing, because the resulting +// verdict is the evidence that it does. +var cliManagedFields = map[string]string{ + "lifecycle.prevent_destroy": "acted on by the bundle, never sent to any API; gates destroy", +} + +// loadFieldValues reads testdata/fields/.yml, expanding the same $VARS the +// corpus configs use so a value can name the workspace's own user rather than a placeholder +// only the fake server knows. +func loadFieldValues(resourceType string, vars map[string]string) (*fieldValues, error) { + fv := &fieldValues{skip: map[string]string{}, fields: map[string][]any{}, base: nil, localOnly: "", clouds: nil, deps: nil, variables: nil} + maps.Copy(fv.skip, cliManagedFields) + + path := filepath.Join(fieldsDir, resourceType+".yml") + data, err := os.ReadFile(path) + if errors.Is(err, fs.ErrNotExist) { + return fv, nil + } + if err != nil { + return nil, err + } + + var missing string + expanded := os.Expand(string(data), func(key string) string { + // UNIQUE_NAME belongs to one deploy, and a library is read once for the whole run: the + // base is rendered into a bundle per harness, and a rebuild gets a new name. Passing it + // through keeps it for renderBundle to expand there. + if key == uniqueNameVar { + return "$" + uniqueNameVar + } + if value, ok := vars[key]; ok { + return value + } + // A bundle's own interpolation shares this syntax and belongs to the config: base declares + // ${var.secret_value} and a dep reference like ${resources.postgres_projects.x.name}. Told + // apart by the dot, as in renderBundle, which is where they are finally emitted. + if strings.Contains(key, ".") { + return "${" + key + "}" + } + missing = key + return "" + }) + if missing != "" { + return nil, fmt.Errorf("%s uses $%s, which this suite does not provide here", path, missing) + } + + var file struct { + Skip map[string]string `yaml:"skip"` + Fields map[string][]any `yaml:"fields"` + Base any `yaml:"base"` + LocalOnly string `yaml:"local_only"` + Deps map[string]any `yaml:"deps"` + Variables map[string]string `yaml:"variables"` + Clouds []string `yaml:"clouds"` + } + if err := yaml.Unmarshal([]byte(expanded), &file); err != nil { + return nil, fmt.Errorf("%s: %w", path, err) + } + maps.Copy(fv.skip, file.Skip) + maps.Copy(fv.fields, file.Fields) + fv.base = file.Base + fv.localOnly = file.LocalOnly + fv.deps = file.Deps + fv.variables = file.Variables + fv.clouds = file.Clouds + + return fv, nil +} + +// isSubResource reports whether a path belongs to a sub-resource block. +func isSubResource(path string) bool { + name, _, _ := strings.Cut(path, ".") + name, _, _ = strings.Cut(name, "[") + return slices.Contains(subResourceKinds, name) +} + +// isContainer reports whether a kind has fields or elements to descend into. +func isContainer(kind reflect.Kind) bool { + switch kind { + case reflect.Struct, reflect.Slice, reflect.Array, reflect.Map: + return true + default: + return false + } +} + +// defaultValues returns the values to try for a leaf field. Two per field is enough to +// observe a value-to-value transition on top of add and remove; more would multiply the +// matrix without testing a different code path. +func defaultValues(typ reflect.Type) []any { + for typ.Kind() == reflect.Pointer { + typ = typ.Elem() + } + if values := enumValues(typ); values != nil { + return values + } + // A list of scalars is a value in its own right, and the config often leaves it empty -- + // a job's email_notifications.on_start, say -- so containerValues has nothing to trim and + // the fields inside it do not exist. One and two elements give the same coverage from + // nothing: with absent in the set, that is add, grow, shrink and remove. + if typ.Kind() == reflect.Slice { + elements := defaultValues(typ.Elem()) + if len(elements) == 0 { + return nil + } + if len(elements) == 1 { + // One element is all the type offers -- a list of an enum with a single valid value. + // The empty list and that one still cover adding, clearing and removing. + return []any{[]any{}, []any{elements[0]}} + } + // The empty list is its own value, not the same as absent: it is what a user writing + // "on_start: []" produces, and it reaches the API as an explicit empty array through + // ForceSendFields -- the only way to clear a list the backend already holds. + return []any{[]any{}, []any{elements[0]}, []any{elements[0], elements[1]}} + } + return kindValues(typ.Kind()) +} + +// enumValues returns two values of an SDK enum, which declares its own: every generated +// enum type has a Values() method on its pointer receiver. Without this an enum field gets +// the generic "x" and "y", which a real backend rejects or silently ignores -- so the field +// reports nothing about whether the engine handles a change to it. +// +// The values are sorted because the SDK documents no order, and the report is a golden. +func enumValues(typ reflect.Type) []any { + if typ.Kind() != reflect.String || typ == reflect.TypeFor[string]() { + return nil + } + method, ok := reflect.PointerTo(typ).MethodByName("Values") + if !ok || method.Type.NumIn() != 1 || method.Type.NumOut() != 1 { + return nil + } + result := method.Func.Call([]reflect.Value{reflect.New(typ)})[0] + if result.Kind() != reflect.Slice || result.Type().Elem() != typ { + return nil + } + + all := make([]string, 0, result.Len()) + for i := range result.Len() { + value := result.Index(i).String() + // A protobuf enum's zero member means "unset", not a choice: the backend normalizes it + // to whichever value it defaults to, so asking for it reads as the write being ignored. + if value == "" || strings.HasSuffix(value, "_UNSPECIFIED") { + continue + } + all = append(all, value) + } + slices.Sort(all) + if len(all) == 0 { + return nil + } + // One is enough: with absent in the set, the field still gets an add and a remove. There + // is no valid second value to invent, and falling back to the generic "x" and "y" would + // test values the backend rejects -- a pipeline's deployment.kind only permits BUNDLE. + all = all[:min(2, len(all))] + + values := make([]any, 0, len(all)) + for _, value := range all { + values = append(values, reflect.ValueOf(value).Convert(typ).Interface()) + } + return values +} + +// kindValues are the fallback values for a plain Go kind. +// +//nolint:exhaustive // the default branch covers every kind without a generic value +func kindValues(kind reflect.Kind) []any { + switch kind { + case reflect.Bool: + return []any{false, true} + case reflect.String: + return []any{"x", "y"} + case reflect.Int, reflect.Int32, reflect.Int64: + return []any{1, 2} + case reflect.Float32, reflect.Float64: + return []any{1.0, 2.0} + default: + // The remaining kinds (interface, unsigned, complex, ...) have no meaningful + // generic value; such a field needs an entry in the value library. + return nil + } +} + +// field is one testable leaf of the resource's input struct. +type field struct { + // seed drives this field's value order and the order its transitions are walked in. + // Derived from the run seed and the field path, so it is stable for a given commit + // and different for every field. + seed uint64 + + path string // structpath path, e.g. "comment" or "email_notifications.on_failure" + kind reflect.Kind + values []any + + // aliases maps a value's own label to the short alias the report shows instead, for values + // whose label would otherwise be an unreadable truncation. Only populated for those. + aliases map[string]string + + // required fields do not get an "absent" transition: a config missing one is + // rejected by bundle validate, so removing it is not something a user can deploy. + required bool +} + +// transition is one move of a field from one value to another. +type transition struct { + from, to any + // The labels the report and the subtest name use, which for an unwieldy value is a short + // alias the legend explains rather than the value itself. + fromLabel, toLabel string +} + +// label names the transition for the subtest. Kept free of shell metacharacters so a +// single case can be re-run without quoting: +// +// go test ./bundle/direct/autotest -run TestFields/pipelines/.*/dry_run/absent_to_true -v +func (t transition) label() string { + return t.fromLabel + "_to_" + t.toLabel +} + +// label names one of the field's values for the report and the subtest. An unwieldy value gets a +// short alias -- v1, v2 -- which the legend maps back, because the alternative is a subtest called +// "absent_to_alerts@databricks.i~4539": unreadable, and no easier to retype than v1. +func (f field) label(value any) string { + own := valueLabel(value) + if alias, ok := f.aliases[own]; ok { + return alias + } + return own +} + +// assignAliases gives every value whose own label is not a clean short token an alias. Numbered by +// sorted label rather than by position, so the alias for a value does not move when the run order +// does. +func assignAliases(values []any) map[string]string { + var unwieldy []string + for _, value := range values { + if own := valueLabel(value); own != cleanLabel(own) { + unwieldy = append(unwieldy, own) + } + } + if len(unwieldy) == 0 { + return nil + } + slices.Sort(unwieldy) + aliases := make(map[string]string, len(unwieldy)) + for i, own := range slices.Compact(unwieldy) { + aliases[own] = "v" + strconv.Itoa(i+1) + } + return aliases +} + +// transitions returns every ordered pair of the field's values, ordered as a single walk: +// each transition starts where the previous one ended, so reaching a starting value costs +// nothing and the field is deployed once per transition instead of twice. absent is one of +// the values, so adding and removing the field are the pairs with absent on one side. +// +// The values form a complete digraph, where every vertex has equal in- and out-degree, so +// an Eulerian circuit always exists and covers each ordered pair exactly once. The order +// among a vertex's outgoing edges is shuffled from a seed derived from the field path: +// fixed for a given field, so the report stays byte-stable, but different between fields, +// so the suite is not always walking the same shape of path. +func (f field) transitions() []transition { + // A fresh slice either way, since the shuffle below is in place: transitions() is called + // twice per field -- once to decide whether the field has anything to test, once to run it + // -- and shuffling the field's own values would make the second call walk a different chain + // than the first agreed to. + values := append([]any{absent}, f.values...) + if f.required { + values = slices.Clone(f.values) + } + if len(values) < 2 { + return nil + } + + rng := rand.New(rand.NewPCG(f.seed, 0)) + rng.Shuffle(len(values), func(i, j int) { values[i], values[j] = values[j], values[i] }) + unused := make([][]int, len(values)) + for from := range values { + for to := range values { + if from != to { + unused[from] = append(unused[from], to) + } + } + rng.Shuffle(len(unused[from]), func(i, j int) { + unused[from][i], unused[from][j] = unused[from][j], unused[from][i] + }) + } + + // Hierholzer: walk until stuck, then splice in circuits from vertices that still have + // unused edges. The reversed visit order is the circuit. + var stack, circuit []int + stack = append(stack, 0) + for len(stack) > 0 { + v := stack[len(stack)-1] + if len(unused[v]) == 0 { + circuit = append(circuit, v) + stack = stack[:len(stack)-1] + continue + } + next := unused[v][0] + unused[v] = unused[v][1:] + stack = append(stack, next) + } + slices.Reverse(circuit) + + out := make([]transition, 0, len(circuit)-1) + for i := 1; i < len(circuit); i++ { + from, to := values[circuit[i-1]], values[circuit[i]] + out = append(out, transition{ + from: from, to: to, + fromLabel: f.label(from), toLabel: f.label(to), + }) + } + return out +} + +// fieldSeed mixes the run seed with a field path, so every field gets its own order while +// the whole run stays reproducible from the commit alone. +func fieldSeed(runSeed uint64, path string) uint64 { + h := fnv.New64a() + _, _ = fmt.Fprintf(h, "%d/%s", runSeed, path) + return h.Sum64() +} + +// requiredFields returns the field names the bundle schema marks required directly +// under the given path inside a resource, e.g. "" for the resource itself or +// "evaluation" for a nested object. The data is generated from the config structs +// (bundle/internal/validation), not from the engine's resources.yml. +func requiredFields(resourceType, parent string) []string { + key := "resources." + resourceType + ".*" + if parent != "" { + key += "." + parent + } + return generated.RequiredFields[key] +} + +// skipReason returns why a field is excluded, if it is. A key may be a pattern, matched the +// same way the planner matches its own field rules -- so "aliases[*].id" covers +// "aliases[0].id", and a trailing ".*" or "[*]" covers everything beneath a block that only +// works as a whole. +func (fv *fieldValues) skipReason(path string) (string, bool) { + if reason, ok := fv.skip[path]; ok { + return reason, true + } + + concrete, err := structpath.ParsePath(path) + if err != nil { + // A pattern, which cannot be matched against another pattern: compare textually. A key + // naming a whole subtree may itself end in a wildcard, so trim that first -- otherwise + // "a.b.*" would not cover the pattern "a.b.c". + for key, reason := range fv.skip { + prefix := strings.TrimSuffix(strings.TrimSuffix(key, "[*]"), ".*") + if strings.HasPrefix(path, prefix+".") || strings.HasPrefix(path, prefix+"[") { + return reason, true + } + } + return "", false + } + + for key, reason := range fv.skip { + pattern, err := structpath.ParsePattern(key) + if err != nil { + continue + } + if concrete.HasPatternPrefix(pattern) { + return reason, true + } + } + return "", false +} + +func isRequired(resourceType, path string) bool { + // The generated keys are patterns, so an index has to become a wildcard first: + // "tasks[0].task_key" is declared under "resources.jobs.*.tasks[*]". + parent, name := "", patternOf(path) + if i := strings.LastIndex(name, "."); i >= 0 { + parent, name = name[:i], name[i+1:] + } + return slices.Contains(requiredFields(resourceType, parent), name) +} + +// enumerateFields walks the resource's input config type the way cmd/bundle/debug +// refschema does, and pairs each field with the values to try. +// +// resource is the deployed resource, which is what makes slices and maps testable. A slice or map the config populates becomes a field in its own right, +// with add-an-entry and remove-an-entry transitions; and a pattern like +// "tasks[*].description" is expanded to the indices that exist, so the fields inside an +// element get the same treatment as any other. A pattern with nothing behind it in the +// config is reported as not covered rather than silently tested against nothing. +func enumerateFields(resourceType string, inputType reflect.Type, fv *fieldValues, resource any, runSeed uint64, unsettable []dresources.FieldRule, unique string) (fields []field, uncovered []string, inertFields map[string]string) { + inertFields = map[string]string{} + // One row per field path. A path can be reached twice -- a cluster policy's definition is + // declared both on the bundle struct and on the embedded SDK struct under the same json + // name -- and structaccess resolves it to one field, so testing it twice would just print + // every result twice. + added := map[string]bool{} + add := func(path string, kind reflect.Kind, values []any) { + if added[path] { + return + } + added[path] = true + if _, skipped := fv.skipReason(path); skipped { + return + } + if reason, ok := ruleReason(unsettable, path); ok { + inertFields[path] = reason + return + } + f := field{ + seed: fieldSeed(runSeed, path), + path: path, + kind: kind, + values: values, + aliases: assignAliases(values), + required: isRequired(resourceType, path), + } + // A field with nothing to walk is a gap, not something to drop on the floor: no + // generic value exists for its type (an `any` field like serialized_dashboard), or + // it is required and the library gives it one value, so there is no second value to + // move to and no absent to move from. + if len(f.transitions()) == 0 { + uncovered = append(uncovered, patternOf(path)) + return + } + fields = append(fields, f) + } + + _ = structwalk.WalkType(inputType, func(p *structpath.PatternNode, typ reflect.Type, sf *reflect.StructField) bool { + if p.IsRoot() { + return true + } + for typ.Kind() == reflect.Pointer { + typ = typ.Elem() + } + if sf != nil && isNotUserSettable(sf) { + return false + } + + path := p.String() + if isSubResource(path) { + // permissions and grants are separate plan nodes with their own adapters, and + // this suite strips them from every config. They are out of scope as subjects, + // so they are not fields here and not gaps either. + return false + } + isWildcard := strings.Contains(path, "*") + + if isContainer(typ.Kind()) { + // A struct is only a grouping; a slice or map is also a value the user can + // grow and shrink, so test it as a field before descending into it. + if typ.Kind() != reflect.Struct { + for _, concrete := range expandPattern(resource, path) { + values := fv.fields[path] + if values == nil { + values = containerValues(resource, concrete) + } + if values == nil { + // The config leaves it empty, so there is nothing to trim -- but a + // list of scalars needs no invention: two of its element type cover + // adding, growing, shrinking and removing. + values = defaultValues(typ) + } + add(concrete, typ.Kind(), values) + } + } + return true + } + + if !isWildcard { + values := fv.fields[path] + if values == nil { + values = defaultValues(typ) + } + add(path, typ.Kind(), uniqueIdentityValues(values, path, unique, resource)) + return false + } + + concrete := expandPattern(resource, path) + if len(concrete) == 0 { + // A field the user cannot set is not a coverage gap. Checked on the pattern here + // because there is no concrete path to check: the container is absent. + if reason, ok := ruleReason(unsettable, path); ok { + inertFields[path] = reason + return false + } + if _, skipped := fv.skipReason(path); !skipped { + uncovered = append(uncovered, path) + } + return false + } + for _, c := range concrete { + // Explicit values are keyed by the pattern, since an index is incidental. + values := fv.fields[path] + if values == nil { + values = defaultValues(typ) + } + add(c, typ.Kind(), uniqueIdentityValues(values, path, unique, resource)) + } + return false + }) + + // Shuffle the fields too. Each field restores the config before the next one starts, + // so the order costs nothing in requests -- but a field that only passes because of + // what ran before it should not keep getting away with it. + rng := rand.New(rand.NewPCG(runSeed, 1)) + rng.Shuffle(len(fields), func(i, j int) { fields[i], fields[j] = fields[j], fields[i] }) + + slices.Sort(uncovered) + return fields, slices.Compact(uncovered), inertFields +} + +// uniqueIdentityValues appends the run's own suffix to a string value of a field that names +// the resource, so no two runs against one workspace ask for the same name: the second would +// get "already exists", which says nothing about the engine. +// +// Such a field is recognised by the corpus config templating its value with $UNIQUE_NAME, +// which is the config author saying the value has to be unique. That is a narrower rule than +// the resource's own provided_id_fields, deliberately: those include the catalog and schema a +// registered model lives in, which name *other* objects and must be left alone -- while a +// warehouse name has to be unique despite the id being a generated uuid. +// +// The report stays stable because the suffix is redacted the way every other generated id is. +func uniqueIdentityValues(values []any, path, unique string, resource any) []any { + if !namesThisResource(path, unique, resource) { + return values + } + out := make([]any, 0, len(values)) + for _, value := range values { + text, ok := value.(string) + if !ok || text == "" { + // Only a name can carry a suffix; an id field of another type keeps its value. + return values + } + // A placeholder, not the suffix itself: a rebuild gives the resource a new suffix while + // the old one is still alive under its old name, so a value fixed at enumeration time + // would collide with it. Substituted on the way into the config. + out = append(out, text+"-"+uniqueMarker) + } + return out +} + +// uniqueMarker stands in for the run's unique suffix inside a value. It is substituted for the +// harness's own suffix when the value is written, and redacted when the value is labelled, so +// the report is stable across runs. +const uniqueMarker = "@UNIQUE@" + +func namesThisResource(path, unique string, resource any) bool { + current, err := structaccess.GetByString(resource, path) + if err != nil { + return false + } + text, ok := current.(string) + return ok && strings.Contains(text, unique) +} + +// isNotUserSettable reports whether the bundle marks a field as something the user +// never writes: an output the CLI fills in (id, url) or an internal bookkeeping field. +func isNotUserSettable(sf *reflect.StructField) bool { + tag := sf.Tag.Get("bundle") + return strings.Contains(tag, "readonly") || + strings.Contains(tag, "internal") || + sf.Tag.Get("json") == "-" +} + +// containerValues returns the values to try for a slice or map field that the deployed +// resource actually populates: the resource's own value, and that value with one entry +// dropped. Combined with the implicit "absent", one field then covers adding and removing +// the whole container as well as adding and removing a single entry -- all with data the +// backend has already accepted, so nothing has to be invented. +// +// Both are deep copies: a later edit rewrites the same map or slice in place, which would +// otherwise change a value recorded here. +func containerValues(resource any, path string) []any { + current, err := structaccess.GetByString(resource, path) + if err != nil || isAbsent(current) { + return nil + } + + // Both values are deep copies. A shallow one would share the pointers inside an + // element -- a task's notebook_task, an init script's workspace block -- and a later + // edit to a field under that element would reach into a value recorded here. + full, err := clone(current) + if err != nil { + return nil + } + trimmed, err := clone(current) + if err != nil { + return nil + } + + value := reflect.ValueOf(trimmed) + switch value.Kind() { + case reflect.Slice: + if value.Len() == 0 { + // The config declares it empty, which is a value in its own right: with absent in the + // set that still covers adding the empty container and taking it away. + return []any{full} + } + trimmed = value.Slice(0, value.Len()-1).Interface() + + case reflect.Map: + keys := sortedMapKeys(value) + if len(keys) == 0 { + return []any{full} + } + value.SetMapIndex(keys[len(keys)-1], reflect.Value{}) + trimmed = value.Interface() + + default: + return nil + } + + return []any{full, trimmed} +} + +// clone deep-copies a value through the JSON representation the API uses, which the SDK +// types define themselves -- so ForceSendFields survives the copy. +func clone(value any) (any, error) { + body, err := json.Marshal(value) + if err != nil { + return nil, err + } + copied := reflect.New(reflect.TypeOf(value)) + if err := json.Unmarshal(body, copied.Interface()); err != nil { + return nil, err + } + return copied.Elem().Interface(), nil +} + +// sortedMapKeys orders a map's keys so that which entry gets dropped, and which entries a +// wildcard expands to, is the same on every run. A Go map has no order of its own. +func sortedMapKeys(value reflect.Value) []reflect.Value { + keys := value.MapKeys() + slices.SortFunc(keys, func(a, b reflect.Value) int { return strings.Compare(a.String(), b.String()) }) + return keys +} + +// expandPattern turns a pattern from the type walk into the concrete paths the deployed +// resource actually has: "tasks[*].description" against a resource with one task yields +// "tasks[0].description". A pattern with nothing behind it yields nothing, which is how a +// field under an absent container is reported as not covered. +func expandPattern(resource any, pattern string) []string { + paths := []string{""} + for _, seg := range splitPattern(pattern) { + var next []string + for _, prefix := range paths { + switch seg { + case "[*]": + value, err := valueAt(resource, prefix) + if err != nil || value.Kind() != reflect.Slice { + continue + } + for i := range value.Len() { + next = append(next, prefix+"["+strconv.Itoa(i)+"]") + } + case "*": + value, err := valueAt(resource, prefix) + if err != nil || value.Kind() != reflect.Map { + continue + } + for _, key := range sortedMapKeys(value) { + next = append(next, prefix+"["+quoteKey(key.String())+"]") + } + default: + next = append(next, joinPath(prefix, seg)) + } + } + paths = next + } + return paths +} + +// valueAt reads a path off the resource, with "" meaning the resource itself. +func valueAt(resource any, path string) (reflect.Value, error) { + if path == "" { + return reflect.Indirect(reflect.ValueOf(resource)), nil + } + value, err := structaccess.GetByString(resource, path) + if err != nil { + return reflect.Value{}, err + } + if isAbsent(value) { + return reflect.Value{}, fmt.Errorf("%s is absent", path) + } + return reflect.Indirect(reflect.ValueOf(value)), nil +} + +// splitPattern breaks a pattern into field names, "[*]" and "*" segments. +func splitPattern(pattern string) []string { + var segs []string + for part := range strings.SplitSeq(pattern, ".") { + // A dot-separated part is either the map wildcard on its own, or a field name + // followed by any number of [*] element wildcards. + if part == "*" { + segs = append(segs, "*") + continue + } + for { + name, rest, found := strings.Cut(part, "[*]") + if name != "" { + segs = append(segs, name) + } + if !found { + break + } + segs = append(segs, "[*]") + part = rest + } + } + return segs +} + +func joinPath(prefix, name string) string { + if prefix == "" { + return name + } + return prefix + "." + name +} + +// quoteKey renders a map key the way structpath parses it back. +func quoteKey(key string) string { + return "'" + strings.ReplaceAll(key, "'", "''") + "'" +} diff --git a/bundle/direct/autotest/values_unit_test.go b/bundle/direct/autotest/values_unit_test.go new file mode 100644 index 00000000000..00fe33ee516 --- /dev/null +++ b/bundle/direct/autotest/values_unit_test.go @@ -0,0 +1,385 @@ +package autotest + +import ( + "fmt" + "reflect" + "slices" + "strings" + "testing" + "unicode/utf8" + + "github.com/databricks/cli/libs/structs/structpath" + + "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/databricks-sdk-go/service/jobs" + "github.com/databricks/databricks-sdk-go/service/sql" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// A field's values form a complete digraph, so one Eulerian circuit covers every ordered +// pair exactly once and no second chain is ever needed. This pins that down: full +// coverage, no repeats, and every step starting where the last one ended. +func TestTransitionsCoverEveryPairInOneChain(t *testing.T) { + for n := 2; n <= 6; n++ { + values := make([]any, 0, n-1) + for i := 1; i < n; i++ { + values = append(values, i) + } + f := field{path: "some.field", values: values} //exhaustruct:ignore + + got := f.transitions() + + // n-1 explicit values plus the implicit absent. + want := n * (n - 1) + require.Len(t, got, want, "n=%d", n) + + seen := map[string]bool{} + for i, tr := range got { + pair := tr.label() + assert.False(t, seen[pair], "pair %s repeated at %d (n=%d)", pair, i, n) + seen[pair] = true + assert.NotEqual(t, valueLabel(tr.from), valueLabel(tr.to), "self-loop at %d", i) + if i > 0 { + assert.Equal(t, valueLabel(got[i-1].to), valueLabel(tr.from), + "step %d does not start where step %d ended (n=%d)", i, i-1, n) + } + } + assert.Len(t, seen, want, "n=%d", n) + } +} + +// A required field has no absent value, so its chain is one vertex smaller. +func TestTransitionsSkipAbsentForRequiredField(t *testing.T) { + f := field{path: "name", values: []any{"a", "b"}, required: true} //exhaustruct:ignore + + got := f.transitions() + + require.Len(t, got, 2) + for _, tr := range got { + assert.NotEqual(t, "absent", valueLabel(tr.from)) + assert.NotEqual(t, "absent", valueLabel(tr.to)) + } +} + +// A pattern from the type walk has to expand against the deployed resource, or the fields +// inside a container are silently never tested. Map wildcards ("tags.*") and element +// wildcards ("tasks[*]") take different paths through splitPattern, so both are pinned here. +func TestExpandPattern(t *testing.T) { + //exhaustruct:ignore + job := &resources.Job{JobSettings: jobs.JobSettings{ + Name: "n", + Tags: map[string]string{"team": "eng"}, + Tasks: []jobs.Task{ + {TaskKey: "a"}, //exhaustruct:ignore + {TaskKey: "b"}, //exhaustruct:ignore + }, + EmailNotifications: &jobs.JobEmailNotifications{OnFailure: []string{"a@b.test"}}, //exhaustruct:ignore + }} + + for _, tc := range []struct { + pattern string + want []string + }{ + {"name", []string{"name"}}, + {"tags.*", []string{"tags['team']"}}, + {"tasks[*].task_key", []string{"tasks[0].task_key", "tasks[1].task_key"}}, + {"email_notifications.on_failure[*]", []string{"email_notifications.on_failure[0]"}}, + // Nothing behind it in the resource, which is what makes a field "not covered". + {"job_clusters[*].job_cluster_key", nil}, + {"tasks[0].notebook_task.base_parameters.*", nil}, + } { + t.Run(tc.pattern, func(t *testing.T) { + assert.Equal(t, tc.want, expandPattern(job, tc.pattern)) + }) + } +} + +// setValue applies one edit the way setField does, without a bundle around it. +func setValue(resource any, path string, value any) error { + node, err := structpath.ParsePath(path) + if err != nil { + return err + } + return setNode(resource, node, value) +} + +// setField writes into the typed resource, where "absent" is the zero value with the field +// dropped from ForceSendFields -- the distinction the API sees. A map entry and a slice +// element have to be removed outright, which is a different code path. +func TestSetFieldAbsentAndEmpty(t *testing.T) { + //exhaustruct:ignore + job := &resources.Job{JobSettings: jobs.JobSettings{ + Name: "n", + Tags: map[string]string{"team": "eng", "env": "dev"}, + Tasks: []jobs.Task{ + {TaskKey: "a"}, //exhaustruct:ignore + {TaskKey: "b"}, //exhaustruct:ignore + }, + }} + + require.NoError(t, setValue(job, "description", "")) + assert.Contains(t, job.ForceSendFields, "Description") + + require.NoError(t, setValue(job, "description", nil)) + assert.NotContains(t, job.ForceSendFields, "Description") + + // A nested object the config leaves out is allocated on the way in. + require.NoError(t, setValue(job, "email_notifications.no_alert_for_skipped_runs", true)) + require.NotNil(t, job.EmailNotifications) + assert.True(t, job.EmailNotifications.NoAlertForSkippedRuns) + + require.NoError(t, setValue(job, "tags['env']", nil)) + assert.Equal(t, map[string]string{"team": "eng"}, job.Tags) + + // Only the last element can be made absent: removing an earlier one would shift its + // successor into the same path, so the field would still be there holding another value. + require.ErrorContains(t, setValue(job, "tasks[0]", nil), "would shift into it") + require.NoError(t, setValue(job, "tasks[1]", nil)) + assert.Equal(t, []jobs.Task{{TaskKey: "a"}}, job.Tasks) //exhaustruct:ignore + require.NoError(t, setValue(job, "tasks[0]", nil)) + assert.Empty(t, job.Tasks) + + // A list from the value library arrives as []any and has to be decoded into the + // field's own element type. + require.NoError(t, setValue(job, "email_notifications.on_failure", []any{"a@b.test"})) + assert.Equal(t, []string{"a@b.test"}, job.EmailNotifications.OnFailure) +} + +// A skip key may be a pattern, since a field inside a slice has no fixed index and the +// value library cannot name one. Written as a table because the three key shapes -- exact, +// element wildcard, subtree -- take different paths through the matcher. +func TestSkipReasonMatchesPatterns(t *testing.T) { + fv := &fieldValues{skip: map[string]string{ + "storage_root": "needs an external location", + "aliases[*].id": "assigned by the backend", + "telemetry_config.*": "only accepted as a whole", + }} //exhaustruct:ignore + + for _, tc := range []struct { + path string + reason string + }{ + {"storage_root", "needs an external location"}, + {"aliases[0].id", "assigned by the backend"}, + {"aliases[3].id", "assigned by the backend"}, + {"telemetry_config.enabled", "only accepted as a whole"}, + // A pattern of its own, which is what a field under an absent container is. + {"telemetry_config.sinks[*].name", "only accepted as a whole"}, + {"aliases[0].alias_name", ""}, + {"comment", ""}, + // A key naming a whole subtree, written with the trailing wildcard, still covers a + // pattern beneath it -- which is the form a field under an absent container takes. + {"telemetry_config.sinks[*].endpoint", "only accepted as a whole"}, + } { + t.Run(tc.path, func(t *testing.T) { + reason, skipped := fv.skipReason(tc.path) + assert.Equal(t, tc.reason != "", skipped) + assert.Equal(t, tc.reason, reason) + }) + } +} + +// The generated required-field data is keyed by pattern, so a concrete index has to be +// turned back into a wildcard before the lookup -- otherwise a required field inside a +// slice looks optional and gets an "absent" transition a user cannot deploy. +func TestIsRequiredInsideSlice(t *testing.T) { + assert.True(t, isRequired("jobs", "tasks[0].task_key")) + assert.True(t, isRequired("jobs", "tasks[3].task_key")) + assert.False(t, isRequired("jobs", "tasks[0].description")) + + assert.True(t, isRequired("alerts", "display_name")) + assert.False(t, isRequired("alerts", "custom_summary")) +} + +// Every id in a message has to redact to the same placeholder on both sides, or a report from +// a real workspace could never match one from the fake server: the fake hands out UUIDs where +// a workspace hands out hex. +func TestRedactIDs(t *testing.T) { + for _, tc := range []struct{ in, want string }{ + {"updating id=79f5e1f2-de43-03a0-79f5-e1f2de4303a1: nope", "updating id=[ID]: nope"}, + {"updating id=00145E75FBD6DCBB: nope", "updating id=[ID]: nope"}, + {"Catalog 'test-schema-f0ad2bd52de6e45119d42' missing", "Catalog 'test-schema-[UNIQUE_NAME]' missing"}, + {"principal 79f5e1f2-de43-03a0-79f5-e1f2de4303a1 denied", "principal [UUID] denied"}, + } { + assert.Equal(t, tc.want, redactIDs(tc.in)) + } +} + +// The planner names a slice element by a key-value selector where this suite names it by index, +// so the two forms have to be recognised as the same field -- otherwise a change recorded +// against "tasks[task_key='seeded'].run_if" looks unrelated to "tasks[0].run_if". +func TestSameField(t *testing.T) { + for _, tc := range []struct { + drifting, path string + want bool + }{ + {"comment", "comment", true}, + {"tasks", "tasks[0].run_if", true}, + {"tasks[task_key='seeded'].run_if", "tasks[0].run_if", true}, + {"tasks[task_key='seeded']", "tasks[0].run_if", true}, + {"tasks[task_key='seeded'].notebook_task.source", "tasks[0].notebook_task", true}, + {"config", "config.auto_capture_config.enabled", true}, + {"comment", "name", false}, + {"tasks[task_key='seeded'].run_if", "tasks[0].timeout_seconds", false}, + } { + assert.Equal(t, tc.want, sameField(tc.drifting, tc.path), "%s vs %s", tc.drifting, tc.path) + } +} + +// An SDK enum declares its own values, and the *_UNSPECIFIED member is a protobuf sentinel for +// "unset" rather than a choice: asking for it reads as the write being ignored, since the +// backend normalizes it away. +func TestEnumValues(t *testing.T) { + // AlertOperator has no UNSPECIFIED member, so the two alphabetically-first values come back. + got := enumValues(reflect.TypeFor[sql.AlertOperator]()) + require.Len(t, got, 2) + for _, value := range got { + assert.IsType(t, sql.AlertOperator(""), value) + assert.NotContains(t, value, "UNSPECIFIED") + } + + // SpotInstancePolicy does have one, and it must not be offered. + got = enumValues(reflect.TypeFor[sql.SpotInstancePolicy]()) + require.NotEmpty(t, got) + for _, value := range got { + assert.NotContains(t, fmt.Sprint(value), "UNSPECIFIED") + } + + // A plain string is not an enum, whatever the backend constrains it to. + assert.Nil(t, enumValues(reflect.TypeFor[string]())) +} + +// A container's values are the resource's own and that value with one entry dropped, both deep +// copies -- a shallow one would share the pointers inside an element with the live resource, so +// a later edit would reach into a value recorded here. +func TestContainerValuesAreIndependent(t *testing.T) { + //exhaustruct:ignore + job := &resources.Job{JobSettings: jobs.JobSettings{ + Tasks: []jobs.Task{ + {TaskKey: "a", NotebookTask: &jobs.NotebookTask{NotebookPath: "one"}}, //exhaustruct:ignore + {TaskKey: "b"}, //exhaustruct:ignore + }, + }} + + values := containerValues(job, "tasks") + require.Len(t, values, 2) + full := values[0].([]jobs.Task) + trimmed := values[1].([]jobs.Task) + require.Len(t, full, 2) + require.Len(t, trimmed, 1) + + // Editing the live resource through the pointer inside an element must not be visible in + // either recorded value. + require.NoError(t, setValue(job, "tasks[0].notebook_task.notebook_path", "two")) + assert.Equal(t, "one", full[0].NotebookTask.NotebookPath) + assert.Equal(t, "one", trimmed[0].NotebookTask.NotebookPath) +} + +// A value long enough or punctuated enough to be unreadable as a subtest name gets a trimmed, +// digest-suffixed label -- stable, and safe to paste back into a test filter. +func TestShortLabel(t *testing.T) { + assert.Equal(t, "x", shortLabel("x")) + assert.Equal(t, "dbfs:/FileStore/a.jar", shortLabel("dbfs:/FileStore/a.jar")) + + long := shortLabel(`{"spark_version":{"type":"fixed","value":"13.3"}}`) + assert.NotContains(t, long, `"`) + assert.LessOrEqual(t, len(long), 24) + // Two values sharing a prefix still get different labels. + assert.NotEqual(t, long, shortLabel(`{"spark_version":{"type":"fixed","value":"14.3"}}`)) + // And the same value always gets the same one. + assert.Equal(t, long, shortLabel(`{"spark_version":{"type":"fixed","value":"13.3"}}`)) +} + +// A sampled run is held to the rows of the fields it picked. Section headers, blank lines and +// the summary have no per-field meaning and are dropped from both sides; a row the harness +// records against itself is always kept, since it means the type did not run at all. +func TestSampledRows(t *testing.T) { + body := strings.Join([]string{ + "=== schema.yml.tmpl", + "comment x absent UPDATE_IGNORED", + "properties keys1 absent UPDATE_IGNORED", + "(base config) BASE_ERROR boom", + "", + "=== summary", + "OK 16", + "", + }, "\n") + + kept := sampledRows(body, map[string]bool{"properties": true}) + assert.Equal(t, "properties keys1 absent UPDATE_IGNORED\n"+ + "(base config) BASE_ERROR boom\n", kept) + + // Nothing sampled still keeps the harness's own row. + assert.Equal(t, "(base config) BASE_ERROR boom\n", + sampledRows(body, map[string]bool{})) +} + +// The draw is by seed alone, so the same seed picks the same fields whatever order enumeration +// produced, and the picks are recorded for the golden comparison. +func TestSample(t *testing.T) { + var fields []field + for _, path := range []string{"a", "b", "c", "d", "e"} { + fields = append(fields, field{path: path}) //exhaustruct:ignore + } + + rep := &report{resourceType: "t"} //exhaustruct:ignore + got := pickSample(fields, 7, 2, rep) + require.Len(t, got, 2) + assert.Len(t, rep.sampled, 2) + for _, f := range got { + assert.True(t, rep.sampled[f.path], "%s recorded", f.path) + } + + // Same seed, reversed input: the same two fields. + reversed := slices.Clone(fields) + slices.Reverse(reversed) + again := pickSample(reversed, 7, 2, &report{resourceType: "t"}) //exhaustruct:ignore + assert.Equal(t, paths(got), paths(again)) + // The seed is what decides the pick, so some seed picks something else. Scanned rather than + // asserted against one other seed: two of five fields collide once in ten, which would make + // this test fail on its own about as often as it caught anything. + distinct := map[string]bool{} + for seed := range uint64(20) { + distinct[strings.Join(paths(pickSample(fields, seed, 2, &report{resourceType: "t"})), ",")] = true //exhaustruct:ignore + } + assert.Greater(t, len(distinct), 1, "the seed decides the pick") + + // A type with no more fields than the sample size is covered whole, and records nothing: + // its report stays comparable in full, summary included. + whole := &report{resourceType: "t"} //exhaustruct:ignore + assert.Len(t, pickSample(fields, 7, 6, whole), 5) + assert.Nil(t, whole.sampled) +} + +func paths(fields []field) []string { + out := make([]string, 0, len(fields)) + for _, f := range fields { + out = append(out, f.path) + } + slices.Sort(out) + return out +} + +// A row's detail is one line, and a long URL in the middle of an error pushes out the part that +// says what went wrong. +func TestOneLineKeepsTheCause(t *testing.T) { + err := `cannot plan resources.registered_models.foo: reading id="main.default.test-model-abc": ` + + `Get "https://dbc-61ef35eb-01e7.cloud.databricks.com/api/2.1/unity-catalog/models/main.default.test-model-abc?": ` + + `read tcp 10.0.0.1:1234->3.4.5.6:443: read: operation timed out` + + line := oneLine(err) + assert.Contains(t, line, "operation timed out") + assert.NotContains(t, line, "dbc-61ef35eb") + assert.LessOrEqual(t, len(line), 143) +} + +// Truncation is by rune, since an API message can carry any character and the ellipsis oneLine +// inserts for a URL is itself multi-byte: a byte index can land inside one. +func TestOneLineKeepsValidUTF8(t *testing.T) { + // Long enough to truncate, and multi-byte throughout so every cut lands inside a rune if the + // truncation is done by byte. + line := oneLine(strings.Repeat("héllo wörld ", 20)) + assert.True(t, utf8.ValidString(line), "%q is not valid UTF-8", line) + assert.Contains(t, line, "...") +} diff --git a/bundle/direct/dresources/all_test.go b/bundle/direct/dresources/all_test.go index 36eaa3e27df..9114ca91884 100644 --- a/bundle/direct/dresources/all_test.go +++ b/bundle/direct/dresources/all_test.go @@ -41,6 +41,15 @@ var testConfig map[string]any = map[string]any{ }, }, + // A policy has to say what it constrains: the API rejects one with no definition and no + // policy family to take one from. + "cluster_policies": &resources.ClusterPolicy{ + CreatePolicy: compute.CreatePolicy{ + Name: "mypolicy", + Definition: `{"spark_version":{"type":"fixed","value":"13.3.x-scala2.12"}}`, + }, + }, + "catalogs": &resources.Catalog{ CreateCatalog: catalog.CreateCatalog{ Name: "mycatalog", @@ -82,6 +91,16 @@ var testConfig map[string]any = map[string]any{ }, }, + // The API requires all three of these on a create, and libs/testserver applies the same checks: + // a warehouse with no name, no cluster_size or max_num_clusters outside 1..40 is refused. + "sql_warehouses": &resources.SqlWarehouse{ + CreateWarehouseRequest: sql.CreateWarehouseRequest{ + Name: "my-warehouse", + ClusterSize: "2X-Small", + MaxNumClusters: 1, + }, + }, + "instance_pools": &resources.InstancePool{ CreateInstancePool: compute.CreateInstancePool{ InstancePoolName: "my-instance-pool", diff --git a/libs/structs/structaccess/bundle_test.go b/libs/structs/structaccess/bundle_test.go index 1d75afe0b10..895ef9820db 100644 --- a/libs/structs/structaccess/bundle_test.go +++ b/libs/structs/structaccess/bundle_test.go @@ -76,3 +76,38 @@ func TestGet_ConfigRoot_JobTagsAccess(t *testing.T) { require.Error(t, ValidateByString(reflect.TypeFor[config.Root](), "resources.apps.my_app.url.inner")) require.Error(t, ValidateByString(reflect.TypeFor[config.Root](), "resources.apps.my_app.url1")) } + +// A bundle resource embeds a config struct that embeds the SDK request struct, so its +// fields sit two levels down. Get, Set and ValidatePath all have to reach them, and +// ForceSendFields belongs to the struct that declares the field -- not to the outer one +// that shadows the name. +func TestGetSet_DoublyEmbeddedField(t *testing.T) { + project := &resources.PostgresProject{} //exhaustruct:ignore + project.ProjectId = "p" + + require.NoError(t, ValidateByString(reflect.TypeOf(project), "budget_policy_id")) + + require.NoError(t, SetByString(project, "budget_policy_id", "abc")) + require.Equal(t, "abc", project.BudgetPolicyId) + + value, err := GetByString(project, "budget_policy_id") + require.NoError(t, err) + require.Equal(t, "abc", value) + + // An explicit empty value is recorded on ProjectSpec, which declares the field. + require.NoError(t, SetByString(project, "budget_policy_id", "")) + require.Contains(t, project.ProjectSpec.ForceSendFields, "BudgetPolicyId") + require.NotContains(t, project.ForceSendFields, "BudgetPolicyId") + + value, err = GetByString(project, "budget_policy_id") + require.NoError(t, err) + // The empty string, not nil: that is what separates an explicit "" from an absent field. + require.Equal(t, any(""), value) + + // And dropping it again leaves the field absent. + require.NoError(t, SetByString(project, "budget_policy_id", nil)) + require.NotContains(t, project.ProjectSpec.ForceSendFields, "BudgetPolicyId") + value, err = GetByString(project, "budget_policy_id") + require.NoError(t, err) + require.Nil(t, value) +} diff --git a/libs/structs/structaccess/get.go b/libs/structs/structaccess/get.go index a5433adee71..a56e4404852 100644 --- a/libs/structs/structaccess/get.go +++ b/libs/structs/structaccess/get.go @@ -138,19 +138,13 @@ func Get(v any, path *structpath.PathNode) (any, error) { func accessKey(v reflect.Value, key string, path *structpath.PathNode) (reflect.Value, error) { switch v.Kind() { case reflect.Struct: - // Precalculate ForceSendFields mappings for this struct hierarchy - forceSendFieldsMap := getForceSendFieldsForFromTyped(v) - - fv, sf, embeddedIndex, ok := findStructFieldByKey(v, key) + fv, sf, owner, ok := findStructFieldByKey(v, key) if !ok { return reflect.Value{}, fmt.Errorf("%s: field %q not found in %s", path.String(), key, v.Type()) } - // Check ForceSendFields using precalculated map - var force bool - if fields, exists := forceSendFieldsMap[embeddedIndex]; exists { - force = containsString(fields, sf.Name) - } + // ForceSendFields is only managed by the struct that declares the field. + force := forceSendFieldsContains(owner, sf.Name) // Honor omitempty: if present and value is empty and not forced, treat as omitted (nil). jsonTag := structtag.JSONTag(sf.Tag.Get("json")) @@ -270,28 +264,92 @@ func findFieldInStruct(v reflect.Value, key string) (reflect.Value, reflect.Stru // findStructFieldByKey searches exported fields of struct v for a field matching key. // It matches json tag name (when present and not "-") only. -// It also searches embedded anonymous structs (flattening semantics). -// Returns: fieldValue, structField, embeddedIndex, found -// embeddedIndex is -1 for direct fields, or the index of the embedded struct containing the field. -func findStructFieldByKey(v reflect.Value, key string) (reflect.Value, reflect.StructField, int, bool) { +// It also searches embedded anonymous structs recursively (flattening semantics), which +// FindStructFieldByKeyType does too: a bundle resource embeds a config struct that embeds +// the SDK request struct, so its fields sit two levels down. +// Returns: fieldValue, structField, owner (the struct value declaring the field), found +func findStructFieldByKey(v reflect.Value, key string) (reflect.Value, reflect.StructField, reflect.Value, bool) { t := v.Type() // First pass: direct fields if fv, sf, found := findFieldInStruct(v, key); found { - return fv, sf, -1, true + return fv, sf, v, true } - // Second pass: search embedded anonymous structs (flattening semantics) + // Ambiguity is a property of the type, not of the value: a name declared at the same + // embedding depth by two members is one encoding/json omits, whether or not one of those + // members happens to be a nil pointer right now. Asking the type walk first keeps Get, Set + // and ValidatePattern agreeing on which names exist. + if _, _, ok := FindStructFieldByKeyType(t, key); !ok { + return reflect.Value{}, reflect.StructField{}, reflect.Value{}, false + } + + // Second pass: search embedded anonymous structs (flattening semantics) breadth-first, one + // level of embedding at a time. Not depth-first: encoding/json resolves a name declared at + // two embedding depths in favour of the shallower one, so descending fully into the first + // embed could pick a field three levels down over the same name two levels down in a + // later one -- and then reading or writing the field would not be the field serialized + // under that name. + // Guards against a cyclic embedding (a struct embedding a pointer to itself), which would + // otherwise enqueue the same type forever when the key is not found at all. Only types from + // *earlier* levels are excluded: a type reachable twice within one level is a diamond, and + // the two matches it produces are exactly the ambiguity json resolves by omitting the field. + seen := map[reflect.Type]bool{v.Type(): true} + level := embeddedStructs(v) + for len(level) > 0 { + var next []reflect.Value + var found []struct { + value reflect.Value + field reflect.StructField + owner reflect.Value + } + for _, fv := range level { + if out, sf, ok := findFieldInStruct(fv, key); ok { + found = append(found, struct { + value reflect.Value + field reflect.StructField + owner reflect.Value + }{out, sf, fv}) + continue + } + for _, deeper := range embeddedStructs(fv) { + if seen[deeper.Type()] { + continue + } + next = append(next, deeper) + } + } + for _, fv := range next { + seen[fv.Type()] = true + } + if len(found) == 1 { + return found[0].value, found[0].field, found[0].owner, true + } + if len(found) > 1 { + // Two embedded structs declare the same name at the same depth. encoding/json calls + // that ambiguous and omits the field entirely, so there is no field to read or + // write: picking one would target data that is never serialized. + return reflect.Value{}, reflect.StructField{}, reflect.Value{}, false + } + level = next + } + + return reflect.Value{}, reflect.StructField{}, reflect.Value{}, false +} + +// embeddedStructs returns the anonymous struct fields of v, dereferenced, skipping any that +// cannot be descended into. +func embeddedStructs(v reflect.Value) []reflect.Value { + var out []reflect.Value + t := v.Type() for i := range t.NumField() { - sf := t.Field(i) - if !sf.Anonymous { + if !t.Field(i).Anonymous { continue } fv := v.Field(i) - // Dereference pointer anonymous structs for fv.Kind() == reflect.Pointer { if fv.IsNil() { - // Not initialized; can't descend + // Not initialized; can't descend. break } fv = fv.Elem() @@ -299,59 +357,35 @@ func findStructFieldByKey(v reflect.Value, key string) (reflect.Value, reflect.S if fv.Kind() != reflect.Struct { continue } - if out, osf, found := findFieldInStruct(fv, key); found { - return out, osf, i, true - } + out = append(out, fv) } - - return reflect.Value{}, reflect.StructField{}, -1, false + return out } -// getForceSendFieldsForFromTyped collects ForceSendFields values for FromTyped operations -// Returns map[structKey][]fieldName where structKey is -1 for direct fields, embedded index for embedded fields -func getForceSendFieldsForFromTyped(v reflect.Value) map[int][]string { - if !v.IsValid() || v.Type().Kind() != reflect.Struct { - return make(map[int][]string) +// forceSendFields returns the ForceSendFields slice a struct declares itself. A struct that +// embeds another shadows it deliberately -- see resources.PostgresProjectConfig -- so only +// the declaring struct tracks a field of its own. +func forceSendFields(owner reflect.Value) reflect.Value { + if !owner.IsValid() || owner.Kind() != reflect.Struct { + return reflect.Value{} } - - result := make(map[int][]string) - - for i := range v.Type().NumField() { - field := v.Type().Field(i) - fieldValue := v.Field(i) - + for i := range owner.Type().NumField() { + field := owner.Type().Field(i) if field.Name == "ForceSendFields" && !field.Anonymous { - // Direct ForceSendFields (structKey = -1) - if fields, ok := reflect.TypeAssert[[]string](fieldValue); ok { - result[-1] = fields - } - } else if field.Anonymous { - // Embedded struct - check for ForceSendFields inside it - if embeddedStruct := getEmbeddedStructForReading(fieldValue); embeddedStruct.IsValid() { - if forceSendField := embeddedStruct.FieldByName("ForceSendFields"); forceSendField.IsValid() { - if fields, ok := reflect.TypeAssert[[]string](forceSendField); ok { - result[i] = fields - } - } - } + return owner.Field(i) } } - - return result + return reflect.Value{} } -// Helper function for reading - doesn't create nil pointers -func getEmbeddedStructForReading(fieldValue reflect.Value) reflect.Value { - if fieldValue.Kind() == reflect.Pointer { - if fieldValue.IsNil() { - return reflect.Value{} // Don't create, just return invalid - } - fieldValue = fieldValue.Elem() - } - if fieldValue.Kind() == reflect.Struct { - return fieldValue +// forceSendFieldsContains reports whether a struct forces the named field to be sent. +func forceSendFieldsContains(owner reflect.Value, name string) bool { + fsf := forceSendFields(owner) + if !fsf.IsValid() { + return false } - return reflect.Value{} + fields, ok := reflect.TypeAssert[[]string](fsf) + return ok && containsString(fields, name) } // containsString checks if a slice contains a specific string diff --git a/libs/structs/structaccess/set.go b/libs/structs/structaccess/set.go index 2285d470fa9..187ad8b41fd 100644 --- a/libs/structs/structaccess/set.go +++ b/libs/structs/structaccess/set.go @@ -130,7 +130,7 @@ func setFieldOrMapValue(parentVal reflect.Value, key string, valueVal reflect.Va // setStructField sets a field in a struct and handles ForceSendFields func setStructField(parentVal reflect.Value, fieldName string, valueVal reflect.Value) error { - fv, sf, embeddedIndex, ok := findStructFieldByKey(parentVal, fieldName) + fv, sf, owner, ok := findStructFieldByKey(parentVal, fieldName) if !ok { return fmt.Errorf("field %q not found in %s", fieldName, parentVal.Type()) } @@ -140,13 +140,26 @@ func setStructField(parentVal reflect.Value, fieldName string, valueVal reflect. return fmt.Errorf("field %q cannot be set", sf.Name) } - // Handle ForceSendFields: remove if setting nil, add if setting empty value - err := updateForceSendFields(parentVal, sf.Name, embeddedIndex, valueVal, sf) + // Assign first: a value that cannot be converted must leave the struct exactly as it was, + // and updating ForceSendFields before the assignment left the field's send-behaviour + // changed after a failed Set. + converted, err := convertValue(valueVal, fv.Type()) if err != nil { return err } + if err := assignValue(fv, converted); err != nil { + return err + } - return assignValue(fv, valueVal) + // ForceSendFields is decided from the converted value, not the caller's: setting an + // omitempty int64 from the string "0" stores 0, which is empty and must be forced, while + // the string "0" is not empty and would have left the field to be omitted. + if !valueVal.IsValid() { + // Setting nil: the field is being made absent, which convertValue renders as the zero + // value. Pass the invalid value through so it is removed from ForceSendFields. + return updateForceSendFields(owner, sf.Name, valueVal, sf) + } + return updateForceSendFields(owner, sf.Name, converted, sf) } // setMapValue sets a value in a map @@ -303,7 +316,7 @@ func convertValue(valueVal reflect.Value, targetType reflect.Type) (reflect.Valu // - If setting nil: remove field from ForceSendFields // - If setting empty value: add field to ForceSendFields (if not already present) // Only applies to fields with omitempty tag -func updateForceSendFields(parentVal reflect.Value, fieldName string, embeddedIndex int, valueVal reflect.Value, structField reflect.StructField) error { +func updateForceSendFields(owner reflect.Value, fieldName string, valueVal reflect.Value, structField reflect.StructField) error { isSettingNil := !valueVal.IsValid() isSettingEmptyValue := valueVal.IsValid() && isEmptyForOmitEmpty(valueVal) @@ -319,8 +332,8 @@ func updateForceSendFields(parentVal reflect.Value, fieldName string, embeddedIn return nil } - // Find the appropriate ForceSendFields slice to modify - forceSendFieldsSlice := findForceSendFieldsForSetting(parentVal, embeddedIndex) + // Only the struct that declares the field tracks it. + forceSendFieldsSlice := forceSendFields(owner) if !forceSendFieldsSlice.IsValid() { // No ForceSendFields to update return nil @@ -337,64 +350,10 @@ func updateForceSendFields(parentVal reflect.Value, fieldName string, embeddedIn return nil } -// findForceSendFieldsForSetting finds the correct ForceSendFields slice to modify -// This should match the logic in get.go's getForceSendFieldsForFromTyped -// Only the struct that contains the ForceSendFields can manage its own fields -// embeddedIndex: -1 for direct fields, or the index of the embedded struct -func findForceSendFieldsForSetting(parentVal reflect.Value, embeddedIndex int) reflect.Value { - if embeddedIndex == -1 { - // Direct field - check if parent struct has its own ForceSendFields - // We need to check the struct type directly, not through field promotion - parentType := parentVal.Type() - for i := range parentType.NumField() { - field := parentType.Field(i) - if field.Name == "ForceSendFields" && !field.Anonymous { - // Parent has direct ForceSendFields - return parentVal.Field(i) - } - } - // Parent struct has no direct ForceSendFields, so no management possible - return reflect.Value{} - } else { - // Embedded field - look for ForceSendFields in the embedded struct - embeddedField := parentVal.Field(embeddedIndex) - embeddedStruct := getEmbeddedStructForSetting(embeddedField) - if !embeddedStruct.IsValid() { - return reflect.Value{} - } - fsf := embeddedStruct.FieldByName("ForceSendFields") - if fsf.IsValid() { - return fsf - } - // Embedded struct has no ForceSendFields, so no management possible - return reflect.Value{} - } -} - -// getEmbeddedStructForSetting gets the embedded struct for setting operations -// Creates nil pointers if needed -func getEmbeddedStructForSetting(fieldValue reflect.Value) reflect.Value { - if fieldValue.Kind() == reflect.Pointer { - if fieldValue.IsNil() { - // Create new instance if needed - if fieldValue.CanSet() { - fieldValue.Set(reflect.New(fieldValue.Type().Elem())) - } else { - return reflect.Value{} - } - } - fieldValue = fieldValue.Elem() - } - if fieldValue.Kind() == reflect.Struct { - return fieldValue - } - return reflect.Value{} -} - // removeFromForceSendFields removes fieldName from the ForceSendFields slice func removeFromForceSendFields(forceSendFieldsSlice reflect.Value, fieldName string) { // Get the original []string slice - fields := forceSendFieldsSlice.Interface().([]string) + fields, _ := reflect.TypeAssert[[]string](forceSendFieldsSlice) // Find the index of the field to remove index := slices.Index(fields, fieldName) @@ -410,7 +369,7 @@ func removeFromForceSendFields(forceSendFieldsSlice reflect.Value, fieldName str // addToForceSendFields adds fieldName to the ForceSendFields slice if not already present func addToForceSendFields(forceSendFieldsSlice reflect.Value, fieldName string) { // Get the original []string slice - fields := forceSendFieldsSlice.Interface().([]string) + fields, _ := reflect.TypeAssert[[]string](forceSendFieldsSlice) // Check if already present if slices.Contains(fields, fieldName) { diff --git a/libs/structs/structaccess/set_test.go b/libs/structs/structaccess/set_test.go index 1b294494eca..05a852b0f77 100644 --- a/libs/structs/structaccess/set_test.go +++ b/libs/structs/structaccess/set_test.go @@ -1,6 +1,8 @@ package structaccess_test import ( + "encoding/json" + "reflect" "testing" "github.com/databricks/cli/libs/structs/structaccess" @@ -791,3 +793,192 @@ func TestSet_MixedForceSendFields(t *testing.T) { assert.Equal(t, []string{"SecondFieldOmit"}, obj.Second.ForceSendFields) // no duplicates }) } + +// encoding/json resolves a name declared at two embedding depths in favour of the shallower +// one. Get and Set have to agree with it, so the embedded search goes level by level: a +// depth-first search would find Deep.Value first, since its embed is declared first. +type deepValue struct { + Value string `json:"value"` +} + +type deepEmbed struct { + deepValue +} + +type shallowEmbed struct { + Value string `json:"value"` +} + +type deeperEmbed struct { + deepEmbed +} + +type embedDepths struct { + deepEmbed + shallowEmbed +} + +// The same name three levels down in the first member, against two levels down in a later +// one. json picks the shallower, so the search has to be breadth-first across the whole tree +// rather than depth-first per member. +type embedDepthsAcrossMembers struct { + deeperEmbed + deepEmbed +} + +func TestSet_ShallowerEmbedWinsAcrossMembers(t *testing.T) { + target := &embedDepthsAcrossMembers{} + + require.NoError(t, structaccess.SetByString(target, "value", "set")) + assert.Equal(t, "set", target.Value) + assert.Empty(t, target.deeperEmbed.Value) + + require.NoError(t, structaccess.ValidateByString(reflect.TypeOf(target), "value")) + + blob, err := json.Marshal(target) + require.NoError(t, err) + assert.JSONEq(t, `{"value":"set"}`, string(blob)) +} + +func TestSet_ShallowerEmbedWins(t *testing.T) { + target := &embedDepths{} + + require.NoError(t, structaccess.SetByString(target, "value", "set")) + assert.Equal(t, "set", target.Value) + assert.Empty(t, target.deepEmbed.Value) + + got, err := structaccess.GetByString(target, "value") + require.NoError(t, err) + assert.Equal(t, "set", got) + + // The same field json.Marshal picks, which is the contract being matched. + blob, err := json.Marshal(target) + require.NoError(t, err) + assert.JSONEq(t, `{"value":"set"}`, string(blob)) +} + +// Two embedded structs declaring one name at the same depth: encoding/json calls that +// ambiguous and omits the field, so there is nothing to read or write either. +type ambiguousA struct { + Value string `json:"value"` +} + +type ambiguousB struct { + Value string `json:"value"` +} + +type ambiguousEmbeds struct { + ambiguousA + ambiguousB //nolint:govet // the repeated json tag is the point: both embeds declare "value" +} + +func TestSet_AmbiguousEmbedIsNotFound(t *testing.T) { + target := &ambiguousEmbeds{} + + require.Error(t, structaccess.SetByString(target, "value", "set")) + require.Error(t, structaccess.ValidateByString(reflect.TypeOf(target), "value")) + + // Which is what json does with it: the name resolves to no field at all. + blob, err := json.Marshal(target) + require.NoError(t, err) + assert.JSONEq(t, `{}`, string(blob)) +} + +// A value that cannot be converted must leave the struct untouched, ForceSendFields included. +func TestSet_FailedConversionLeavesForceSendFields(t *testing.T) { + job := &jobs.JobSettings{Name: "n"} //exhaustruct:ignore + + require.Error(t, structaccess.SetByString(job, "max_concurrent_runs", "not-a-number")) + assert.Empty(t, job.ForceSendFields) + assert.Equal(t, "n", job.Name) +} + +// A struct embedding a pointer to itself: the search must not walk the same type twice, or a +// key it never finds sends it round forever. +type cyclicEmbed struct { + *cyclicEmbed + Name string `json:"name"` +} + +func TestGet_CyclicEmbedTerminates(t *testing.T) { + target := &cyclicEmbed{Name: "n"} //exhaustruct:ignore + target.cyclicEmbed = target + + got, err := structaccess.GetByString(target, "name") + require.NoError(t, err) + assert.Equal(t, "n", got) + + _, err = structaccess.GetByString(target, "nope") + require.Error(t, err) + require.Error(t, structaccess.ValidateByString(reflect.TypeOf(target), "nope")) +} + +// A diamond: two embeds reaching one type, so the name sits at the same depth twice. +// encoding/json omits it, and the search has to see both paths to notice. +type diamondLeaf struct { + Value string `json:"value"` +} + +type diamondLeft struct { + diamondLeaf +} + +type diamondRight struct { + diamondLeaf +} + +type diamondEmbeds struct { + diamondLeft + diamondRight +} + +func TestSet_DiamondEmbedIsAmbiguous(t *testing.T) { + target := &diamondEmbeds{} + + require.Error(t, structaccess.SetByString(target, "value", "set")) + require.Error(t, structaccess.ValidateByString(reflect.TypeOf(target), "value")) + + blob, err := json.Marshal(target) + require.NoError(t, err) + assert.JSONEq(t, `{}`, string(blob)) +} + +// ForceSendFields is decided from the value actually stored, not the one the caller passed: +// setting an omitempty numeric field from the string "0" stores zero, which has to be forced +// or the field marshals as absent. +func TestSet_StringZeroIntoOmitemptyNumberIsForced(t *testing.T) { + job := &jobs.JobSettings{Name: "n"} //exhaustruct:ignore + + require.NoError(t, structaccess.SetByString(job, "max_concurrent_runs", "0")) + assert.Equal(t, 0, job.MaxConcurrentRuns) + assert.Contains(t, job.ForceSendFields, "MaxConcurrentRuns") + + blob, err := json.Marshal(job) + require.NoError(t, err) + assert.Contains(t, string(blob), `"max_concurrent_runs":0`) +} + +// Whether a name is ambiguous is a property of the type: two embedded pointers declaring it at +// the same depth make it one encoding/json omits, and that must not change with whether one of +// them happens to be nil right now. +type ambiguousPtrEmbeds struct { + *ambiguousA + *ambiguousB //nolint:govet // the repeated json tag is the point: both embeds declare "value" +} + +func TestSet_AmbiguousPointerEmbedsIgnoreNilness(t *testing.T) { + // One embed present, the other nil: the name is still ambiguous. + target := &ambiguousPtrEmbeds{ambiguousA: &ambiguousA{}} //exhaustruct:ignore + require.Error(t, structaccess.SetByString(target, "value", "set")) + _, err := structaccess.GetByString(target, "value") + require.Error(t, err) + + // And with both present, unchanged. + target = &ambiguousPtrEmbeds{ambiguousA: &ambiguousA{}, ambiguousB: &ambiguousB{}} + require.Error(t, structaccess.SetByString(target, "value", "set")) + require.Error(t, structaccess.ValidateByString(reflect.TypeOf(target), "value")) + + blob, err := json.Marshal(target) + require.NoError(t, err) + assert.JSONEq(t, `{}`, string(blob)) +} diff --git a/libs/structs/structaccess/typecheck.go b/libs/structs/structaccess/typecheck.go index 7147fa0f435..547b12d42a7 100644 --- a/libs/structs/structaccess/typecheck.go +++ b/libs/structs/structaccess/typecheck.go @@ -147,25 +147,81 @@ func FindStructFieldByKeyType(t reflect.Type, key string) (reflect.StructField, } // First pass: direct fields + if sf, ok := findDirectFieldByKeyType(t, key); ok { + return sf, t, true + } + + // Second pass: search embedded anonymous structs breadth-first, mirroring findStructFieldByKey + // (get.go) so a path validates against the same field Get and Set resolve it to, which is + // the one encoding/json serializes: the shallower of two same-named fields. + // See findStructFieldByKey: a cycle must not be walked twice, but a type reachable twice + // within one level is a diamond and its two matches are the ambiguity json omits. + seen := map[reflect.Type]bool{t: true} + level := embeddedStructTypes(t) + for len(level) > 0 { + var next []reflect.Type + var found []struct { + field reflect.StructField + owner reflect.Type + } + for _, ft := range level { + if sf, ok := findDirectFieldByKeyType(ft, key); ok { + found = append(found, struct { + field reflect.StructField + owner reflect.Type + }{sf, ft}) + continue + } + for _, deeper := range embeddedStructTypes(ft) { + if seen[deeper] { + continue + } + next = append(next, deeper) + } + } + for _, ft := range next { + seen[ft] = true + } + if len(found) == 1 { + return found[0].field, found[0].owner, true + } + if len(found) > 1 { + // Ambiguous at this depth, which encoding/json resolves by omitting the field; see + // findStructFieldByKey in get.go. + return reflect.StructField{}, reflect.TypeOf(nil), false + } + level = next + } + + return reflect.StructField{}, reflect.TypeOf(nil), false +} + +// findDirectFieldByKeyType matches key against the struct's own fields, by json tag name. +func findDirectFieldByKeyType(t reflect.Type, key string) (reflect.StructField, bool) { for sf := range t.Fields() { if sf.PkgPath != "" { // unexported continue } name := structtag.JSONTag(sf.Tag.Get("json")).Name() if name == "-" || sf.Name == EmbeddedSliceFieldName { - name = "" + continue } - if name != "" && name == key { - // Skip fields marked as internal/readonly - btag := structtag.BundleTag(sf.Tag.Get("bundle")) - if btag.Internal() || btag.ReadOnly() { - continue - } - return sf, t, true + if name != key { + continue + } + // Skip fields marked as internal/readonly + btag := structtag.BundleTag(sf.Tag.Get("bundle")) + if btag.Internal() || btag.ReadOnly() { + continue } + return sf, true } + return reflect.StructField{}, false +} - // Second pass: search embedded anonymous structs recursively (flattening semantics) +// embeddedStructTypes returns the anonymous struct fields of t, dereferenced. +func embeddedStructTypes(t reflect.Type) []reflect.Type { + var out []reflect.Type for sf := range t.Fields() { if !sf.Anonymous { continue @@ -174,19 +230,9 @@ func FindStructFieldByKeyType(t reflect.Type, key string) (reflect.StructField, for ft.Kind() == reflect.Pointer { ft = ft.Elem() } - if ft.Kind() != reflect.Struct { - continue - } - if osf, owner, ok := FindStructFieldByKeyType(ft, key); ok { - // Skip fields marked as internal/readonly - btag := structtag.BundleTag(osf.Tag.Get("bundle")) - if btag.Internal() || btag.ReadOnly() { - // Treat as not found and continue - continue - } - return osf, owner, true + if ft.Kind() == reflect.Struct { + out = append(out, ft) } } - - return reflect.StructField{}, reflect.TypeOf(nil), false + return out } diff --git a/libs/testserver/apps.go b/libs/testserver/apps.go index c70115d060f..45dcb7d18f6 100644 --- a/libs/testserver/apps.go +++ b/libs/testserver/apps.go @@ -256,6 +256,15 @@ func (s *FakeWorkspace) AppsDelete(name string) Response { return Response{StatusCode: 404} } + if s.SettleAsyncImmediately { + // The real API leaves the app in DELETING for up to ~20 minutes and keeps the name + // reserved, so re-creating it blocks. Delete outright instead, which is what a suite + // doing thousands of updates needs; the DELETING behaviour is covered by the + // acceptance suite, which leaves the simulation on. + delete(s.Apps, name) + return Response{Body: map[string]string{}} + } + if app.ComputeStatus != nil && app.ComputeStatus.State == apps.ComputeStateDeleting { return Response{ StatusCode: http.StatusBadRequest, diff --git a/libs/testserver/cluster_policies.go b/libs/testserver/cluster_policies.go index 13c48f0770e..325fbd84b0a 100644 --- a/libs/testserver/cluster_policies.go +++ b/libs/testserver/cluster_policies.go @@ -102,11 +102,63 @@ func (s *FakeWorkspace) ClusterPoliciesEdit(req Request) any { if policy.Definition == "" && policy.PolicyFamilyId != "" { policy.Definition = policyFamilyDefinition(policy.PolicyFamilyId) } + + // A policy is named, and it takes its rules from either its own definition or a policy + // family -- never both. + if policy.Name == "" { + return Response{ + StatusCode: 400, + Body: map[string]string{ + "error_code": "INVALID_PARAMETER_VALUE", + "message": "'name' must be supplied.", + }, + } + } + if request.Definition != "" && request.PolicyFamilyId != "" { + return Response{ + StatusCode: 400, + Body: map[string]string{ + "error_code": "INVALID_PARAMETER_VALUE", + "message": "policy_family_id and definition cannot both be set.", + }, + } + } + + // A policy has to say what it constrains, so the API refuses an edit that leaves it with + // no definition and no family to take one from. + if policy.Definition == "" { + return Response{ + StatusCode: 400, + Body: map[string]string{ + "error_code": "INVALID_PARAMETER_VALUE", + "message": "'definition' must be supplied.", + }, + } + } + + // A library entry names an artifact, so an entry with none is rejected rather than stored + // as an empty object. "Unknown library" is the API's own wording for it. + if slices.ContainsFunc(policy.Libraries, isEmptyLibrary) { + return Response{ + StatusCode: 400, + Body: map[string]string{ + "error_code": "INVALID_PARAMETER_VALUE", + "message": "Unknown library", + }, + } + } + s.ClusterPolicies[request.PolicyId] = policy return Response{} } +// isEmptyLibrary reports whether a library entry names no artifact at all. +func isEmptyLibrary(library compute.Library) bool { + return library.Jar == "" && library.Egg == "" && library.Whl == "" && + library.Requirements == "" && library.Pypi == nil && library.Maven == nil && library.Cran == nil +} + func (s *FakeWorkspace) ClusterPoliciesDelete(req Request) any { var request compute.DeletePolicy if err := json.Unmarshal(req.Body, &request); err != nil { diff --git a/libs/testserver/clusters.go b/libs/testserver/clusters.go index 80991b0091a..ea2042c1beb 100644 --- a/libs/testserver/clusters.go +++ b/libs/testserver/clusters.go @@ -20,6 +20,10 @@ func (s *FakeWorkspace) ClustersCreate(req Request) any { defer s.LockUnlock()() + if response, ok := rejectAutotermination(request.AutoterminationMinutes); !ok { + return response + } + clusterId := nextUUID() request.ClusterId = clusterId // Clusters start in PENDING state when created; ClustersGet transitions them to RUNNING. @@ -65,6 +69,26 @@ func specSnapshot(body []byte) *compute.ClusterSpec { } } +// minAutoterminationMinutes is the shortest autotermination the API accepts. Zero is also accepted +// and means never. +const minAutoterminationMinutes = 10 + +// rejectAutotermination applies the API's own range check (aws, 2026-09), which the fake server did +// not: a value of 1 was stored happily here and refused there, so the catalog's rows for the field +// were measuring nothing. +func rejectAutotermination(minutes int) (Response, bool) { + if minutes == 0 || minutes >= minAutoterminationMinutes { + return Response{}, true + } + return Response{ + StatusCode: 400, + Body: map[string]string{ + "error_code": "INVALID_PARAMETER_VALUE", + "message": "The cluster autotermination time cannot be less than 10 minutes.", + }, + }, false +} + func (s *FakeWorkspace) ClustersResize(req Request) any { var request compute.ResizeCluster if err := json.Unmarshal(req.Body, &request); err != nil { @@ -110,6 +134,10 @@ func (s *FakeWorkspace) ClustersEdit(req Request) any { return Response{StatusCode: 404} } + if response, ok := rejectAutotermination(request.AutoterminationMinutes); !ok { + return response + } + // Preserve runtime-only fields that the Edit API request doesn't include. request.State = existing.State request.ClusterId = existing.ClusterId diff --git a/libs/testserver/dashboards.go b/libs/testserver/dashboards.go index 84b8527212a..1b8c4e2bb5b 100644 --- a/libs/testserver/dashboards.go +++ b/libs/testserver/dashboards.go @@ -103,6 +103,18 @@ func (s *FakeWorkspace) DashboardCreate(req Request) Response { } } + // A dashboard is addressed by its display name, which becomes its file name, so the API + // refuses to create one without it. + if dashboard.DisplayName == "" { + return Response{ + StatusCode: 400, + Body: map[string]string{ + "error_code": "INVALID_PARAMETER_VALUE", + "message": "[Request Validation] display name cannot be empty", + }, + } + } + // Default to user's home directory if parent_path is not provided (matches cloud behavior) if dashboard.ParentPath == "" { dashboard.ParentPath = "/Users/" + s.CurrentUser().UserName diff --git a/libs/testserver/fake_workspace.go b/libs/testserver/fake_workspace.go index 2f077b1b631..a9d75257d30 100644 --- a/libs/testserver/fake_workspace.go +++ b/libs/testserver/fake_workspace.go @@ -175,6 +175,11 @@ type FakeWorkspace struct { url string isServicePrincipal bool + // SettleAsyncImmediately turns off the "not ready yet on the first read" simulation + // for asynchronous resources. Off by default, so the CLI's waiters stay exercised. + // See SetSettleAsyncImmediately for when turning it on is worth it. + SettleAsyncImmediately bool + directories map[string]workspace.ObjectInfo files map[string]FileEntry repoIdByPath map[string]int64 @@ -402,6 +407,20 @@ func MapDelete[K comparable, V any](w *FakeWorkspace, collection map[K]V, key K) return Response{} } +// SetSettleAsyncImmediately makes asynchronous resources report themselves ready on the +// first read after a write, instead of reporting in-progress once so the CLI's waiter is +// exercised. +// +// Only turn it on for a suite that performs thousands of updates. The cost is not the +// extra request, it is the sleep before it: the SDK's poller uses a hardcoded backoff of +// attempt*1s plus 50-750ms of jitter (retries.backoff in databricks-sdk-go/retries/ +// retries.go), and retries.Poll does not pass a backoff option -- there is no exported way +// to shorten it. So every update costs at least a second of wall time. +func (s *FakeWorkspace) SetSettleAsyncImmediately(v bool) { + defer s.LockUnlock()() + s.SettleAsyncImmediately = v +} + func NewFakeWorkspace(url, token string) *FakeWorkspace { return &FakeWorkspace{ url: url, diff --git a/libs/testserver/genie_spaces.go b/libs/testserver/genie_spaces.go index b60df650dfd..717a6344c8a 100644 --- a/libs/testserver/genie_spaces.go +++ b/libs/testserver/genie_spaces.go @@ -10,6 +10,9 @@ import ( "github.com/databricks/databricks-sdk-go/service/dashboards" ) +// defaultGenieSpaceTitle is what the backend names a space created without a title. +const defaultGenieSpaceTitle = "New Agent" + // generateGenieSpaceId returns a random 32-character hex string. func generateGenieSpaceId() (string, error) { randomBytes := make([]byte, 16) @@ -64,6 +67,14 @@ func (s *FakeWorkspace) GenieSpaceCreate(req Request) Response { } } + // A create that names no title gets one from the backend, not an empty string (observed on + // aws, 2026-08). It matters because the config then has no title while the remote does, so + // the space drifts for as long as it lives -- a fake that stored "" would show a clean plan + // and hide that. + if createReq.Title == "" { + createReq.Title = defaultGenieSpaceTitle + } + // Default to user's home directory if parent_path is not provided (matches cloud behavior) if createReq.ParentPath == "" { createReq.ParentPath = "/Users/" + s.CurrentUser().UserName diff --git a/libs/testserver/handlers.go b/libs/testserver/handlers.go index 8c44d97eae7..f366455bee0 100644 --- a/libs/testserver/handlers.go +++ b/libs/testserver/handlers.go @@ -583,7 +583,7 @@ func AddDefaultHandlers(server *Server) { // Registered Models: server.Handle("GET", "/api/2.1/unity-catalog/models/{full_name}", func(req Request) any { - return MapGet(req.Workspace, req.Workspace.RegisteredModels, req.Vars["full_name"]) + return MapGetUC(req.Workspace, req.Workspace.RegisteredModels, req.Vars["full_name"], "Registered Model") }) server.Handle("POST", "/api/2.1/unity-catalog/models", func(req Request) any { diff --git a/libs/testserver/jobs.go b/libs/testserver/jobs.go index 3b16c61b023..ea070f065a5 100644 --- a/libs/testserver/jobs.go +++ b/libs/testserver/jobs.go @@ -146,6 +146,19 @@ func (s *FakeWorkspace) JobsReset(req Request) Response { request.NewSettings.RunAs = prevjob.Settings.RunAs } + // parent_path is the same shape and verified the same way: the folder a job lives in is + // fixed when the job is created, and a reset naming a different one is accepted and + // ignored. The engine plans an update for it anyway, which is why a config that changes it + // never converges -- recorded in bundle/direct/autotest/output/jobs.txt. + request.NewSettings.ParentPath = prevjob.Settings.ParentPath + if request.NewSettings.ParentPath == "" { + // And a job with no parent folder omits the field on read rather than reporting it + // empty, so drop any ForceSendFields entry the request carried for it. + request.NewSettings.ForceSendFields = slices.DeleteFunc(request.NewSettings.ForceSendFields, func(name string) bool { + return name == "ParentPath" + }) + } + s.Jobs[jobId] = jobs.Job{ JobId: jobId, CreatorUserName: prevjob.CreatorUserName, @@ -953,12 +966,20 @@ func (s *FakeWorkspace) JobsGetRun(req Request) Response { return Response{StatusCode: 404} } - // Simulate cloud behavior: first poll returns RUNNING, next the terminal state. + // Simulate cloud behavior: first poll returns RUNNING, next the terminal state. The waiter then + // pays the SDK's backoff -- attempt times a second, plus jitter -- for every run, which a suite + // deploying thousands of them cannot afford. SettleAsyncImmediately reports the terminal state + // on the first poll instead; the acceptance suite leaves the simulation on, so the waiter and + // its logging stay covered there. if run.State.LifeCycleState == jobs.RunLifeCycleStateRunning { // Transition stored state to TERMINATED for the next poll. terminateRun(&run) s.JobRuns[runIdInt] = run + if s.SettleAsyncImmediately { + return Response{Body: run} + } + // Return RUNNING for this poll (before the transition). runResp := run runResp.State = &jobs.RunState{ diff --git a/libs/testserver/merge.go b/libs/testserver/merge.go new file mode 100644 index 00000000000..4c48aa33974 --- /dev/null +++ b/libs/testserver/merge.go @@ -0,0 +1,54 @@ +package testserver + +import "encoding/json" + +// mergeInto applies the fields a request body carries over an existing resource, leaving the +// rest as they were. +// +// It models what an update does to a field the body omits: nothing. A client sends its whole +// desired state, but the SDK request types are omitempty, so a field the config cleared is +// absent from the body and the backend keeps the value it had -- which is why clearing such a +// field never converges. A handler that unmarshals the body into a fresh struct instead +// applies the zero value, so the fake accepts a clear the real API ignores. +// +// Arrays are replaced whole, as the backend does: an element is not separately addressable, +// so a list is only ever sent entire. +func mergeInto[T any](existing T, body []byte) (T, error) { + var out T + stored, err := json.Marshal(existing) + if err != nil { + return out, err + } + + var base, patch any + if err := json.Unmarshal(stored, &base); err != nil { + return out, err + } + if err := json.Unmarshal(body, &patch); err != nil { + return out, err + } + + merged, err := json.Marshal(mergeValue(base, patch)) + if err != nil { + return out, err + } + err = json.Unmarshal(merged, &out) + return out, err +} + +// mergeValue applies patch over base, recursing into objects and replacing anything else. +func mergeValue(base, patch any) any { + baseMap, baseIsMap := base.(map[string]any) + patchMap, patchIsMap := patch.(map[string]any) + if !baseIsMap || !patchIsMap { + return patch + } + for key, value := range patchMap { + if existing, ok := baseMap[key]; ok { + baseMap[key] = mergeValue(existing, value) + continue + } + baseMap[key] = value + } + return baseMap +} diff --git a/libs/testserver/merge_test.go b/libs/testserver/merge_test.go new file mode 100644 index 00000000000..fa362af556f --- /dev/null +++ b/libs/testserver/merge_test.go @@ -0,0 +1,46 @@ +package testserver + +import ( + "testing" + + "github.com/databricks/databricks-sdk-go/service/sql" + "github.com/stretchr/testify/require" +) + +func TestMergeInto(t *testing.T) { + existing := sql.GetWarehouseResponse{ //exhaustruct:ignore + Name: "wh", + ClusterSize: "2X-Small", + AutoStopMins: 10, + Channel: &sql.Channel{Name: sql.ChannelNameChannelNameCurrent}, //exhaustruct:ignore + Tags: &sql.EndpointTags{CustomTags: []sql.EndpointTagPair{ //exhaustruct:ignore + {Key: "team", Value: "eng"}, + }}, + } + + // A field the body omits keeps its value; that is what makes clearing it never converge. + got, err := mergeInto(existing, []byte(`{"name":"wh2"}`)) + require.NoError(t, err) + require.Equal(t, "wh2", got.Name) + require.Equal(t, "2X-Small", got.ClusterSize) + require.Equal(t, 10, got.AutoStopMins) + + // An explicit empty value does clear it, which is what ForceSendFields produces. + got, err = mergeInto(existing, []byte(`{"cluster_size":""}`)) + require.NoError(t, err) + require.Empty(t, got.ClusterSize) + require.Equal(t, "wh", got.Name) + + // A nested object merges key by key rather than being replaced. + got, err = mergeInto(existing, []byte(`{"channel":{"dbsql_version":"2024.15"}}`)) + require.NoError(t, err) + require.Equal(t, sql.ChannelNameChannelNameCurrent, got.Channel.Name) + require.Equal(t, "2024.15", got.Channel.DbsqlVersion) + + // A list is replaced whole: an element is not separately addressable. + got, err = mergeInto(existing, []byte(`{"tags":{"custom_tags":[{"key":"owner","value":"me"}]}}`)) + require.NoError(t, err) + require.Len(t, got.Tags.CustomTags, 1) + require.Equal(t, "owner", got.Tags.CustomTags[0].Key) + require.Equal(t, "me", got.Tags.CustomTags[0].Value) +} diff --git a/libs/testserver/models.go b/libs/testserver/models.go index 3314327d8a7..c4798d6eced 100644 --- a/libs/testserver/models.go +++ b/libs/testserver/models.go @@ -68,6 +68,18 @@ func (s *FakeWorkspace) ModelRegistryUpdateModel(req Request) any { } } + // MLflow refuses to clear a description: the update carries the field unconditionally, + // so a config that drops it sends "" and the real API rejects the request. + if request.Description == "" { + return Response{ + StatusCode: 400, + Body: map[string]string{ + "error_code": "INVALID_PARAMETER_VALUE", + "message": "Description cannot be empty.", + }, + } + } + // Update the model existingModel.Description = request.Description s.ModelRegistryModels[request.Name] = existingModel diff --git a/libs/testserver/pipelines.go b/libs/testserver/pipelines.go index 0aedc44c5d3..d75ec98b811 100644 --- a/libs/testserver/pipelines.go +++ b/libs/testserver/pipelines.go @@ -117,6 +117,24 @@ func (s *FakeWorkspace) PipelineUpdate(req Request, pipelineId string) Response } } + // A pipeline that has a storage location is not a Unity Catalog pipeline, and the backend will + // not convert one into the other in place. Verified directly (aws, 2026-09): a pipeline created + // with neither storage nor catalog is given a default storage location, and a later PUT adding + // a catalog is refused -- so a backend-assigned storage counts, not just one the user wrote. + // Only the fake server having allowed it hid the fact that the engine neither recreates nor + // rejects the change: it sends the update and the API refuses it. + if item.Spec != nil && item.Spec.Storage != "" && spec.Catalog != "" && spec.Catalog != item.Spec.Catalog { + return Response{ + StatusCode: 400, + Body: map[string]string{ + "error_code": "INVALID_PARAMETER_VALUE", + "message": fmt.Sprintf("Cannot add catalog to an existing pipeline with defined storage location, "+ + "if you want to use UC create a new pipeline and set catalog.\nExisting storage location: %q\nRequested catalog: %q", + item.Spec.Storage, spec.Catalog), + }, + } + } + // parameters is on EditPipeline, not PipelineSpec; round-trip it like // PipelineCreate does. var edit pipelines.EditPipeline diff --git a/libs/testserver/postgres.go b/libs/testserver/postgres.go index b6067a37355..7554bf9cf02 100644 --- a/libs/testserver/postgres.go +++ b/libs/testserver/postgres.go @@ -1611,6 +1611,18 @@ func (s *FakeWorkspace) PostgresSyncedTableCreate(req Request, syncedTableID str } } + // pg_type is an enum, and the API rejects anything outside it as a missing required field + // rather than a bad value (aws, 2026-08): a Postgres type name like "bigint" is not one of + // PG_SPECIFIC_TYPE_HALFVEC / _VARCHAR / _VECTOR. Checked against the enum, not against the + // zero value: the SDK stores an unrecognised string verbatim rather than dropping it. + if table.Spec != nil { + for _, override := range table.Spec.TypeOverrides { + if !slices.Contains(override.PgType.Values(), override.PgType) { + return postgresErrorResponse(400, "INVALID_PARAMETER_VALUE", `Field 'synced_table.spec.type_overrides.pg_type' is required, expected non-default value (not "")!`) + } + } + } + name := "synced_tables/" + syncedTableID if _, exists := s.PostgresSyncedTables[name]; exists { diff --git a/libs/testserver/registered_models.go b/libs/testserver/registered_models.go index b6fb800a623..7e7a73497f9 100644 --- a/libs/testserver/registered_models.go +++ b/libs/testserver/registered_models.go @@ -23,9 +23,28 @@ func (s *FakeWorkspace) RegisteredModelsCreate(req Request) Response { } } + // UC requires all three parts of the name; without this the fake would store a model + // under a key like "..name" that no read can address. + for _, required := range []struct{ field, value string }{ + {"catalog_name", createRequest.CatalogName}, + {"schema_name", createRequest.SchemaName}, + {"name", createRequest.Name}, + } { + if required.value == "" { + return Response{ + StatusCode: http.StatusBadRequest, + Body: map[string]string{ + "error_code": "INVALID_PARAMETER_VALUE", + "message": "CreateRegisteredModel Missing required field: " + required.field, + }, + } + } + } + // Build full name from catalog.schema.name fullName := createRequest.CatalogName + "." + createRequest.SchemaName + "." + createRequest.Name + // Aliases are not settable here; see the note in RegisteredModelsUpdate. registeredModel := catalog.RegisteredModelInfo{ CatalogName: createRequest.CatalogName, Comment: createRequest.Comment, @@ -85,6 +104,12 @@ func (s *FakeWorkspace) RegisteredModelsUpdate(req Request, fullName string) Res fullName = existing.CatalogName + "." + existing.SchemaName + "." + updateRequest.NewName } + // An alias belongs to a model version and is created through its own API, so neither + // create nor UpdateRegisteredModel sets one -- the request field exists but the backend + // does not honour it. Keeping whatever the model already had (nothing, since create does + // not set aliases either) is what makes a config that sets aliases report the truth. + existing.Aliases = nil + existing.UpdatedAt = nowMilli() s.RegisteredModels[fullName] = existing return Response{ diff --git a/libs/testserver/secret_scopes.go b/libs/testserver/secret_scopes.go index 8dab8217384..a20b734f1a4 100644 --- a/libs/testserver/secret_scopes.go +++ b/libs/testserver/secret_scopes.go @@ -32,6 +32,18 @@ func (s *FakeWorkspace) SecretsCreateScope(req Request) Response { backendType = workspace.ScopeBackendTypeDatabricks } + // An Azure Key Vault scope is backed by a real vault, so the API requires its metadata + // and rejects the scope without it. + if backendType == workspace.ScopeBackendTypeAzureKeyvault && request.BackendAzureKeyvault == nil { + return Response{ + StatusCode: 400, + Body: map[string]string{ + "error_code": "INVALID_PARAMETER_VALUE", + "message": "Scope with Azure KeyVault must have AzureKeyVaultSecretScopeMetadata defined!", + }, + } + } + scope := workspace.SecretScope{ Name: request.Scope, BackendType: backendType, diff --git a/libs/testserver/server.go b/libs/testserver/server.go index 7f949ebf5cd..721c563fdc4 100644 --- a/libs/testserver/server.go +++ b/libs/testserver/server.go @@ -73,6 +73,21 @@ type Server struct { RequestCallback func(request *Request) ResponseCallback func(request *Request, response *EncodedResponse) + + // settleAsyncImmediately is applied to every workspace this server creates; see + // FakeWorkspace.SetSettleAsyncImmediately. + settleAsyncImmediately bool +} + +// SettleAsyncImmediately turns off the "not ready yet on the first read" simulation for +// every workspace this server serves. Call it before the first request. +func (s *Server) SettleAsyncImmediately() { + s.mu.Lock() + defer s.mu.Unlock() + s.settleAsyncImmediately = true + for _, workspace := range s.fakeWorkspaces { + workspace.SetSettleAsyncImmediately(true) + } } type Request struct { @@ -349,7 +364,9 @@ func (s *Server) getWorkspaceForToken(token string) *FakeWorkspace { defer s.mu.Unlock() if _, ok := s.fakeWorkspaces[key]; !ok { - s.fakeWorkspaces[key] = NewFakeWorkspace(s.URL, token) + workspace := NewFakeWorkspace(s.URL, token) + workspace.SettleAsyncImmediately = s.settleAsyncImmediately + s.fakeWorkspaces[key] = workspace } return s.fakeWorkspaces[key] diff --git a/libs/testserver/serving_endpoints.go b/libs/testserver/serving_endpoints.go index efa10d47463..aa6af369cc3 100644 --- a/libs/testserver/serving_endpoints.go +++ b/libs/testserver/serving_endpoints.go @@ -224,6 +224,28 @@ func (s *FakeWorkspace) ServingEndpointCreate(req Request) Response { } } + // An endpoint created without a config block has nothing for these to apply to, so the API + // rejects them outright rather than storing them. + if createReq.Config == nil { + for _, rejected := range []struct { + field string + set bool + }{ + {"ai_gateway", createReq.AiGateway != nil}, + {"rate_limits", len(createReq.RateLimits) > 0}, + } { + if rejected.set { + return Response{ + StatusCode: 400, + Body: map[string]string{ + "error_code": "INVALID_PARAMETER_VALUE", + "message": "Cannot specify " + rejected.field + " when creating endpoints without a config.", + }, + } + } + } + } + // Convert config to output format var config *serving.EndpointCoreConfigOutput if createReq.Config != nil { @@ -251,13 +273,17 @@ func (s *FakeWorkspace) ServingEndpointCreate(req Request) Response { now := nowMilli() endpoint := serving.ServingEndpointDetailed{ - AiGateway: createReq.AiGateway, - BudgetPolicyId: createReq.BudgetPolicyId, - Config: config, - CreationTimestamp: now, - Creator: s.CurrentUser().UserName, - Description: createReq.Description, - EmailNotifications: createReq.EmailNotifications, + AiGateway: createReq.AiGateway, + BudgetPolicyId: createReq.BudgetPolicyId, + Config: config, + CreationTimestamp: now, + Creator: s.CurrentUser().UserName, + Description: createReq.Description, + // Not carried over from the create: a real workspace does not echo the notifications a + // create asked for (aws, 2026-09 -- the field is absent from a GET straight afterwards), + // so the endpoint drifts from the moment it exists. An update does apply them, which is + // why they are set in the update handler below and not here. + EmailNotifications: nil, Id: nextUUID(), LastUpdatedTimestamp: now, Name: createReq.Name, @@ -299,13 +325,19 @@ func (s *FakeWorkspace) ServingEndpointGet(name string) Response { } if endpointUpdating(endpoint) { - // This response stays IN_PROGRESS; settle the stored copy for the next read. settled := endpoint settled.State = &serving.EndpointState{ ConfigUpdate: serving.EndpointStateConfigUpdateNotUpdating, Ready: endpoint.State.Ready, } s.ServingEndpoints[name] = settled + + // By default this response stays IN_PROGRESS and only the stored copy settles, so + // the caller has to poll at least once. SettleAsyncImmediately reports the settled + // state right away instead. + if s.SettleAsyncImmediately { + endpoint = settled + } } return Response{Body: endpoint} diff --git a/libs/testserver/sql_warehouses.go b/libs/testserver/sql_warehouses.go index 7f36ace72fc..24b391f0e1c 100644 --- a/libs/testserver/sql_warehouses.go +++ b/libs/testserver/sql_warehouses.go @@ -4,10 +4,26 @@ import ( "encoding/json" "fmt" "net/http" + "slices" "github.com/databricks/databricks-sdk-go/service/sql" ) +// maxWarehouseClusters is the ceiling the API puts on max_num_clusters. +const maxWarehouseClusters = 40 + +// sqlWarehouseError returns a rejection in the API's shape: a 400 whose message the suite +// records verbatim. +func sqlWarehouseError(message string) Response { + return Response{ + StatusCode: 400, + Body: map[string]string{ + "error_code": "INVALID_PARAMETER_VALUE", + "message": message, + }, + } +} + func sqlWarehouseFixUps(warehouse *sql.GetWarehouseResponse, userName string) { if warehouse.CreatorName == "" { warehouse.CreatorName = userName @@ -21,6 +37,24 @@ func sqlWarehouseFixUps(warehouse *sql.GetWarehouseResponse, userName string) { if warehouse.WarehouseType == "" { warehouse.WarehouseType = sql.GetWarehouseResponseWarehouseTypeClassic } + + // Defaults the backend applies to a create that omits them (observed on aws, 2026-09: a + // create with only a name, cluster_size and max_num_clusters reads back with both set). + // Storing the zero value instead made a config asking for the non-default value look like a + // no-op, since state and remote already agreed on it. + if !warehouse.EnablePhoton && !forceSent(warehouse.ForceSendFields, "EnablePhoton") { + warehouse.EnablePhoton = true + warehouse.ForceSendFields = append(warehouse.ForceSendFields, "EnablePhoton") + } + if warehouse.SpotInstancePolicy == "" { + warehouse.SpotInstancePolicy = sql.SpotInstancePolicyCostOptimized + } +} + +// forceSent reports whether a field was explicitly present in the request body, as opposed to +// omitted: an explicit false and an absent one are the same zero value otherwise. +func forceSent(fields []string, name string) bool { + return slices.Contains(fields, name) } func (s *FakeWorkspace) SqlWarehousesUpsert(req Request, warehouseId string) Response { @@ -35,20 +69,49 @@ func (s *FakeWorkspace) SqlWarehousesUpsert(req Request, warehouseId string) Res defer s.LockUnlock()() - if warehouseId != "" { - _, ok := s.SqlWarehouses[warehouseId] + isCreate := warehouseId == "" + if !isCreate { + existing, ok := s.SqlWarehouses[warehouseId] if !ok { return Response{ StatusCode: 404, } } + // An edit applies only the fields the request body carries. The client sends its whole + // desired state, but every field is omitempty, so a field the config cleared is absent + // from the body and the backend keeps the value it had -- which is why clearing one + // never converges. Unmarshalling into a fresh struct instead would apply the zero + // value, and the suite would report the field as freely clearable. + merged, err := mergeInto(existing, req.Body) + if err != nil { + return Response{ + Body: fmt.Sprintf("internal error: %s", err), + StatusCode: http.StatusInternalServerError, + } + } + warehouse = merged } else { warehouseId = nextUUID() } - warehouse.Id = warehouseId - if warehouse.Name == "" { - warehouse.Name = warehouseId + + // The create validations the real API applies, in the words it uses (observed on aws, + // 2026-08). An edit is exempt: it merges onto a stored warehouse that already passed them, + // and the body carries only what changed. + if isCreate { + if warehouse.Name == "" { + return sqlWarehouseError("Invalid value for SQL Endpoint name, it cannot be empty.") + } + if warehouse.ClusterSize == "" { + return sqlWarehouseError("Required field 'cluster_size' is missing.") + } + // The CLI defaults this to 1 (see resourcemutator's defaults table), so a bundle only + // reaches 0 by clearing it deliberately. + if warehouse.MaxNumClusters < 1 || warehouse.MaxNumClusters > maxWarehouseClusters { + return sqlWarehouseError(fmt.Sprintf("%d is not a valid value for max_num_clusters. The value must be greater than or equal to 1, and less than or equal to %d.", warehouse.MaxNumClusters, maxWarehouseClusters)) + } } + + warehouse.Id = warehouseId warehouse.State = sql.StateRunning sqlWarehouseFixUps(&warehouse, s.CurrentUser().UserName) s.SqlWarehouses[warehouseId] = warehouse diff --git a/libs/testserver/vector_search_endpoints.go b/libs/testserver/vector_search_endpoints.go index c051d599ccc..ae207c012ca 100644 --- a/libs/testserver/vector_search_endpoints.go +++ b/libs/testserver/vector_search_endpoints.go @@ -106,7 +106,12 @@ func (s *FakeWorkspace) VectorSearchEndpointUpdate(req Request, endpointName str if endpoint.ScalingInfo == nil { endpoint.ScalingInfo = &vectorsearch.EndpointScalingInfo{} } - endpoint.ScalingInfo.RequestedTargetQps = patchReq.TargetQps + // target_qps is omitempty, so a config that clears it drops it from the body and the backend + // keeps the value it had (aws, 2026-09: the clear is accepted and has no effect). Assigning the + // zero value here instead made the field look freely clearable. + if patchReq.TargetQps != 0 { + endpoint.ScalingInfo.RequestedTargetQps = patchReq.TargetQps + } endpoint.LastUpdatedTimestamp = nowMilli() endpoint.LastUpdatedUser = s.CurrentUser().UserName