Catalog how the direct engine handles every settable field

.agents/rules/testing.md

@@ -185,6 +185,7 @@ Available on `PATH` during test execution (from `acceptance/bin/`):
 - `diff.py DIR1 DIR2` or `diff.py FILE1 FILE2`: recursive diff with test replacements applied.
 - `print_state.py [-t TARGET] [--backup]`: print deployment state (terraform or direct).
 - `edit_resource.py TYPE ID < script.py`: fetch resource by ID, execute Python on it (resource in `r`), then update it. TYPE is `jobs` or `pipelines`.
+- `verify_no_drift.py PLAN.json`: assert every action in a JSON plan is `skip`.
 - `gron.py`: flatten JSON into greppable discrete assignments (simpler than `jq` for searching JSON).
 - `jq` is also available for JSON processing.
 

Taskfile.yml

@@ -8,6 +8,9 @@ vars:
   # .github/workflows/check.yml and python/Taskfile.yml.
   RUFF: uvx ruff@0.15.17
   TEST_PACKAGES: ./acceptance/internal ./libs/... ./internal/... ./cmd/... ./bundle/... ./experimental/ssh/... .
+  # Field-support catalog. Covered by TEST_PACKAGES for local/CI runs via ./bundle/...,
+  # but named separately because the cloud tasks below list their packages explicitly.
+  FIELDS_PACKAGE: ./bundle/direct/autotest
   ACCEPTANCE_TEST_FILTER: ""
   # Single brace-expansion glob covering every //go:embed target in the repo,
   # computed by grepping `//go:embed` directives. Evaluated lazily by Task so
@@ -590,6 +593,31 @@ tasks:
     cmds:
       - "go test ./acceptance -run '^TestAccept$' -update -timeout=${LOCAL_TIMEOUT:-60m}"
 
+  test-update-fields:
+    desc: Update the resource field-support catalog (local)
+    cmds:
+      - "go test {{.FIELDS_PACKAGE}} -run '^TestFields$' -update -timeout=${LOCAL_TIMEOUT:-60m}"
+
+  # The catalog runs against the fake server as part of `./task test`, and needs no entry of its
+  # own for that. These two are the real-workspace runs: they need CLOUD_ENV and credentials, and
+  # hold the same committed reports as the local run, so a divergence is the fake server or the
+  # engine differing from the backend.
+  autotest-cloud:
+    desc: Run the resource field-support catalog against a real workspace (every field; nightly)
+    cmds:
+      # -sample 0 says every field: without it a cloud run samples unless the commit title carries
+      # AUTOTEST_ALL, which is what keeps a PR's integration run affordable.
+      - "go test {{.FIELDS_PACKAGE}} -run '^TestFields$' -count=1 -sample 0 -timeout=${CLOUD_TIMEOUT:-300m}"
+
+  autotest-cloud-pr:
+    desc: Run a two-field sample of the field-support catalog against a real workspace (PRs)
+    cmds:
+      # Which fields are sampled comes from HEAD, so each commit covers different ground. This
+      # checks that every resource type still deploys and that the sampled fields' findings read
+      # as recorded; a change between two passing verdicts only shows in the full run above.
+      - "go test {{.FIELDS_PACKAGE}} -run '^TestFields$' -count=1 -sample 2 -timeout=${CLOUD_TIMEOUT:-90m}"
+
+
   test-update-templates:
     desc: Update acceptance test template output
     sources: *ACC_SOURCES_UPDATE
@@ -601,6 +629,7 @@ tasks:
     desc: Update all acceptance test outputs
     cmds:
       - task: test-update
+      - task: test-update-fields
 
   # Follows upstream HEAD, so its result changes over time: keep it out of
   # `generate-check`, which requires byte-for-byte reproducible output.
@@ -755,7 +784,7 @@ tasks:
           --format github-actions \
           --rerun-fails \
           --jsonfile output.json \
-          --packages "./acceptance ./integration/..." \
+          --packages "./acceptance ./integration/... {{.FIELDS_PACKAGE}}" \
           -- -parallel 4 -timeout=12h
 
   integration-short:
@@ -769,7 +798,7 @@ tasks:
           --format github-actions \
           --rerun-fails \
           --jsonfile output.json \
-          --packages "./acceptance ./integration/..." \
+          --packages "./acceptance ./integration/... {{.FIELDS_PACKAGE}}" \
           -- -parallel 4 -timeout=12h -short
 
   integration-short-skiplocal:
@@ -783,7 +812,7 @@ tasks:
           --format github-actions \
           --rerun-fails \
           --jsonfile output.json \
-          --packages "./acceptance ./integration/..." \
+          --packages "./acceptance ./integration/... {{.FIELDS_PACKAGE}}" \
           -- -parallel 4 -timeout=2h -short
 
   dbr-integration:

acceptance/bundle/deployment/bind/pipelines/update/out.bind-fail.direct.txt

@@ -3,7 +3,6 @@
 Plan: update resources.pipelines.foo
 
 Changes detected:
-  ~ catalog: null -> "main"
   ~ channel: null -> "CURRENT"
   ~ deployment: null -> {"kind":"BUNDLE","metadata_file_path":"/Workspace/Users/[USERNAME]/.bundle/test-pipeline-recreate/default/state/metadata.json"}
   ~ edition: null -> "ADVANCED"

acceptance/bundle/deployment/bind/pipelines/update/out.bind-fail.terraform.txt

@@ -10,7 +10,6 @@ Terraform will perform the following actions:
 
   # databricks_pipeline.foo will be updated in-place
   ~ resource "databricks_pipeline" "foo" {
-      + catalog                = "main"
       + channel                = "CURRENT"
       + edition                = "ADVANCED"
         id                     = "[NEW_PIPELINE_ID]"

acceptance/bundle/deployment/bind/pipelines/update/out.deploy.requests.terraform.json

@@ -31,7 +31,6 @@
         }
       }
     ],
-    "name": "test-pipeline",
-    "storage": "dbfs:/pipelines/[NEW_PIPELINE_ID]"
+    "name": "test-pipeline"
   }
 }

acceptance/bundle/deployment/bind/pipelines/update/pipeline.json

@@ -1,5 +1,6 @@
 {
     "name": "lakeflow-pipeline",
+    "catalog": "main",
     "libraries": [
         {
             "glob": {

acceptance/bundle/resources/model_serving_endpoints/update/email-notifications/out.get_email_notifications.terraform.json

@@ -1,5 +1 @@
-{
-  "on_update_success": [
-    "user1@example.com"
-  ]
-}
+null

acceptance/bundle/resources/model_serving_endpoints/update/email-notifications/out.plan.direct.json

@@ -60,11 +60,6 @@
           "creation_timestamp": [UNIX_TIME_MILLIS][0],
           "creator": "[USERNAME]",
           "description": "",
-          "email_notifications": {
-            "on_update_success": [
-              "user1@example.com"
-            ]
-          },
           "id": "[UUID]",
           "last_updated_timestamp": [UNIX_TIME_MILLIS][0],
           "name": "[ENDPOINT_ID]",
@@ -100,11 +95,6 @@
             ]
           }
         },
-        "email_notifications": {
-          "on_update_success": [
-            "user1@example.com"
-          ]
-        },
         "name": "[ENDPOINT_ID]"
       },
       "changes": {
@@ -126,11 +116,23 @@
           "reason": "empty",
           "remote": ""
         },
+        "email_notifications": {
+          "action": "update",
+          "old": {
+            "on_update_success": [
+              "user1@example.com"
+            ]
+          },
+          "new": {
+            "on_update_success": [
+              "user2@example.com"
+            ]
+          }
+        },
         "email_notifications.on_update_success[0]": {
           "action": "update",
           "old": "user1@example.com",
-          "new": "user2@example.com",
-          "remote": "user1@example.com"
+          "new": "user2@example.com"
         },
         "route_optimized": {
           "action": "skip",

acceptance/bundle/resources/model_serving_endpoints/update/email-notifications/output.txt

@@ -6,11 +6,7 @@ Files: 5 uploaded, 0 deleted
 Resources: 1 created, 0 changed, 0 deleted, 0 unchanged
 
 >>> [CLI] serving-endpoints get [ENDPOINT_ID]
-{
-  "on_update_success": [
-    "user1@example.com"
-  ]
-}
+null
 
 >>> update_file.py databricks.yml user1@example.com user2@example.com
 

acceptance/bundle/resources/models/readplan-permissions/databricks.yml

@@ -5,7 +5,7 @@ resources:
   models:
     mymodel:
       name: test-model
-      description: initial # TO_REMOVE
+      description: initial
       permissions:
         - level: CAN_READ
           user_name: viewer@example.com

acceptance/bundle/resources/models/readplan-permissions/output.txt

@@ -29,7 +29,7 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged
   }
 }
 
-=== Remove manager permission and description, deploy from planUploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files...
+=== Remove manager permission, change description, deploy from planUploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files...
 Updated models.mymodel
 Updated models.mymodel.permissions
 Files: 4 uploaded, 0 deleted

acceptance/bundle/resources/models/readplan-permissions/script

@@ -15,8 +15,11 @@ trace $CLI bundle deploy
 register_model_id
 trace print_requests.py //permissions/registered-models
 
-title "Remove manager permission and description, deploy from plan"
+title "Remove manager permission, change description, deploy from plan"
 grep -v TO_REMOVE databricks.yml > updated.yml && mv updated.yml databricks.yml
+# Changed rather than removed: MLflow rejects an empty description, so clearing the field
+# fails the update. See libs/testserver/models.go.
+update_file.py databricks.yml "description: initial" "description: changed"
 $CLI bundle plan -o json > tmp.plan.json
 $CLI bundle deploy $(readplanarg tmp.plan.json)
 register_model_id

acceptance/cmd/workspace/create-scope/output.txt

@@ -8,18 +8,16 @@ Error: expected JSON object, received sequence
 >>> [CLI] secrets create-scope --scope-backend-type AZURE_KEYVAULT --json {
   "scope": "xxxxxxxx-xxx",
   "scope_backend_type": "AZURE_KEYVAULT",
-  "keyvault_metadata": {
+  "backend_azure_keyvault": {
     "dns_name": "test",
     "resource_id": "123456"
   }
 }
-Warning: unknown field: keyvault_metadata
-  in (inline):4:4
-
 
 >>> [CLI] secrets delete-scope xxxxxxxx-xxx
 
->>> [CLI] secrets create-scope --scope-backend-type AZURE_KEYVAULT --json {
+>>> musterr [CLI] secrets create-scope --scope-backend-type AZURE_KEYVAULT --json {
   "scope": "xxxxxxxx-xxx",
   "scope_backend_type": "AZURE_KEYVAULT"
 }
+Error: Scope with Azure KeyVault must have AzureKeyVaultSecretScopeMetadata defined!

acceptance/cmd/workspace/create-scope/script

@@ -5,15 +5,17 @@ trace musterr $CLI secrets create-scope --scope-backend-type AZURE_KEYVAULT --js
 trace $CLI secrets create-scope --scope-backend-type AZURE_KEYVAULT --json '{
   "scope": "xxxxxxxx-xxx",
   "scope_backend_type": "AZURE_KEYVAULT",
-  "keyvault_metadata": {
+  "backend_azure_keyvault": {
     "dns_name": "test",
     "resource_id": "123456"
   }
 }'
 
 trace $CLI secrets delete-scope xxxxxxxx-xxx
 
-trace $CLI secrets create-scope --scope-backend-type AZURE_KEYVAULT --json '{
+# The API requires the metadata that names the vault: "Scope with Azure KeyVault must have
+# AzureKeyVaultSecretScopeMetadata defined!"
+trace musterr $CLI secrets create-scope --scope-backend-type AZURE_KEYVAULT --json '{
   "scope": "xxxxxxxx-xxx",
   "scope_backend_type": "AZURE_KEYVAULT"
 }'

bundle/direct/autotest/.gitignore

@@ -0,0 +1,3 @@
+# What a run produces for reading rather than for review: the full report, its cloud counterpart,
+# and the previous copy of each. output/ holds the committed goldens and nothing else.
+logs/