From 4aa1187aa22e8d39b0d66c4a7e8f78ed7bb0dee0 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Thu, 5 Nov 2026 16:13:43 +0100 Subject: [PATCH 01/17] UC secrets follow a renamed catalog or schema UC moves a secret with its renamed parent, so the planner now re-keys it (update_id, parent_renamed) instead of recreating it, which failed with ALREADY_EXISTS. A secret cannot be renamed itself, so its DoUpdateWithID only re-keys; the adapter check now accepts DoUpdateWithID for a resource whose provided_id_fields a parent rename can change. Co-authored-by: Isaac --- .nextchanges/bundles/secret-parent-rename.md | 1 + .../secrets/parent_rename/databricks.yml.tmpl | 22 + .../secrets/parent_rename/out.test.toml | 2 + .../secrets/parent_rename/output.txt | 440 ++++++++++++++++++ .../resources/secrets/parent_rename/script | 39 ++ .../resources/secrets/parent_rename/test.toml | 7 + bundle/direct/dresources/adapter.go | 6 +- bundle/direct/dresources/configs/secrets.yml | 9 + bundle/direct/dresources/secret.go | 11 + libs/testserver/catalogs.go | 9 + libs/testserver/schemas.go | 9 + 11 files changed, 553 insertions(+), 2 deletions(-) create mode 100644 .nextchanges/bundles/secret-parent-rename.md create mode 100644 acceptance/bundle/resources/secrets/parent_rename/databricks.yml.tmpl create mode 100644 acceptance/bundle/resources/secrets/parent_rename/out.test.toml create mode 100644 acceptance/bundle/resources/secrets/parent_rename/output.txt create mode 100644 acceptance/bundle/resources/secrets/parent_rename/script create mode 100644 acceptance/bundle/resources/secrets/parent_rename/test.toml diff --git a/.nextchanges/bundles/secret-parent-rename.md b/.nextchanges/bundles/secret-parent-rename.md new file mode 100644 index 00000000000..45f0144d674 --- /dev/null +++ b/.nextchanges/bundles/secret-parent-rename.md @@ -0,0 +1 @@ +* UC secrets inside a renamed catalog or schema now follow it to the new name; recreating them failed because they already exist there. diff --git a/acceptance/bundle/resources/secrets/parent_rename/databricks.yml.tmpl b/acceptance/bundle/resources/secrets/parent_rename/databricks.yml.tmpl new file mode 100644 index 00000000000..98e310a0dc4 --- /dev/null +++ b/acceptance/bundle/resources/secrets/parent_rename/databricks.yml.tmpl @@ -0,0 +1,22 @@ +bundle: + name: test-bundle-$UNIQUE_NAME + +variables: + secret_value: + description: The secret's value + +resources: + catalogs: + catalog1: + name: test_catalog_$UNIQUE_NAME + schemas: + schema1: + catalog_name: ${resources.catalogs.catalog1.name} + name: myschema + secrets: + secret1: + catalog_name: ${resources.catalogs.catalog1.name} + schema_name: ${resources.schemas.schema1.name} + name: mysecret + value: ${var.secret_value} + comment: c1 diff --git a/acceptance/bundle/resources/secrets/parent_rename/out.test.toml b/acceptance/bundle/resources/secrets/parent_rename/out.test.toml new file mode 100644 index 00000000000..0f167effef1 --- /dev/null +++ b/acceptance/bundle/resources/secrets/parent_rename/out.test.toml @@ -0,0 +1,2 @@ +Cloud = true +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/resources/secrets/parent_rename/output.txt b/acceptance/bundle/resources/secrets/parent_rename/output.txt new file mode 100644 index 00000000000..5ba571ac7c1 --- /dev/null +++ b/acceptance/bundle/resources/secrets/parent_rename/output.txt @@ -0,0 +1,440 @@ + +=== Deploy a catalog with a schema and a secret in it +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... +Created catalogs.catalog1 +Created schemas.schema1 +Created secrets.secret1 +Files: 2 uploaded, 0 deleted +Resources: 3 created, 0 changed, 0 deleted, 0 unchanged + +>>> print_requests.py //unity-catalog +{ + "method": "POST", + "path": "/api/2.1/unity-catalog/catalogs", + "body": { + "name": "test_catalog_[UNIQUE_NAME]" + } +} +{ + "method": "POST", + "path": "/api/2.1/unity-catalog/schemas", + "body": { + "catalog_name": "test_catalog_[UNIQUE_NAME]", + "name": "myschema" + } +} +{ + "method": "POST", + "path": "/api/2.1/unity-catalog/secrets", + "body": { + "catalog_name": "test_catalog_[UNIQUE_NAME]", + "comment": "c1", + "name": "mysecret", + "schema_name": "myschema", + "value": "secret-value" + } +} + +=== Rename the schema: UC moves the secret with it, so the secret is re-keyed without a request +>>> update_file.py databricks.yml name: myschema name: myschema2 + +>>> [CLI] bundle plan --output json +{ + "resources.catalogs.catalog1": { + "action": "skip", + "remote_state": { + "catalog_type": "MANAGED_CATALOG", + "created_by": "[USERNAME]", + "full_name": "test_catalog_[UNIQUE_NAME]", + "name": "test_catalog_[UNIQUE_NAME]", + "owner": "[USERNAME]" + } + }, + "resources.schemas.schema1": { + "depends_on": [ + { + "node": "resources.catalogs.catalog1", + "label": "${resources.catalogs.catalog1.name}" + } + ], + "action": "update_id", + "new_state": { + "value": { + "catalog_name": "test_catalog_[UNIQUE_NAME]", + "name": "myschema2" + } + }, + "remote_state": { + "catalog_name": "test_catalog_[UNIQUE_NAME]", + "catalog_type": "MANAGED_CATALOG", + "created_by": "[USERNAME]", + "full_name": "test_catalog_[UNIQUE_NAME].myschema", + "name": "myschema", + "owner": "[USERNAME]", + "schema_id": "[UUID]" + }, + "changes": { + "name": { + "action": "update_id", + "reason": "id_changes", + "old": "myschema", + "new": "myschema2", + "remote": "myschema" + } + } + }, + "resources.secrets.secret1": { + "depends_on": [ + { + "node": "resources.catalogs.catalog1", + "label": "${resources.catalogs.catalog1.name}" + }, + { + "node": "resources.schemas.schema1", + "label": "${resources.schemas.schema1.name}" + } + ], + "action": "update_id", + "new_state": { + "value": { + "catalog_name": "test_catalog_[UNIQUE_NAME]", + "comment": "c1", + "name": "mysecret", + "schema_name": "myschema2", + "value": "secret-value" + } + }, + "remote_state": { + "catalog_name": "test_catalog_[UNIQUE_NAME]", + "comment": "c1", + "create_time": "[SECRET_CREATE_TIME]", + "created_by": "[USERNAME]", + "effective_owner": "[USERNAME]", + "effective_value": "[redacted]", + "full_name": "test_catalog_[UNIQUE_NAME].myschema.mysecret", + "name": "mysecret", + "owner": "[USERNAME]", + "schema_name": "myschema", + "update_time": "[SECRET_CREATE_TIME]", + "value": "[redacted]" + }, + "changes": { + "create_time": { + "action": "skip", + "reason": "spec:output_only", + "remote": "[SECRET_CREATE_TIME]" + }, + "created_by": { + "action": "skip", + "reason": "spec:output_only", + "remote": "[USERNAME]" + }, + "effective_owner": { + "action": "skip", + "reason": "spec:output_only", + "remote": "[USERNAME]" + }, + "effective_value": { + "action": "skip", + "reason": "spec:output_only", + "remote": "[redacted]" + }, + "full_name": { + "action": "skip", + "reason": "spec:output_only", + "remote": "test_catalog_[UNIQUE_NAME].myschema.mysecret" + }, + "owner": { + "action": "skip", + "reason": "spec:input_only", + "remote": "[USERNAME]" + }, + "schema_name": { + "action": "update_id", + "reason": "parent_renamed", + "old": "myschema", + "new": "myschema2", + "remote": "myschema" + }, + "update_time": { + "action": "skip", + "reason": "spec:output_only", + "remote": "[SECRET_CREATE_TIME]" + }, + "value": { + "action": "skip", + "reason": "custom", + "old": "", + "new": "[redacted]", + "remote": "[redacted]" + } + } + } +} + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... +Updated schemas.schema1 +Updated secrets.secret1 +Files: 1 uploaded, 0 deleted +Resources: 0 created, 2 changed, 0 deleted, 1 unchanged + +>>> print_requests.py //unity-catalog +{ + "method": "PATCH", + "path": "/api/2.1/unity-catalog/schemas/test_catalog_[UNIQUE_NAME].myschema", + "body": { + "comment": "", + "custom_max_retention_hours": 0, + "new_name": "myschema2" + } +} + +>>> [CLI] secrets-uc get-secret test_catalog_[UNIQUE_NAME].myschema2.mysecret +{ + "full_name": "test_catalog_[UNIQUE_NAME].myschema2.mysecret", + "comment": "c1", + "create_time": "[SECRET_CREATE_TIME]" +} + +=== Rename the catalog and change the secret's comment: the update reaches the secret at its new path +>>> update_file.py databricks.yml test_catalog_ test_catalog_renamed_ + +>>> update_file.py databricks.yml comment: c1 comment: c2 + +>>> [CLI] bundle plan --output json +{ + "resources.catalogs.catalog1": { + "action": "update_id", + "new_state": { + "value": { + "name": "test_catalog_renamed_[UNIQUE_NAME]" + } + }, + "remote_state": { + "catalog_type": "MANAGED_CATALOG", + "created_by": "[USERNAME]", + "full_name": "test_catalog_[UNIQUE_NAME]", + "name": "test_catalog_[UNIQUE_NAME]", + "owner": "[USERNAME]" + }, + "changes": { + "name": { + "action": "update_id", + "reason": "id_changes", + "old": "test_catalog_[UNIQUE_NAME]", + "new": "test_catalog_renamed_[UNIQUE_NAME]", + "remote": "test_catalog_[UNIQUE_NAME]" + } + } + }, + "resources.schemas.schema1": { + "depends_on": [ + { + "node": "resources.catalogs.catalog1", + "label": "${resources.catalogs.catalog1.name}" + } + ], + "action": "update_id", + "new_state": { + "value": { + "catalog_name": "test_catalog_renamed_[UNIQUE_NAME]", + "name": "myschema2" + } + }, + "remote_state": { + "catalog_name": "test_catalog_[UNIQUE_NAME]", + "catalog_type": "MANAGED_CATALOG", + "comment": "", + "created_by": "[USERNAME]", + "custom_max_retention_hours": 0, + "full_name": "test_catalog_[UNIQUE_NAME].myschema2", + "name": "myschema2", + "owner": "[USERNAME]", + "schema_id": "[UUID]" + }, + "changes": { + "catalog_name": { + "action": "update_id", + "reason": "parent_renamed", + "old": "test_catalog_[UNIQUE_NAME]", + "new": "test_catalog_renamed_[UNIQUE_NAME]", + "remote": "test_catalog_[UNIQUE_NAME]" + }, + "comment": { + "action": "skip", + "reason": "empty", + "remote": "" + }, + "custom_max_retention_hours": { + "action": "skip", + "reason": "empty", + "remote": 0 + } + } + }, + "resources.secrets.secret1": { + "depends_on": [ + { + "node": "resources.catalogs.catalog1", + "label": "${resources.catalogs.catalog1.name}" + }, + { + "node": "resources.schemas.schema1", + "label": "${resources.schemas.schema1.name}" + } + ], + "action": "update_id", + "new_state": { + "value": { + "catalog_name": "test_catalog_renamed_[UNIQUE_NAME]", + "comment": "c2", + "name": "mysecret", + "schema_name": "myschema2", + "value": "secret-value" + } + }, + "remote_state": { + "catalog_name": "test_catalog_[UNIQUE_NAME]", + "comment": "c1", + "create_time": "[SECRET_CREATE_TIME]", + "created_by": "[USERNAME]", + "effective_owner": "[USERNAME]", + "effective_value": "[redacted]", + "full_name": "test_catalog_[UNIQUE_NAME].myschema2.mysecret", + "name": "mysecret", + "owner": "[USERNAME]", + "schema_name": "myschema2", + "update_time": "[SECRET_CREATE_TIME]", + "value": "[redacted]" + }, + "changes": { + "catalog_name": { + "action": "update_id", + "reason": "parent_renamed", + "old": "test_catalog_[UNIQUE_NAME]", + "new": "test_catalog_renamed_[UNIQUE_NAME]", + "remote": "test_catalog_[UNIQUE_NAME]" + }, + "comment": { + "action": "update", + "old": "c1", + "new": "c2", + "remote": "c1" + }, + "create_time": { + "action": "skip", + "reason": "spec:output_only", + "remote": "[SECRET_CREATE_TIME]" + }, + "created_by": { + "action": "skip", + "reason": "spec:output_only", + "remote": "[USERNAME]" + }, + "effective_owner": { + "action": "skip", + "reason": "spec:output_only", + "remote": "[USERNAME]" + }, + "effective_value": { + "action": "skip", + "reason": "spec:output_only", + "remote": "[redacted]" + }, + "full_name": { + "action": "skip", + "reason": "spec:output_only", + "remote": "test_catalog_[UNIQUE_NAME].myschema2.mysecret" + }, + "owner": { + "action": "skip", + "reason": "spec:input_only", + "remote": "[USERNAME]" + }, + "update_time": { + "action": "skip", + "reason": "spec:output_only", + "remote": "[SECRET_CREATE_TIME]" + }, + "value": { + "action": "skip", + "reason": "custom", + "old": "", + "new": "[redacted]", + "remote": "[redacted]" + } + } + } +} + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... +Updated catalogs.catalog1 +Updated schemas.schema1 +Updated secrets.secret1 +Files: 1 uploaded, 0 deleted +Resources: 0 created, 3 changed, 0 deleted, 0 unchanged + +>>> print_requests.py //unity-catalog +{ + "method": "PATCH", + "path": "/api/2.1/unity-catalog/catalogs/test_catalog_[UNIQUE_NAME]", + "body": { + "comment": "", + "custom_max_retention_hours": 0, + "new_name": "test_catalog_renamed_[UNIQUE_NAME]" + } +} +{ + "method": "PATCH", + "path": "/api/2.1/unity-catalog/secrets/test_catalog_renamed_[UNIQUE_NAME].myschema2.mysecret", + "q": { + "update_mask": "*" + }, + "body": { + "catalog_name": "test_catalog_renamed_[UNIQUE_NAME]", + "comment": "c2", + "name": "mysecret", + "schema_name": "myschema2", + "value": "secret-value" + } +} + +>>> [CLI] secrets-uc get-secret test_catalog_renamed_[UNIQUE_NAME].myschema2.mysecret +{ + "full_name": "test_catalog_renamed_[UNIQUE_NAME].myschema2.mysecret", + "comment": "c2", + "create_time": "[SECRET_CREATE_TIME]" +} + +=== The state tracks the new path and the plan is a no-op +>>> [CLI] bundle summary --output json +{ + "catalogs": { + "catalog1": "test_catalog_renamed_[UNIQUE_NAME]" + }, + "schemas": { + "schema1": "test_catalog_renamed_[UNIQUE_NAME].myschema2" + }, + "secrets": { + "secret1": "test_catalog_renamed_[UNIQUE_NAME].myschema2.mysecret" + } +} + +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 3 unchanged + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.catalogs.catalog1 + delete resources.schemas.schema1 + delete resources.secrets.secret1 + +This action will result in the deletion of the following UC schemas. Any underlying data may be lost: + delete resources.schemas.schema1 + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default + +Destroy: 3 deleted diff --git a/acceptance/bundle/resources/secrets/parent_rename/script b/acceptance/bundle/resources/secrets/parent_rename/script new file mode 100644 index 00000000000..8c5fa6cc7c9 --- /dev/null +++ b/acceptance/bundle/resources/secrets/parent_rename/script @@ -0,0 +1,39 @@ +envsubst < databricks.yml.tmpl > databricks.yml +export BUNDLE_VAR_secret_value=secret-value + +cleanup() { + trace $CLI bundle destroy --auto-approve + + # Consume the destroy's requests so none are left recorded. + print_requests.py //unity-catalog &> LOG.destroy_requests +} +trap cleanup EXIT + +# Drop the remote_state fields that differ between clouds or from the test server. +plan() { + trace $CLI bundle plan --output json | jq .plan | normalize_uc_payload.py created_at updated_at updated_by metastore_id browse_only effective_predictive_optimization_flag enable_predictive_optimization isolation_mode securable_type +} + +title "Deploy a catalog with a schema and a secret in it" +trace $CLI bundle deploy +trace print_requests.py //unity-catalog +add_repl "$($CLI secrets-uc get-secret "test_catalog_${UNIQUE_NAME}.myschema.mysecret" | jq -r .create_time)" SECRET_CREATE_TIME + +title "Rename the schema: UC moves the secret with it, so the secret is re-keyed without a request" +trace update_file.py databricks.yml "name: myschema" "name: myschema2" +plan +trace $CLI bundle deploy +trace print_requests.py //unity-catalog +trace $CLI secrets-uc get-secret "test_catalog_${UNIQUE_NAME}.myschema2.mysecret" | jq '{full_name, comment, create_time}' + +title "Rename the catalog and change the secret's comment: the update reaches the secret at its new path" +trace update_file.py databricks.yml test_catalog_ test_catalog_renamed_ +trace update_file.py databricks.yml "comment: c1" "comment: c2" +plan +trace $CLI bundle deploy +trace print_requests.py //unity-catalog +trace $CLI secrets-uc get-secret "test_catalog_renamed_${UNIQUE_NAME}.myschema2.mysecret" | jq '{full_name, comment, create_time}' + +title "The state tracks the new path and the plan is a no-op" +trace $CLI bundle summary --output json | jq '.resources | map_values(map_values(.id))' +trace $CLI bundle plan diff --git a/acceptance/bundle/resources/secrets/parent_rename/test.toml b/acceptance/bundle/resources/secrets/parent_rename/test.toml new file mode 100644 index 00000000000..4272a80cf61 --- /dev/null +++ b/acceptance/bundle/resources/secrets/parent_rename/test.toml @@ -0,0 +1,7 @@ +Cloud = true + +Ignore = [ + ".databricks", + ".gitignore", + "databricks.yml", +] diff --git a/bundle/direct/dresources/adapter.go b/bundle/direct/dresources/adapter.go index 21ac7a02344..a74a652cbd1 100644 --- a/bundle/direct/dresources/adapter.go +++ b/bundle/direct/dresources/adapter.go @@ -480,8 +480,10 @@ func (a *Adapter) validate() error { if hasUpdateWithIDTrigger && a.doUpdateWithID == nil { return errors.New("resourceConfig has updatable_id_fields but DoUpdateWithID is not implemented") } - if a.doUpdateWithID != nil && !hasUpdateWithIDTrigger { - return errors.New("DoUpdateWithID is implemented but resourceConfig lacks updatable_id_fields") + // A parent's rename also re-keys a resource through its provided_id_fields (see followParentRenames). + canUpdateWithID := hasUpdateWithIDTrigger || (a.resourceConfig != nil && len(a.resourceConfig.ProvidedIDFields) > 0) + if a.doUpdateWithID != nil && !canUpdateWithID { + return errors.New("DoUpdateWithID is implemented but resourceConfig lacks updatable_id_fields and provided_id_fields") } } diff --git a/bundle/direct/dresources/configs/secrets.yml b/bundle/direct/dresources/configs/secrets.yml index 0b5918467ed..142492aca34 100644 --- a/bundle/direct/dresources/configs/secrets.yml +++ b/bundle/direct/dresources/configs/secrets.yml @@ -3,6 +3,15 @@ parent_id_fields: reason: parent - field: schema_name reason: parent +provided_id_fields: + # The ID is the full name. UC cannot rename a secret, but it moves the secret with a renamed + # catalog or schema, which re-keys it (see followParentRenames). + - field: catalog_name + reason: id_field + - field: schema_name + reason: id_field + - field: name + reason: id_field sensitive_fields: - field: value - field: effective_value diff --git a/bundle/direct/dresources/secret.go b/bundle/direct/dresources/secret.go index 7cc30eec4d5..2a031f3e1bd 100644 --- a/bundle/direct/dresources/secret.go +++ b/bundle/direct/dresources/secret.go @@ -109,6 +109,17 @@ func (*ResourceSecret) IDFromState(state *catalog.Secret) string { return state.CatalogName + "." + state.SchemaName + "." + state.Name } +// DoUpdateWithID re-keys the secret after its catalog or schema is renamed: UC moves the secret +// with its parent. A secret cannot be renamed itself, so this is the only way its ID changes. +func (r *ResourceSecret) DoUpdateWithID(ctx context.Context, _ string, state *catalog.Secret, entry *PlanEntry) (string, *catalog.Secret, error) { + newID := state.CatalogName + "." + state.SchemaName + "." + state.Name + if entry.Changes.OnlyParentRenamed() { + return newID, nil, nil + } + remote, err := r.DoUpdate(ctx, newID, state, entry) + return newID, remote, err +} + // DoDelete deletes the secret. func (r *ResourceSecret) DoDelete(ctx context.Context, id string, _ *catalog.Secret) error { return r.client.SecretsUc.DeleteSecret(ctx, catalog.DeleteSecretRequest{ diff --git a/libs/testserver/catalogs.go b/libs/testserver/catalogs.go index 53740b3bd89..72f75c598be 100644 --- a/libs/testserver/catalogs.go +++ b/libs/testserver/catalogs.go @@ -171,6 +171,15 @@ func (s *FakeWorkspace) moveCatalogChildren(oldCatalogName, newCatalogName strin secret.FullName = newCatalogName + "." + secret.SchemaName + "." + secret.Name s.UCSecrets[secret.FullName] = secret } + for key, secret := range s.UCSecrets { + if secret.CatalogName != oldCatalogName { + continue + } + delete(s.UCSecrets, key) + secret.CatalogName = newCatalogName + secret.FullName = newCatalogName + "." + secret.SchemaName + "." + secret.Name + s.UCSecrets[secret.FullName] = secret + } } // CatalogsDelete mirrors UC, where deleting a catalog with force deletes its schemas, volumes, registered models and secrets too. diff --git a/libs/testserver/schemas.go b/libs/testserver/schemas.go index c934adcdc15..a26128587f3 100644 --- a/libs/testserver/schemas.go +++ b/libs/testserver/schemas.go @@ -153,6 +153,15 @@ func (s *FakeWorkspace) moveSchemaChildren(catalogName, oldSchemaName, newSchema secret.FullName = catalogName + "." + newSchemaName + "." + secret.Name s.UCSecrets[secret.FullName] = secret } + for key, secret := range s.UCSecrets { + if secret.CatalogName != catalogName || secret.SchemaName != oldSchemaName { + continue + } + delete(s.UCSecrets, key) + secret.SchemaName = newSchemaName + secret.FullName = catalogName + "." + newSchemaName + "." + secret.Name + s.UCSecrets[secret.FullName] = secret + } } // SchemasDelete mirrors UC, where deleting a schema with force deletes the volumes, registered models and secrets in it too. From 29326cd36a32da268b515d52755a9b5d362e2b85 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Fri, 6 Nov 2026 09:50:38 +0100 Subject: [PATCH 02/17] Link PR in changelog fragment Co-authored-by: Isaac --- .nextchanges/bundles/secret-parent-rename.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.nextchanges/bundles/secret-parent-rename.md b/.nextchanges/bundles/secret-parent-rename.md index 45f0144d674..0d3c9b7d1ce 100644 --- a/.nextchanges/bundles/secret-parent-rename.md +++ b/.nextchanges/bundles/secret-parent-rename.md @@ -1 +1 @@ -* UC secrets inside a renamed catalog or schema now follow it to the new name; recreating them failed because they already exist there. +* UC secrets inside a renamed catalog or schema now follow it to the new name; recreating them failed because they already exist there. ([#7381](https://github.com/databricks/cli/pull/7381)) From 8d4f4035d82142e4149c51b45cf94b42c19fce3c Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Mon, 9 Nov 2026 12:51:46 +0100 Subject: [PATCH 03/17] acc: show the skipped schema_id change in the secret parent-rename plans Co-authored-by: Isaac --- .../resources/secrets/parent_rename/output.txt | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/acceptance/bundle/resources/secrets/parent_rename/output.txt b/acceptance/bundle/resources/secrets/parent_rename/output.txt index 5ba571ac7c1..13db17659c0 100644 --- a/acceptance/bundle/resources/secrets/parent_rename/output.txt +++ b/acceptance/bundle/resources/secrets/parent_rename/output.txt @@ -81,6 +81,12 @@ Resources: 3 created, 0 changed, 0 deleted, 0 unchanged "old": "myschema", "new": "myschema2", "remote": "myschema" + }, + "schema_id": { + "action": "skip", + "reason": "internal_id", + "old": "[UUID]", + "remote": "[UUID]" } } }, @@ -271,6 +277,12 @@ Resources: 0 created, 2 changed, 0 deleted, 1 unchanged "action": "skip", "reason": "empty", "remote": 0 + }, + "schema_id": { + "action": "skip", + "reason": "internal_id", + "old": "[UUID]", + "remote": "[UUID]" } } }, From 78c6a60e02e42e097922db9ed777c1d8def6c76f Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Mon, 9 Nov 2026 14:10:28 +0100 Subject: [PATCH 04/17] Point comments at followsParentRename Co-authored-by: Isaac --- bundle/direct/dresources/adapter.go | 2 +- bundle/direct/dresources/configs/secrets.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/bundle/direct/dresources/adapter.go b/bundle/direct/dresources/adapter.go index a74a652cbd1..ae73795ecd7 100644 --- a/bundle/direct/dresources/adapter.go +++ b/bundle/direct/dresources/adapter.go @@ -480,7 +480,7 @@ func (a *Adapter) validate() error { if hasUpdateWithIDTrigger && a.doUpdateWithID == nil { return errors.New("resourceConfig has updatable_id_fields but DoUpdateWithID is not implemented") } - // A parent's rename also re-keys a resource through its provided_id_fields (see followParentRenames). + // A parent's rename also re-keys a resource through its provided_id_fields (see followsParentRename). canUpdateWithID := hasUpdateWithIDTrigger || (a.resourceConfig != nil && len(a.resourceConfig.ProvidedIDFields) > 0) if a.doUpdateWithID != nil && !canUpdateWithID { return errors.New("DoUpdateWithID is implemented but resourceConfig lacks updatable_id_fields and provided_id_fields") diff --git a/bundle/direct/dresources/configs/secrets.yml b/bundle/direct/dresources/configs/secrets.yml index 142492aca34..f4458d2f5f9 100644 --- a/bundle/direct/dresources/configs/secrets.yml +++ b/bundle/direct/dresources/configs/secrets.yml @@ -5,7 +5,7 @@ parent_id_fields: reason: parent provided_id_fields: # The ID is the full name. UC cannot rename a secret, but it moves the secret with a renamed - # catalog or schema, which re-keys it (see followParentRenames). + # catalog or schema, which re-keys it (see followsParentRename). - field: catalog_name reason: id_field - field: schema_name From 52e3bf5a349e729a1f8046a3ee34d2673666090a Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Mon, 9 Nov 2026 15:51:45 +0100 Subject: [PATCH 05/17] direct: let the framework re-key a secret that only follows its parent; drop duplicated test-server moves Co-authored-by: Isaac --- .../bundle/resources/secrets/parent_rename/output.txt | 8 ++++---- bundle/direct/dresources/secret.go | 10 ++++------ libs/testserver/catalogs.go | 9 --------- libs/testserver/schemas.go | 9 --------- 4 files changed, 8 insertions(+), 28 deletions(-) diff --git a/acceptance/bundle/resources/secrets/parent_rename/output.txt b/acceptance/bundle/resources/secrets/parent_rename/output.txt index 13db17659c0..929a3b532dd 100644 --- a/acceptance/bundle/resources/secrets/parent_rename/output.txt +++ b/acceptance/bundle/resources/secrets/parent_rename/output.txt @@ -102,6 +102,7 @@ Resources: 3 created, 0 changed, 0 deleted, 0 unchanged } ], "action": "update_id", + "state_only": true, "new_state": { "value": { "catalog_name": "test_catalog_[UNIQUE_NAME]", @@ -182,9 +183,8 @@ Resources: 3 created, 0 changed, 0 deleted, 0 unchanged >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... Updated schemas.schema1 -Updated secrets.secret1 Files: 1 uploaded, 0 deleted -Resources: 0 created, 2 changed, 0 deleted, 1 unchanged +Resources: 0 created, 1 changed, 0 deleted, 2 unchanged >>> print_requests.py //unity-catalog { @@ -243,6 +243,7 @@ Resources: 0 created, 2 changed, 0 deleted, 1 unchanged } ], "action": "update_id", + "state_only": true, "new_state": { "value": { "catalog_name": "test_catalog_renamed_[UNIQUE_NAME]", @@ -384,10 +385,9 @@ Resources: 0 created, 2 changed, 0 deleted, 1 unchanged >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... Updated catalogs.catalog1 -Updated schemas.schema1 Updated secrets.secret1 Files: 1 uploaded, 0 deleted -Resources: 0 created, 3 changed, 0 deleted, 0 unchanged +Resources: 0 created, 2 changed, 0 deleted, 1 unchanged >>> print_requests.py //unity-catalog { diff --git a/bundle/direct/dresources/secret.go b/bundle/direct/dresources/secret.go index 2a031f3e1bd..38edb0265eb 100644 --- a/bundle/direct/dresources/secret.go +++ b/bundle/direct/dresources/secret.go @@ -109,13 +109,11 @@ func (*ResourceSecret) IDFromState(state *catalog.Secret) string { return state.CatalogName + "." + state.SchemaName + "." + state.Name } -// DoUpdateWithID re-keys the secret after its catalog or schema is renamed: UC moves the secret -// with its parent. A secret cannot be renamed itself, so this is the only way its ID changes. +// DoUpdateWithID updates a secret whose catalog or schema was renamed: UC moved the secret with its +// parent, so it is addressed at its new ID. A secret cannot be renamed itself, so this is the only +// way its ID changes. func (r *ResourceSecret) DoUpdateWithID(ctx context.Context, _ string, state *catalog.Secret, entry *PlanEntry) (string, *catalog.Secret, error) { - newID := state.CatalogName + "." + state.SchemaName + "." + state.Name - if entry.Changes.OnlyParentRenamed() { - return newID, nil, nil - } + newID := r.IDFromState(state) remote, err := r.DoUpdate(ctx, newID, state, entry) return newID, remote, err } diff --git a/libs/testserver/catalogs.go b/libs/testserver/catalogs.go index 72f75c598be..53740b3bd89 100644 --- a/libs/testserver/catalogs.go +++ b/libs/testserver/catalogs.go @@ -171,15 +171,6 @@ func (s *FakeWorkspace) moveCatalogChildren(oldCatalogName, newCatalogName strin secret.FullName = newCatalogName + "." + secret.SchemaName + "." + secret.Name s.UCSecrets[secret.FullName] = secret } - for key, secret := range s.UCSecrets { - if secret.CatalogName != oldCatalogName { - continue - } - delete(s.UCSecrets, key) - secret.CatalogName = newCatalogName - secret.FullName = newCatalogName + "." + secret.SchemaName + "." + secret.Name - s.UCSecrets[secret.FullName] = secret - } } // CatalogsDelete mirrors UC, where deleting a catalog with force deletes its schemas, volumes, registered models and secrets too. diff --git a/libs/testserver/schemas.go b/libs/testserver/schemas.go index a26128587f3..c934adcdc15 100644 --- a/libs/testserver/schemas.go +++ b/libs/testserver/schemas.go @@ -153,15 +153,6 @@ func (s *FakeWorkspace) moveSchemaChildren(catalogName, oldSchemaName, newSchema secret.FullName = catalogName + "." + newSchemaName + "." + secret.Name s.UCSecrets[secret.FullName] = secret } - for key, secret := range s.UCSecrets { - if secret.CatalogName != catalogName || secret.SchemaName != oldSchemaName { - continue - } - delete(s.UCSecrets, key) - secret.SchemaName = newSchemaName - secret.FullName = catalogName + "." + newSchemaName + "." + secret.Name - s.UCSecrets[secret.FullName] = secret - } } // SchemasDelete mirrors UC, where deleting a schema with force deletes the volumes, registered models and secrets in it too. From 916a787fd4f810e6556e7fdc37a8f82c29e3bbda Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Tue, 10 Nov 2026 15:10:43 +0100 Subject: [PATCH 06/17] acc: a secret recreated while its schema is renamed (Badness) The recreate deletes the secret at its recorded path after the schema's rename moved it, so the moved secret is left behind. Co-authored-by: Isaac --- .../databricks.yml.tmpl | 21 ++ .../recreate_in_renamed_schema/out.test.toml | 2 + .../recreate_in_renamed_schema/output.txt | 221 ++++++++++++++++++ .../secrets/recreate_in_renamed_schema/script | 38 +++ .../recreate_in_renamed_schema/test.toml | 8 + 5 files changed, 290 insertions(+) create mode 100644 acceptance/bundle/resources/secrets/recreate_in_renamed_schema/databricks.yml.tmpl create mode 100644 acceptance/bundle/resources/secrets/recreate_in_renamed_schema/out.test.toml create mode 100644 acceptance/bundle/resources/secrets/recreate_in_renamed_schema/output.txt create mode 100644 acceptance/bundle/resources/secrets/recreate_in_renamed_schema/script create mode 100644 acceptance/bundle/resources/secrets/recreate_in_renamed_schema/test.toml diff --git a/acceptance/bundle/resources/secrets/recreate_in_renamed_schema/databricks.yml.tmpl b/acceptance/bundle/resources/secrets/recreate_in_renamed_schema/databricks.yml.tmpl new file mode 100644 index 00000000000..30a41a92d18 --- /dev/null +++ b/acceptance/bundle/resources/secrets/recreate_in_renamed_schema/databricks.yml.tmpl @@ -0,0 +1,21 @@ +bundle: + name: test-bundle-$UNIQUE_NAME + +variables: + secret_value: + description: The secret's value + +resources: + catalogs: + catalog1: + name: test_catalog_$UNIQUE_NAME + schemas: + schema1: + catalog_name: ${resources.catalogs.catalog1.name} + name: myschema + secrets: + secret1: + catalog_name: ${resources.catalogs.catalog1.name} + schema_name: ${resources.schemas.schema1.name} + name: mysecret + value: ${var.secret_value} diff --git a/acceptance/bundle/resources/secrets/recreate_in_renamed_schema/out.test.toml b/acceptance/bundle/resources/secrets/recreate_in_renamed_schema/out.test.toml new file mode 100644 index 00000000000..0f167effef1 --- /dev/null +++ b/acceptance/bundle/resources/secrets/recreate_in_renamed_schema/out.test.toml @@ -0,0 +1,2 @@ +Cloud = true +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/resources/secrets/recreate_in_renamed_schema/output.txt b/acceptance/bundle/resources/secrets/recreate_in_renamed_schema/output.txt new file mode 100644 index 00000000000..0b09e17af8d --- /dev/null +++ b/acceptance/bundle/resources/secrets/recreate_in_renamed_schema/output.txt @@ -0,0 +1,221 @@ + +=== Deploy a catalog with a schema and a secret in it +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... +Created catalogs.catalog1 +Created schemas.schema1 +Created secrets.secret1 +Files: 2 uploaded, 0 deleted +Resources: 3 created, 0 changed, 0 deleted, 0 unchanged + +=== Rename the schema and the secret in one deploy. UC cannot rename a secret, so the secret is recreated +>>> update_file.py databricks.yml name: myschema name: myschema2 + +>>> update_file.py databricks.yml name: mysecret name: mysecret2 + +>>> [CLI] bundle plan --output json +{ + "resources.catalogs.catalog1": { + "action": "skip", + "remote_state": { + "catalog_type": "MANAGED_CATALOG", + "created_by": "[USERNAME]", + "full_name": "test_catalog_[UNIQUE_NAME]", + "name": "test_catalog_[UNIQUE_NAME]", + "owner": "[USERNAME]" + } + }, + "resources.schemas.schema1": { + "depends_on": [ + { + "node": "resources.catalogs.catalog1", + "label": "${resources.catalogs.catalog1.name}" + } + ], + "action": "update_id", + "new_state": { + "value": { + "catalog_name": "test_catalog_[UNIQUE_NAME]", + "name": "myschema2" + } + }, + "remote_state": { + "catalog_name": "test_catalog_[UNIQUE_NAME]", + "catalog_type": "MANAGED_CATALOG", + "created_by": "[USERNAME]", + "full_name": "test_catalog_[UNIQUE_NAME].myschema", + "name": "myschema", + "owner": "[USERNAME]", + "schema_id": "[UUID]" + }, + "changes": { + "name": { + "action": "update_id", + "reason": "id_changes", + "old": "myschema", + "new": "myschema2", + "remote": "myschema" + }, + "schema_id": { + "action": "skip", + "reason": "internal_id", + "old": "[UUID]", + "remote": "[UUID]" + } + } + }, + "resources.secrets.secret1": { + "depends_on": [ + { + "node": "resources.catalogs.catalog1", + "label": "${resources.catalogs.catalog1.name}" + }, + { + "node": "resources.schemas.schema1", + "label": "${resources.schemas.schema1.name}" + } + ], + "action": "recreate", + "new_state": { + "value": { + "catalog_name": "test_catalog_[UNIQUE_NAME]", + "name": "mysecret2", + "schema_name": "myschema2", + "value": "secret-value" + } + }, + "remote_state": { + "catalog_name": "test_catalog_[UNIQUE_NAME]", + "create_time": "[OLD_SECRET_CREATE_TIME]", + "created_by": "[USERNAME]", + "effective_owner": "[USERNAME]", + "effective_value": "[redacted]", + "full_name": "test_catalog_[UNIQUE_NAME].myschema.mysecret", + "name": "mysecret", + "owner": "[USERNAME]", + "schema_name": "myschema", + "update_time": "[OLD_SECRET_CREATE_TIME]", + "value": "[redacted]" + }, + "changes": { + "create_time": { + "action": "skip", + "reason": "spec:output_only", + "remote": "[OLD_SECRET_CREATE_TIME]" + }, + "created_by": { + "action": "skip", + "reason": "spec:output_only", + "remote": "[USERNAME]" + }, + "effective_owner": { + "action": "skip", + "reason": "spec:output_only", + "remote": "[USERNAME]" + }, + "effective_value": { + "action": "skip", + "reason": "spec:output_only", + "remote": "[redacted]" + }, + "full_name": { + "action": "skip", + "reason": "spec:output_only", + "remote": "test_catalog_[UNIQUE_NAME].myschema.mysecret" + }, + "name": { + "action": "recreate", + "reason": "id_field", + "old": "mysecret", + "new": "mysecret2", + "remote": "mysecret" + }, + "owner": { + "action": "skip", + "reason": "spec:input_only", + "remote": "[USERNAME]" + }, + "schema_name": { + "action": "update_id", + "reason": "parent_renamed", + "old": "myschema", + "new": "myschema2", + "remote": "myschema" + }, + "update_time": { + "action": "skip", + "reason": "spec:output_only", + "remote": "[OLD_SECRET_CREATE_TIME]" + }, + "value": { + "action": "skip", + "reason": "custom", + "old": "", + "new": "[redacted]", + "remote": "[redacted]" + } + } + } +} + +>>> [CLI] bundle deploy --auto-approve +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... +Recreated secrets.secret1 +Updated schemas.schema1 +Files: 2 uploaded, 0 deleted +Resources: 1 created, 1 changed, 1 deleted, 1 unchanged + +>>> print_requests.py //unity-catalog +{ + "method": "PATCH", + "path": "/api/2.1/unity-catalog/schemas/test_catalog_[UNIQUE_NAME].myschema", + "body": { + "comment": "", + "custom_max_retention_hours": 0, + "new_name": "myschema2" + } +} +{ + "method": "DELETE", + "path": "/api/2.1/unity-catalog/secrets/test_catalog_[UNIQUE_NAME].myschema.mysecret" +} +{ + "method": "POST", + "path": "/api/2.1/unity-catalog/secrets", + "body": { + "catalog_name": "test_catalog_[UNIQUE_NAME]", + "name": "mysecret2", + "schema_name": "myschema2", + "value": "secret-value" + } +} + +=== The new secret is in the renamed schema +>>> [CLI] secrets-uc get-secret test_catalog_[UNIQUE_NAME].myschema2.mysecret2 +{ + "full_name": "test_catalog_[UNIQUE_NAME].myschema2.mysecret2" +} + +=== The old secret moved with the renamed schema. It should have been deleted +>>> [CLI] secrets-uc get-secret test_catalog_[UNIQUE_NAME].myschema2.mysecret +{ + "full_name": "test_catalog_[UNIQUE_NAME].myschema2.mysecret", + "create_time": "[OLD_SECRET_CREATE_TIME]" +} + +=== Redeploy is a no-op +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 3 unchanged + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.catalogs.catalog1 + delete resources.schemas.schema1 + delete resources.secrets.secret1 + +This action will result in the deletion of the following UC schemas. Any underlying data may be lost: + delete resources.schemas.schema1 + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default + +Destroy: 3 deleted diff --git a/acceptance/bundle/resources/secrets/recreate_in_renamed_schema/script b/acceptance/bundle/resources/secrets/recreate_in_renamed_schema/script new file mode 100644 index 00000000000..0c463bbe4d1 --- /dev/null +++ b/acceptance/bundle/resources/secrets/recreate_in_renamed_schema/script @@ -0,0 +1,38 @@ +envsubst < databricks.yml.tmpl > databricks.yml +export BUNDLE_VAR_secret_value=secret-value + +CATALOG="test_catalog_${UNIQUE_NAME}" + +cleanup() { + trace $CLI bundle destroy --auto-approve + + # Consume the destroy's requests so none are left recorded. + print_requests.py //unity-catalog &> LOG.destroy_requests +} +trap cleanup EXIT + +# Drop the remote_state fields that differ between clouds or from the test server. +plan() { + trace $CLI bundle plan --output json | jq .plan | normalize_uc_payload.py created_at updated_at updated_by metastore_id browse_only effective_predictive_optimization_flag enable_predictive_optimization isolation_mode securable_type +} + +title "Deploy a catalog with a schema and a secret in it" +trace $CLI bundle deploy +print_requests.py //unity-catalog &> LOG.deploy_requests +add_repl "$($CLI secrets-uc get-secret "$CATALOG.myschema.mysecret" | jq -r .create_time)" OLD_SECRET_CREATE_TIME + +title "Rename the schema and the secret in one deploy. UC cannot rename a secret, so the secret is recreated" +trace update_file.py databricks.yml "name: myschema" "name: myschema2" +trace update_file.py databricks.yml "name: mysecret" "name: mysecret2" +plan +trace $CLI bundle deploy --auto-approve +trace print_requests.py //unity-catalog + +title "The new secret is in the renamed schema" +trace $CLI secrets-uc get-secret "$CATALOG.myschema2.mysecret2" | jq '{full_name}' + +title "The old secret moved with the renamed schema. It should have been deleted" +trace $CLI secrets-uc get-secret "$CATALOG.myschema2.mysecret" | jq '{full_name, create_time}' + +title "Redeploy is a no-op" +trace $CLI bundle plan diff --git a/acceptance/bundle/resources/secrets/recreate_in_renamed_schema/test.toml b/acceptance/bundle/resources/secrets/recreate_in_renamed_schema/test.toml new file mode 100644 index 00000000000..ce926750f86 --- /dev/null +++ b/acceptance/bundle/resources/secrets/recreate_in_renamed_schema/test.toml @@ -0,0 +1,8 @@ +Badness = "The schema's rename moves the secret, but the recreate deletes the secret at its recorded path, which no longer exists. The secret moved with the schema is left behind untracked." +Cloud = true + +Ignore = [ + ".databricks", + ".gitignore", + "databricks.yml", +] From 5a229b9c1d85d4f5c2da3743722ff16169f35ac6 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Tue, 10 Nov 2026 16:19:16 +0100 Subject: [PATCH 07/17] acc: the secret recreated in a renamed schema is deleted where it moved Co-authored-by: Isaac --- .../secrets/recreate_in_renamed_schema/output.txt | 12 +++++------- .../secrets/recreate_in_renamed_schema/script | 4 ++-- .../secrets/recreate_in_renamed_schema/test.toml | 1 - 3 files changed, 7 insertions(+), 10 deletions(-) diff --git a/acceptance/bundle/resources/secrets/recreate_in_renamed_schema/output.txt b/acceptance/bundle/resources/secrets/recreate_in_renamed_schema/output.txt index 0b09e17af8d..b3dc6b154bb 100644 --- a/acceptance/bundle/resources/secrets/recreate_in_renamed_schema/output.txt +++ b/acceptance/bundle/resources/secrets/recreate_in_renamed_schema/output.txt @@ -65,6 +65,7 @@ Resources: 3 created, 0 changed, 0 deleted, 0 unchanged } }, "resources.secrets.secret1": { + "id": "test_catalog_[UNIQUE_NAME].myschema2.mysecret", "depends_on": [ { "node": "resources.catalogs.catalog1", @@ -177,7 +178,7 @@ Resources: 1 created, 1 changed, 1 deleted, 1 unchanged } { "method": "DELETE", - "path": "/api/2.1/unity-catalog/secrets/test_catalog_[UNIQUE_NAME].myschema.mysecret" + "path": "/api/2.1/unity-catalog/secrets/test_catalog_[UNIQUE_NAME].myschema2.mysecret" } { "method": "POST", @@ -196,12 +197,9 @@ Resources: 1 created, 1 changed, 1 deleted, 1 unchanged "full_name": "test_catalog_[UNIQUE_NAME].myschema2.mysecret2" } -=== The old secret moved with the renamed schema. It should have been deleted ->>> [CLI] secrets-uc get-secret test_catalog_[UNIQUE_NAME].myschema2.mysecret -{ - "full_name": "test_catalog_[UNIQUE_NAME].myschema2.mysecret", - "create_time": "[OLD_SECRET_CREATE_TIME]" -} +=== The old secret, moved with the renamed schema, was deleted there +>>> musterr [CLI] secrets-uc get-secret test_catalog_[UNIQUE_NAME].myschema2.mysecret +Error: Secret test_catalog_[UNIQUE_NAME].myschema2.mysecret not found === Redeploy is a no-op >>> [CLI] bundle plan diff --git a/acceptance/bundle/resources/secrets/recreate_in_renamed_schema/script b/acceptance/bundle/resources/secrets/recreate_in_renamed_schema/script index 0c463bbe4d1..cfe49f51a49 100644 --- a/acceptance/bundle/resources/secrets/recreate_in_renamed_schema/script +++ b/acceptance/bundle/resources/secrets/recreate_in_renamed_schema/script @@ -31,8 +31,8 @@ trace print_requests.py //unity-catalog title "The new secret is in the renamed schema" trace $CLI secrets-uc get-secret "$CATALOG.myschema2.mysecret2" | jq '{full_name}' -title "The old secret moved with the renamed schema. It should have been deleted" -trace $CLI secrets-uc get-secret "$CATALOG.myschema2.mysecret" | jq '{full_name, create_time}' +title "The old secret, moved with the renamed schema, was deleted there" +trace musterr $CLI secrets-uc get-secret "$CATALOG.myschema2.mysecret" title "Redeploy is a no-op" trace $CLI bundle plan diff --git a/acceptance/bundle/resources/secrets/recreate_in_renamed_schema/test.toml b/acceptance/bundle/resources/secrets/recreate_in_renamed_schema/test.toml index ce926750f86..4272a80cf61 100644 --- a/acceptance/bundle/resources/secrets/recreate_in_renamed_schema/test.toml +++ b/acceptance/bundle/resources/secrets/recreate_in_renamed_schema/test.toml @@ -1,4 +1,3 @@ -Badness = "The schema's rename moves the secret, but the recreate deletes the secret at its recorded path, which no longer exists. The secret moved with the schema is left behind untracked." Cloud = true Ignore = [ From 9125eb6e9fffb79a3bac72fdc1529a1005ed9c01 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Tue, 10 Nov 2026 16:25:05 +0100 Subject: [PATCH 08/17] testserver: report a missing UC secret with UC's message Co-authored-by: Isaac --- .../resources/secrets/recreate_in_renamed_schema/output.txt | 2 +- libs/testserver/uc_secrets.go | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/acceptance/bundle/resources/secrets/recreate_in_renamed_schema/output.txt b/acceptance/bundle/resources/secrets/recreate_in_renamed_schema/output.txt index b3dc6b154bb..e8cf5a3cbcb 100644 --- a/acceptance/bundle/resources/secrets/recreate_in_renamed_schema/output.txt +++ b/acceptance/bundle/resources/secrets/recreate_in_renamed_schema/output.txt @@ -199,7 +199,7 @@ Resources: 1 created, 1 changed, 1 deleted, 1 unchanged === The old secret, moved with the renamed schema, was deleted there >>> musterr [CLI] secrets-uc get-secret test_catalog_[UNIQUE_NAME].myschema2.mysecret -Error: Secret test_catalog_[UNIQUE_NAME].myschema2.mysecret not found +Error: Secret 'test_catalog_[UNIQUE_NAME].myschema2.mysecret' does not exist. === Redeploy is a no-op >>> [CLI] bundle plan diff --git a/libs/testserver/uc_secrets.go b/libs/testserver/uc_secrets.go index 28bacf1fbed..425cffdb1ba 100644 --- a/libs/testserver/uc_secrets.go +++ b/libs/testserver/uc_secrets.go @@ -94,7 +94,7 @@ func (s *FakeWorkspace) SecretsUcGetSecret(req Request) Response { StatusCode: 404, Body: map[string]any{ "error_code": "RESOURCE_DOES_NOT_EXIST", - "message": fmt.Sprintf("Secret %s not found", fullName), + "message": fmt.Sprintf("Secret '%s' does not exist.", fullName), }, } } @@ -143,7 +143,7 @@ func (s *FakeWorkspace) SecretsUcUpdateSecret(req Request) Response { StatusCode: 404, Body: map[string]any{ "error_code": "RESOURCE_DOES_NOT_EXIST", - "message": fmt.Sprintf("Secret %s not found", fullName), + "message": fmt.Sprintf("Secret '%s' does not exist.", fullName), }, } } @@ -195,7 +195,7 @@ func (s *FakeWorkspace) SecretsUcDeleteSecret(req Request) Response { StatusCode: 404, Body: map[string]any{ "error_code": "RESOURCE_DOES_NOT_EXIST", - "message": fmt.Sprintf("Secret %s not found", fullName), + "message": fmt.Sprintf("Secret '%s' does not exist.", fullName), }, } } From 65958f229084f95a4d3f29cf157646c5b5a013c8 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Tue, 10 Nov 2026 16:44:34 +0100 Subject: [PATCH 09/17] acc: also deploy the secret rename tests from a saved plan (READPLAN) Co-authored-by: Isaac --- .../bundle/resources/secrets/parent_rename/out.test.toml | 1 + acceptance/bundle/resources/secrets/parent_rename/script | 6 +++--- .../bundle/resources/secrets/parent_rename/test.toml | 7 +++++++ .../secrets/recreate_in_renamed_schema/out.test.toml | 1 + .../resources/secrets/recreate_in_renamed_schema/script | 4 ++-- .../resources/secrets/recreate_in_renamed_schema/test.toml | 7 +++++++ 6 files changed, 21 insertions(+), 5 deletions(-) diff --git a/acceptance/bundle/resources/secrets/parent_rename/out.test.toml b/acceptance/bundle/resources/secrets/parent_rename/out.test.toml index 0f167effef1..7a62f726f25 100644 --- a/acceptance/bundle/resources/secrets/parent_rename/out.test.toml +++ b/acceptance/bundle/resources/secrets/parent_rename/out.test.toml @@ -1,2 +1,3 @@ Cloud = true EnvMatrix.DMS = ["", "true"] +EnvMatrix.READPLAN = ["", "1"] diff --git a/acceptance/bundle/resources/secrets/parent_rename/script b/acceptance/bundle/resources/secrets/parent_rename/script index 8c5fa6cc7c9..efeb9c91c31 100644 --- a/acceptance/bundle/resources/secrets/parent_rename/script +++ b/acceptance/bundle/resources/secrets/parent_rename/script @@ -11,7 +11,7 @@ trap cleanup EXIT # Drop the remote_state fields that differ between clouds or from the test server. plan() { - trace $CLI bundle plan --output json | jq .plan | normalize_uc_payload.py created_at updated_at updated_by metastore_id browse_only effective_predictive_optimization_flag enable_predictive_optimization isolation_mode securable_type + trace $CLI bundle plan --output json | tee .databricks/plan.json | jq .plan | normalize_uc_payload.py created_at updated_at updated_by metastore_id browse_only effective_predictive_optimization_flag enable_predictive_optimization isolation_mode securable_type } title "Deploy a catalog with a schema and a secret in it" @@ -22,7 +22,7 @@ add_repl "$($CLI secrets-uc get-secret "test_catalog_${UNIQUE_NAME}.myschema.mys title "Rename the schema: UC moves the secret with it, so the secret is re-keyed without a request" trace update_file.py databricks.yml "name: myschema" "name: myschema2" plan -trace $CLI bundle deploy +trace $CLI bundle deploy $(readplanarg .databricks/plan.json) trace print_requests.py //unity-catalog trace $CLI secrets-uc get-secret "test_catalog_${UNIQUE_NAME}.myschema2.mysecret" | jq '{full_name, comment, create_time}' @@ -30,7 +30,7 @@ title "Rename the catalog and change the secret's comment: the update reaches th trace update_file.py databricks.yml test_catalog_ test_catalog_renamed_ trace update_file.py databricks.yml "comment: c1" "comment: c2" plan -trace $CLI bundle deploy +trace $CLI bundle deploy $(readplanarg .databricks/plan.json) trace print_requests.py //unity-catalog trace $CLI secrets-uc get-secret "test_catalog_renamed_${UNIQUE_NAME}.myschema2.mysecret" | jq '{full_name, comment, create_time}' diff --git a/acceptance/bundle/resources/secrets/parent_rename/test.toml b/acceptance/bundle/resources/secrets/parent_rename/test.toml index 4272a80cf61..ebd892fd9df 100644 --- a/acceptance/bundle/resources/secrets/parent_rename/test.toml +++ b/acceptance/bundle/resources/secrets/parent_rename/test.toml @@ -1,7 +1,14 @@ Cloud = true +EnvMatrix.READPLAN = ["", "1"] Ignore = [ ".databricks", ".gitignore", "databricks.yml", ] + +# When READPLAN=1, "bundle deploy" is called as "bundle deploy --plan .databricks/plan.json". +# Normalize so both variants produce identical output. +[[Repls]] +Old = ' --plan .databricks/plan.json' +New = '' diff --git a/acceptance/bundle/resources/secrets/recreate_in_renamed_schema/out.test.toml b/acceptance/bundle/resources/secrets/recreate_in_renamed_schema/out.test.toml index 0f167effef1..7a62f726f25 100644 --- a/acceptance/bundle/resources/secrets/recreate_in_renamed_schema/out.test.toml +++ b/acceptance/bundle/resources/secrets/recreate_in_renamed_schema/out.test.toml @@ -1,2 +1,3 @@ Cloud = true EnvMatrix.DMS = ["", "true"] +EnvMatrix.READPLAN = ["", "1"] diff --git a/acceptance/bundle/resources/secrets/recreate_in_renamed_schema/script b/acceptance/bundle/resources/secrets/recreate_in_renamed_schema/script index cfe49f51a49..3c3b984a7e5 100644 --- a/acceptance/bundle/resources/secrets/recreate_in_renamed_schema/script +++ b/acceptance/bundle/resources/secrets/recreate_in_renamed_schema/script @@ -13,7 +13,7 @@ trap cleanup EXIT # Drop the remote_state fields that differ between clouds or from the test server. plan() { - trace $CLI bundle plan --output json | jq .plan | normalize_uc_payload.py created_at updated_at updated_by metastore_id browse_only effective_predictive_optimization_flag enable_predictive_optimization isolation_mode securable_type + trace $CLI bundle plan --output json | tee .databricks/plan.json | jq .plan | normalize_uc_payload.py created_at updated_at updated_by metastore_id browse_only effective_predictive_optimization_flag enable_predictive_optimization isolation_mode securable_type } title "Deploy a catalog with a schema and a secret in it" @@ -25,7 +25,7 @@ title "Rename the schema and the secret in one deploy. UC cannot rename a secret trace update_file.py databricks.yml "name: myschema" "name: myschema2" trace update_file.py databricks.yml "name: mysecret" "name: mysecret2" plan -trace $CLI bundle deploy --auto-approve +trace $CLI bundle deploy $(readplanarg .databricks/plan.json) --auto-approve trace print_requests.py //unity-catalog title "The new secret is in the renamed schema" diff --git a/acceptance/bundle/resources/secrets/recreate_in_renamed_schema/test.toml b/acceptance/bundle/resources/secrets/recreate_in_renamed_schema/test.toml index 4272a80cf61..ebd892fd9df 100644 --- a/acceptance/bundle/resources/secrets/recreate_in_renamed_schema/test.toml +++ b/acceptance/bundle/resources/secrets/recreate_in_renamed_schema/test.toml @@ -1,7 +1,14 @@ Cloud = true +EnvMatrix.READPLAN = ["", "1"] Ignore = [ ".databricks", ".gitignore", "databricks.yml", ] + +# When READPLAN=1, "bundle deploy" is called as "bundle deploy --plan .databricks/plan.json". +# Normalize so both variants produce identical output. +[[Repls]] +Old = ' --plan .databricks/plan.json' +New = '' From 09d0e816d1eed9e9f09757c1b58ef72941dc6785 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Mon, 16 Nov 2026 10:36:25 +0100 Subject: [PATCH 10/17] Make the changelog entry mirror the PR title Co-authored-by: Isaac --- .nextchanges/bundles/secret-parent-rename.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.nextchanges/bundles/secret-parent-rename.md b/.nextchanges/bundles/secret-parent-rename.md index 0d3c9b7d1ce..0491f8af6f1 100644 --- a/.nextchanges/bundles/secret-parent-rename.md +++ b/.nextchanges/bundles/secret-parent-rename.md @@ -1 +1 @@ -* UC secrets inside a renamed catalog or schema now follow it to the new name; recreating them failed because they already exist there. ([#7381](https://github.com/databricks/cli/pull/7381)) +* UC secrets follow a renamed catalog or schema. ([#7381](https://github.com/databricks/cli/pull/7381)) From ed0a51d440bea8e1dad9a7475b57a42d9ca6736d Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Mon, 16 Nov 2026 10:47:59 +0100 Subject: [PATCH 11/17] acc: the secret follows the schema's rename in rename_with_model_and_secret Co-authored-by: Isaac --- .../rename_with_model_and_secret/output.txt | 22 +++---------------- .../rename_with_model_and_secret/script | 2 +- 2 files changed, 4 insertions(+), 20 deletions(-) diff --git a/acceptance/bundle/resources/schemas/rename_with_model_and_secret/output.txt b/acceptance/bundle/resources/schemas/rename_with_model_and_secret/output.txt index 18ac6b3162b..9a9d331d38e 100644 --- a/acceptance/bundle/resources/schemas/rename_with_model_and_secret/output.txt +++ b/acceptance/bundle/resources/schemas/rename_with_model_and_secret/output.txt @@ -9,28 +9,22 @@ Created secrets.secret1 Files: 2 uploaded, 0 deleted Resources: 3 created, 0 changed, 0 deleted, 0 unchanged -=== Rename the schema. UC moves the model and the secret with it; the model follows the rename, the secret cannot yet, so it is recreated where it moved +=== Rename the schema. UC moves the model and the secret with it, and both follow the rename without a request >>> update_file.py databricks.yml test-schema- test-schema-renamed- >>> [CLI] bundle plan update schemas.schema1 -recreate secrets.secret1 -Plan: 1 to add, 1 to change, 1 to delete, 1 unchanged +Plan: 0 to add, 1 to change, 0 to delete, 2 unchanged >>> [CLI] bundle deploy --auto-approve Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... -Recreated secrets.secret1 Updated schemas.schema1 Files: 2 uploaded, 0 deleted -Resources: 1 created, 1 changed, 1 deleted, 1 unchanged +Resources: 0 created, 1 changed, 0 deleted, 2 unchanged >>> print_requests.py --sort //unity-catalog -{ - "method": "DELETE", - "path": "/api/2.1/unity-catalog/secrets/main.test-schema-renamed-[UNIQUE_NAME].mysecret" -} { "method": "PATCH", "path": "/api/2.1/unity-catalog/schemas/main.test-schema-[UNIQUE_NAME]", @@ -40,16 +34,6 @@ Resources: 1 created, 1 changed, 1 deleted, 1 unchanged "new_name": "test-schema-renamed-[UNIQUE_NAME]" } } -{ - "method": "POST", - "path": "/api/2.1/unity-catalog/secrets", - "body": { - "catalog_name": "main", - "name": "mysecret", - "schema_name": "test-schema-renamed-[UNIQUE_NAME]", - "value": "secret-value" - } -} >>> [CLI] registered-models get main.test-schema-renamed-[UNIQUE_NAME].mymodel { diff --git a/acceptance/bundle/resources/schemas/rename_with_model_and_secret/script b/acceptance/bundle/resources/schemas/rename_with_model_and_secret/script index 17140a4e99b..7dbc666c82c 100644 --- a/acceptance/bundle/resources/schemas/rename_with_model_and_secret/script +++ b/acceptance/bundle/resources/schemas/rename_with_model_and_secret/script @@ -15,7 +15,7 @@ title "Deploy a schema with a registered model and a secret in it" trace $CLI bundle deploy print_requests.py //unity-catalog &> LOG.deploy_requests -title "Rename the schema. UC moves the model and the secret with it; the model follows the rename, the secret cannot yet, so it is recreated where it moved" +title "Rename the schema. UC moves the model and the secret with it, and both follow the rename without a request" trace update_file.py databricks.yml test-schema- test-schema-renamed- trace $CLI bundle plan trace $CLI bundle deploy --auto-approve From 7ec06a973a53d87ca04e8d188acc533b7901c0da Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Mon, 16 Nov 2026 10:48:06 +0100 Subject: [PATCH 12/17] acc: regenerate the secret goldens after rebase Co-authored-by: Isaac --- acceptance/bundle/resources/secrets/parent_rename/output.txt | 4 ++++ .../resources/secrets/recreate_in_renamed_schema/output.txt | 5 +++++ 2 files changed, 9 insertions(+) diff --git a/acceptance/bundle/resources/secrets/parent_rename/output.txt b/acceptance/bundle/resources/secrets/parent_rename/output.txt index 929a3b532dd..6cac5f13385 100644 --- a/acceptance/bundle/resources/secrets/parent_rename/output.txt +++ b/acceptance/bundle/resources/secrets/parent_rename/output.txt @@ -1,5 +1,6 @@ === Deploy a catalog with a schema and a secret in it + >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... Created catalogs.catalog1 @@ -37,6 +38,7 @@ Resources: 3 created, 0 changed, 0 deleted, 0 unchanged } === Rename the schema: UC moves the secret with it, so the secret is re-keyed without a request + >>> update_file.py databricks.yml name: myschema name: myschema2 >>> [CLI] bundle plan --output json @@ -205,6 +207,7 @@ Resources: 0 created, 1 changed, 0 deleted, 2 unchanged } === Rename the catalog and change the secret's comment: the update reaches the secret at its new path + >>> update_file.py databricks.yml test_catalog_ test_catalog_renamed_ >>> update_file.py databricks.yml comment: c1 comment: c2 @@ -422,6 +425,7 @@ Resources: 0 created, 2 changed, 0 deleted, 1 unchanged } === The state tracks the new path and the plan is a no-op + >>> [CLI] bundle summary --output json { "catalogs": { diff --git a/acceptance/bundle/resources/secrets/recreate_in_renamed_schema/output.txt b/acceptance/bundle/resources/secrets/recreate_in_renamed_schema/output.txt index e8cf5a3cbcb..c292b659090 100644 --- a/acceptance/bundle/resources/secrets/recreate_in_renamed_schema/output.txt +++ b/acceptance/bundle/resources/secrets/recreate_in_renamed_schema/output.txt @@ -1,5 +1,6 @@ === Deploy a catalog with a schema and a secret in it + >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... Created catalogs.catalog1 @@ -9,6 +10,7 @@ Files: 2 uploaded, 0 deleted Resources: 3 created, 0 changed, 0 deleted, 0 unchanged === Rename the schema and the secret in one deploy. UC cannot rename a secret, so the secret is recreated + >>> update_file.py databricks.yml name: myschema name: myschema2 >>> update_file.py databricks.yml name: mysecret name: mysecret2 @@ -192,16 +194,19 @@ Resources: 1 created, 1 changed, 1 deleted, 1 unchanged } === The new secret is in the renamed schema + >>> [CLI] secrets-uc get-secret test_catalog_[UNIQUE_NAME].myschema2.mysecret2 { "full_name": "test_catalog_[UNIQUE_NAME].myschema2.mysecret2" } === The old secret, moved with the renamed schema, was deleted there + >>> musterr [CLI] secrets-uc get-secret test_catalog_[UNIQUE_NAME].myschema2.mysecret Error: Secret 'test_catalog_[UNIQUE_NAME].myschema2.mysecret' does not exist. === Redeploy is a no-op + >>> [CLI] bundle plan Plan: 0 to add, 0 to change, 0 to delete, 3 unchanged From 7d79924f1363a60a8720c48e5292b37b65fa258b Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Tue, 17 Nov 2026 12:39:01 +0100 Subject: [PATCH 13/17] direct: drop the DoUpdateWithID check change, which never ran for secrets (review S3) The check is skipped for resources that implement OverrideChangeDesc, as secrets do. Co-authored-by: Isaac --- bundle/direct/dresources/adapter.go | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/bundle/direct/dresources/adapter.go b/bundle/direct/dresources/adapter.go index ae73795ecd7..21ac7a02344 100644 --- a/bundle/direct/dresources/adapter.go +++ b/bundle/direct/dresources/adapter.go @@ -480,10 +480,8 @@ func (a *Adapter) validate() error { if hasUpdateWithIDTrigger && a.doUpdateWithID == nil { return errors.New("resourceConfig has updatable_id_fields but DoUpdateWithID is not implemented") } - // A parent's rename also re-keys a resource through its provided_id_fields (see followsParentRename). - canUpdateWithID := hasUpdateWithIDTrigger || (a.resourceConfig != nil && len(a.resourceConfig.ProvidedIDFields) > 0) - if a.doUpdateWithID != nil && !canUpdateWithID { - return errors.New("DoUpdateWithID is implemented but resourceConfig lacks updatable_id_fields and provided_id_fields") + if a.doUpdateWithID != nil && !hasUpdateWithIDTrigger { + return errors.New("DoUpdateWithID is implemented but resourceConfig lacks updatable_id_fields") } } From fd4935254329d22ca8c243b25a08831d45bc61fe Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Tue, 17 Nov 2026 12:39:32 +0100 Subject: [PATCH 14/17] direct: a secret's own name does not follow a renamed parent (review S1) Only parent_id_fields (catalog_name, schema_name, from #7718) follow a parent's rename, so a name that references a renamed schema's name recreates the secret instead of re-keying it to a path that does not exist. Co-authored-by: Isaac --- bundle/direct/bundle_plan_test.go | 7 +++++++ bundle/direct/dresources/configs/secrets.yml | 4 ++-- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/bundle/direct/bundle_plan_test.go b/bundle/direct/bundle_plan_test.go index 947f5bee96d..8645ca5d648 100644 --- a/bundle/direct/bundle_plan_test.go +++ b/bundle/direct/bundle_plan_test.go @@ -307,6 +307,13 @@ func TestFollowsParentRename(t *testing.T) { old: "s_old", expectedAction: deployplan.Recreate, }, + { + name: "own name referencing the renamed parent is recreated", + resource: "secrets", + field: "name", + old: "s_old", + expectedAction: deployplan.Recreate, + }, { name: "recreate_on_changes field is recreated", resource: "volumes", diff --git a/bundle/direct/dresources/configs/secrets.yml b/bundle/direct/dresources/configs/secrets.yml index f4458d2f5f9..e5d0177a7dc 100644 --- a/bundle/direct/dresources/configs/secrets.yml +++ b/bundle/direct/dresources/configs/secrets.yml @@ -4,8 +4,8 @@ parent_id_fields: - field: schema_name reason: parent provided_id_fields: - # The ID is the full name. UC cannot rename a secret, but it moves the secret with a renamed - # catalog or schema, which re-keys it (see followsParentRename). + # The ID is the full name. UC cannot rename a secret; a renamed catalog or schema moves it (see + # parent_id_fields). - field: catalog_name reason: id_field - field: schema_name From 81cc9c6bd5bd4081e3b8c9fda41d78b8992f2ad4 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Tue, 17 Nov 2026 14:06:25 +0100 Subject: [PATCH 15/17] testserver: report a UC secret that already exists with UC's 409 ALREADY_EXISTS Verified on AWS, Azure and GCP: "Secret '' already exists". Co-authored-by: Isaac --- libs/testserver/uc_secrets.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/libs/testserver/uc_secrets.go b/libs/testserver/uc_secrets.go index 425cffdb1ba..1c8e6839a67 100644 --- a/libs/testserver/uc_secrets.go +++ b/libs/testserver/uc_secrets.go @@ -32,13 +32,13 @@ func (s *FakeWorkspace) SecretsUcCreateSecret(req Request) Response { inputSecret.SchemaName, inputSecret.Name) - // Check if secret already exists + // UC answers 409 ALREADY_EXISTS (verified on AWS, Azure and GCP). if _, exists := s.UCSecrets[fullName]; exists { return Response{ StatusCode: 409, Body: map[string]any{ - "error_code": "RESOURCE_ALREADY_EXISTS", - "message": fmt.Sprintf("Secret %s already exists", fullName), + "error_code": "ALREADY_EXISTS", + "message": fmt.Sprintf("Secret '%s' already exists", fullName), }, } } From 306b7f6924f66da93da7fafc57af53c589cad640 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Tue, 17 Nov 2026 14:07:04 +0100 Subject: [PATCH 16/17] acc: a secret whose schema is renamed out of band is created again (Badness, review G1) Co-authored-by: Isaac --- .../databricks.yml.tmpl | 18 +++++ .../parent_rename_out_of_band/out.test.toml | 2 + .../parent_rename_out_of_band/output.txt | 68 +++++++++++++++++++ .../secrets/parent_rename_out_of_band/script | 30 ++++++++ .../parent_rename_out_of_band/test.toml | 8 +++ 5 files changed, 126 insertions(+) create mode 100644 acceptance/bundle/resources/secrets/parent_rename_out_of_band/databricks.yml.tmpl create mode 100644 acceptance/bundle/resources/secrets/parent_rename_out_of_band/out.test.toml create mode 100644 acceptance/bundle/resources/secrets/parent_rename_out_of_band/output.txt create mode 100644 acceptance/bundle/resources/secrets/parent_rename_out_of_band/script create mode 100644 acceptance/bundle/resources/secrets/parent_rename_out_of_band/test.toml diff --git a/acceptance/bundle/resources/secrets/parent_rename_out_of_band/databricks.yml.tmpl b/acceptance/bundle/resources/secrets/parent_rename_out_of_band/databricks.yml.tmpl new file mode 100644 index 00000000000..ac557131422 --- /dev/null +++ b/acceptance/bundle/resources/secrets/parent_rename_out_of_band/databricks.yml.tmpl @@ -0,0 +1,18 @@ +bundle: + name: test-bundle-$UNIQUE_NAME + +variables: + secret_value: + description: The secret's value + +resources: + schemas: + schema1: + catalog_name: main + name: test-schema-$UNIQUE_NAME + secrets: + secret1: + catalog_name: main + schema_name: ${resources.schemas.schema1.name} + name: mysecret + value: ${var.secret_value} diff --git a/acceptance/bundle/resources/secrets/parent_rename_out_of_band/out.test.toml b/acceptance/bundle/resources/secrets/parent_rename_out_of_band/out.test.toml new file mode 100644 index 00000000000..0f167effef1 --- /dev/null +++ b/acceptance/bundle/resources/secrets/parent_rename_out_of_band/out.test.toml @@ -0,0 +1,2 @@ +Cloud = true +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/resources/secrets/parent_rename_out_of_band/output.txt b/acceptance/bundle/resources/secrets/parent_rename_out_of_band/output.txt new file mode 100644 index 00000000000..ce8e31c8c36 --- /dev/null +++ b/acceptance/bundle/resources/secrets/parent_rename_out_of_band/output.txt @@ -0,0 +1,68 @@ + +=== Deploy a schema with a secret in it + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... +Created schemas.schema1 +Created secrets.secret1 +Files: 2 uploaded, 0 deleted +Resources: 2 created, 0 changed, 0 deleted, 0 unchanged + +=== Rename the schema out of band. UC moves the secret with it + +>>> [CLI] schemas update main.test-schema-[UNIQUE_NAME] --new-name test-schema-renamed-[UNIQUE_NAME] +{ + "full_name": "main.test-schema-renamed-[UNIQUE_NAME]" +} + +=== With the config renamed to match, the plan finds the schema by its schema_id. A secret has no such id, so the plan creates it again + +>>> update_file.py databricks.yml test-schema- test-schema-renamed- + +>>> [CLI] bundle plan +create secrets.secret1 + +Plan: 1 to add, 0 to change, 0 to delete, 1 unchanged + +=== The create fails: the secret is already there + +>>> errcode [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... +Error: cannot create resources.secrets.secret1: Secret 'main.test-schema-renamed-[UNIQUE_NAME].mysecret' already exists (409 ALREADY_EXISTS) + +Endpoint: POST [DATABRICKS_URL]/api/2.1/unity-catalog/secrets +HTTP Status: 409 Conflict +API error_code: ALREADY_EXISTS +API message: Secret 'main.test-schema-renamed-[UNIQUE_NAME].mysecret' already exists + +Files: 3 uploaded, 0 deleted + +Exit code: 1 + +>>> print_requests.py //unity-catalog +{ + "method": "POST", + "path": "/api/2.1/unity-catalog/secrets", + "body": { + "catalog_name": "main", + "name": "mysecret", + "schema_name": "test-schema-renamed-[UNIQUE_NAME]", + "value": "secret-value" + } +} + +>>> [CLI] secrets-uc get-secret main.test-schema-renamed-[UNIQUE_NAME].mysecret +{ + "full_name": "main.test-schema-renamed-[UNIQUE_NAME].mysecret" +} + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.schemas.schema1 + +This action will result in the deletion of the following UC schemas. Any underlying data may be lost: + delete resources.schemas.schema1 + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default + +Destroy: 1 deleted diff --git a/acceptance/bundle/resources/secrets/parent_rename_out_of_band/script b/acceptance/bundle/resources/secrets/parent_rename_out_of_band/script new file mode 100644 index 00000000000..3611c7066e4 --- /dev/null +++ b/acceptance/bundle/resources/secrets/parent_rename_out_of_band/script @@ -0,0 +1,30 @@ +envsubst < databricks.yml.tmpl > databricks.yml +export BUNDLE_VAR_secret_value=secret-value + +OLD_SCHEMA="main.test-schema-${UNIQUE_NAME}" +NEW_SCHEMA="main.test-schema-renamed-${UNIQUE_NAME}" + +cleanup() { + trace $CLI bundle destroy --auto-approve + + # Consume the destroy's requests so none are left recorded. + print_requests.py //unity-catalog &> LOG.destroy_requests +} +trap cleanup EXIT + +title "Deploy a schema with a secret in it" +trace $CLI bundle deploy +print_requests.py //unity-catalog &> LOG.deploy_requests + +title "Rename the schema out of band. UC moves the secret with it" +trace $CLI schemas update "$OLD_SCHEMA" --new-name "test-schema-renamed-${UNIQUE_NAME}" | jq '{full_name}' +print_requests.py //unity-catalog &> LOG.rename_requests + +title "With the config renamed to match, the plan finds the schema by its schema_id. A secret has no such id, so the plan creates it again" +trace update_file.py databricks.yml test-schema- test-schema-renamed- +trace $CLI bundle plan + +title "The create fails: the secret is already there" +trace errcode $CLI bundle deploy +trace print_requests.py //unity-catalog +trace $CLI secrets-uc get-secret "$NEW_SCHEMA.mysecret" | jq '{full_name}' diff --git a/acceptance/bundle/resources/secrets/parent_rename_out_of_band/test.toml b/acceptance/bundle/resources/secrets/parent_rename_out_of_band/test.toml new file mode 100644 index 00000000000..b1af5f43017 --- /dev/null +++ b/acceptance/bundle/resources/secrets/parent_rename_out_of_band/test.toml @@ -0,0 +1,8 @@ +Cloud = true +Badness = "The plan does not look for a secret whose schema was renamed outside the bundle (a secret has no recorded id to find it by), so it creates it again: the create fails with ALREADY_EXISTS on every deploy. Same on main." + +Ignore = [ + ".databricks", + ".gitignore", + "databricks.yml", +] From ae7b23ceaf0be1cfb6bb17df5fdfb0205570ff00 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Fri, 20 Nov 2026 13:57:58 +0100 Subject: [PATCH 17/17] acc: the secret out-of-band deploy must fail, so use musterr (review) Co-authored-by: Isaac --- .../resources/secrets/parent_rename_out_of_band/output.txt | 4 +--- .../bundle/resources/secrets/parent_rename_out_of_band/script | 2 +- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/acceptance/bundle/resources/secrets/parent_rename_out_of_band/output.txt b/acceptance/bundle/resources/secrets/parent_rename_out_of_band/output.txt index ce8e31c8c36..624b624468f 100644 --- a/acceptance/bundle/resources/secrets/parent_rename_out_of_band/output.txt +++ b/acceptance/bundle/resources/secrets/parent_rename_out_of_band/output.txt @@ -26,7 +26,7 @@ Plan: 1 to add, 0 to change, 0 to delete, 1 unchanged === The create fails: the secret is already there ->>> errcode [CLI] bundle deploy +>>> musterr [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... Error: cannot create resources.secrets.secret1: Secret 'main.test-schema-renamed-[UNIQUE_NAME].mysecret' already exists (409 ALREADY_EXISTS) @@ -37,8 +37,6 @@ API message: Secret 'main.test-schema-renamed-[UNIQUE_NAME].mysecret' already ex Files: 3 uploaded, 0 deleted -Exit code: 1 - >>> print_requests.py //unity-catalog { "method": "POST", diff --git a/acceptance/bundle/resources/secrets/parent_rename_out_of_band/script b/acceptance/bundle/resources/secrets/parent_rename_out_of_band/script index 3611c7066e4..79d15f33811 100644 --- a/acceptance/bundle/resources/secrets/parent_rename_out_of_band/script +++ b/acceptance/bundle/resources/secrets/parent_rename_out_of_band/script @@ -25,6 +25,6 @@ trace update_file.py databricks.yml test-schema- test-schema-renamed- trace $CLI bundle plan title "The create fails: the secret is already there" -trace errcode $CLI bundle deploy +trace musterr $CLI bundle deploy trace print_requests.py //unity-catalog trace $CLI secrets-uc get-secret "$NEW_SCHEMA.mysecret" | jq '{full_name}'