diff --git a/.nextchanges/bundles/secret-parent-rename.md b/.nextchanges/bundles/secret-parent-rename.md new file mode 100644 index 00000000000..0491f8af6f1 --- /dev/null +++ b/.nextchanges/bundles/secret-parent-rename.md @@ -0,0 +1 @@ +* UC secrets follow a renamed catalog or schema. ([#7381](https://github.com/databricks/cli/pull/7381)) diff --git a/acceptance/bundle/resources/schemas/rename_with_model_and_secret/output.txt b/acceptance/bundle/resources/schemas/rename_with_model_and_secret/output.txt index 18ac6b3162b..9a9d331d38e 100644 --- a/acceptance/bundle/resources/schemas/rename_with_model_and_secret/output.txt +++ b/acceptance/bundle/resources/schemas/rename_with_model_and_secret/output.txt @@ -9,28 +9,22 @@ Created secrets.secret1 Files: 2 uploaded, 0 deleted Resources: 3 created, 0 changed, 0 deleted, 0 unchanged -=== Rename the schema. UC moves the model and the secret with it; the model follows the rename, the secret cannot yet, so it is recreated where it moved +=== Rename the schema. UC moves the model and the secret with it, and both follow the rename without a request >>> update_file.py databricks.yml test-schema- test-schema-renamed- >>> [CLI] bundle plan update schemas.schema1 -recreate secrets.secret1 -Plan: 1 to add, 1 to change, 1 to delete, 1 unchanged +Plan: 0 to add, 1 to change, 0 to delete, 2 unchanged >>> [CLI] bundle deploy --auto-approve Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... -Recreated secrets.secret1 Updated schemas.schema1 Files: 2 uploaded, 0 deleted -Resources: 1 created, 1 changed, 1 deleted, 1 unchanged +Resources: 0 created, 1 changed, 0 deleted, 2 unchanged >>> print_requests.py --sort //unity-catalog -{ - "method": "DELETE", - "path": "/api/2.1/unity-catalog/secrets/main.test-schema-renamed-[UNIQUE_NAME].mysecret" -} { "method": "PATCH", "path": "/api/2.1/unity-catalog/schemas/main.test-schema-[UNIQUE_NAME]", @@ -40,16 +34,6 @@ Resources: 1 created, 1 changed, 1 deleted, 1 unchanged "new_name": "test-schema-renamed-[UNIQUE_NAME]" } } -{ - "method": "POST", - "path": "/api/2.1/unity-catalog/secrets", - "body": { - "catalog_name": "main", - "name": "mysecret", - "schema_name": "test-schema-renamed-[UNIQUE_NAME]", - "value": "secret-value" - } -} >>> [CLI] registered-models get main.test-schema-renamed-[UNIQUE_NAME].mymodel { diff --git a/acceptance/bundle/resources/schemas/rename_with_model_and_secret/script b/acceptance/bundle/resources/schemas/rename_with_model_and_secret/script index 17140a4e99b..7dbc666c82c 100644 --- a/acceptance/bundle/resources/schemas/rename_with_model_and_secret/script +++ b/acceptance/bundle/resources/schemas/rename_with_model_and_secret/script @@ -15,7 +15,7 @@ title "Deploy a schema with a registered model and a secret in it" trace $CLI bundle deploy print_requests.py //unity-catalog &> LOG.deploy_requests -title "Rename the schema. UC moves the model and the secret with it; the model follows the rename, the secret cannot yet, so it is recreated where it moved" +title "Rename the schema. UC moves the model and the secret with it, and both follow the rename without a request" trace update_file.py databricks.yml test-schema- test-schema-renamed- trace $CLI bundle plan trace $CLI bundle deploy --auto-approve diff --git a/acceptance/bundle/resources/secrets/parent_rename/databricks.yml.tmpl b/acceptance/bundle/resources/secrets/parent_rename/databricks.yml.tmpl new file mode 100644 index 00000000000..98e310a0dc4 --- /dev/null +++ b/acceptance/bundle/resources/secrets/parent_rename/databricks.yml.tmpl @@ -0,0 +1,22 @@ +bundle: + name: test-bundle-$UNIQUE_NAME + +variables: + secret_value: + description: The secret's value + +resources: + catalogs: + catalog1: + name: test_catalog_$UNIQUE_NAME + schemas: + schema1: + catalog_name: ${resources.catalogs.catalog1.name} + name: myschema + secrets: + secret1: + catalog_name: ${resources.catalogs.catalog1.name} + schema_name: ${resources.schemas.schema1.name} + name: mysecret + value: ${var.secret_value} + comment: c1 diff --git a/acceptance/bundle/resources/secrets/parent_rename/out.test.toml b/acceptance/bundle/resources/secrets/parent_rename/out.test.toml new file mode 100644 index 00000000000..7a62f726f25 --- /dev/null +++ b/acceptance/bundle/resources/secrets/parent_rename/out.test.toml @@ -0,0 +1,3 @@ +Cloud = true +EnvMatrix.DMS = ["", "true"] +EnvMatrix.READPLAN = ["", "1"] diff --git a/acceptance/bundle/resources/secrets/parent_rename/output.txt b/acceptance/bundle/resources/secrets/parent_rename/output.txt new file mode 100644 index 00000000000..6cac5f13385 --- /dev/null +++ b/acceptance/bundle/resources/secrets/parent_rename/output.txt @@ -0,0 +1,456 @@ + +=== Deploy a catalog with a schema and a secret in it + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... +Created catalogs.catalog1 +Created schemas.schema1 +Created secrets.secret1 +Files: 2 uploaded, 0 deleted +Resources: 3 created, 0 changed, 0 deleted, 0 unchanged + +>>> print_requests.py //unity-catalog +{ + "method": "POST", + "path": "/api/2.1/unity-catalog/catalogs", + "body": { + "name": "test_catalog_[UNIQUE_NAME]" + } +} +{ + "method": "POST", + "path": "/api/2.1/unity-catalog/schemas", + "body": { + "catalog_name": "test_catalog_[UNIQUE_NAME]", + "name": "myschema" + } +} +{ + "method": "POST", + "path": "/api/2.1/unity-catalog/secrets", + "body": { + "catalog_name": "test_catalog_[UNIQUE_NAME]", + "comment": "c1", + "name": "mysecret", + "schema_name": "myschema", + "value": "secret-value" + } +} + +=== Rename the schema: UC moves the secret with it, so the secret is re-keyed without a request + +>>> update_file.py databricks.yml name: myschema name: myschema2 + +>>> [CLI] bundle plan --output json +{ + "resources.catalogs.catalog1": { + "action": "skip", + "remote_state": { + "catalog_type": "MANAGED_CATALOG", + "created_by": "[USERNAME]", + "full_name": "test_catalog_[UNIQUE_NAME]", + "name": "test_catalog_[UNIQUE_NAME]", + "owner": "[USERNAME]" + } + }, + "resources.schemas.schema1": { + "depends_on": [ + { + "node": "resources.catalogs.catalog1", + "label": "${resources.catalogs.catalog1.name}" + } + ], + "action": "update_id", + "new_state": { + "value": { + "catalog_name": "test_catalog_[UNIQUE_NAME]", + "name": "myschema2" + } + }, + "remote_state": { + "catalog_name": "test_catalog_[UNIQUE_NAME]", + "catalog_type": "MANAGED_CATALOG", + "created_by": "[USERNAME]", + "full_name": "test_catalog_[UNIQUE_NAME].myschema", + "name": "myschema", + "owner": "[USERNAME]", + "schema_id": "[UUID]" + }, + "changes": { + "name": { + "action": "update_id", + "reason": "id_changes", + "old": "myschema", + "new": "myschema2", + "remote": "myschema" + }, + "schema_id": { + "action": "skip", + "reason": "internal_id", + "old": "[UUID]", + "remote": "[UUID]" + } + } + }, + "resources.secrets.secret1": { + "depends_on": [ + { + "node": "resources.catalogs.catalog1", + "label": "${resources.catalogs.catalog1.name}" + }, + { + "node": "resources.schemas.schema1", + "label": "${resources.schemas.schema1.name}" + } + ], + "action": "update_id", + "state_only": true, + "new_state": { + "value": { + "catalog_name": "test_catalog_[UNIQUE_NAME]", + "comment": "c1", + "name": "mysecret", + "schema_name": "myschema2", + "value": "secret-value" + } + }, + "remote_state": { + "catalog_name": "test_catalog_[UNIQUE_NAME]", + "comment": "c1", + "create_time": "[SECRET_CREATE_TIME]", + "created_by": "[USERNAME]", + "effective_owner": "[USERNAME]", + "effective_value": "[redacted]", + "full_name": "test_catalog_[UNIQUE_NAME].myschema.mysecret", + "name": "mysecret", + "owner": "[USERNAME]", + "schema_name": "myschema", + "update_time": "[SECRET_CREATE_TIME]", + "value": "[redacted]" + }, + "changes": { + "create_time": { + "action": "skip", + "reason": "spec:output_only", + "remote": "[SECRET_CREATE_TIME]" + }, + "created_by": { + "action": "skip", + "reason": "spec:output_only", + "remote": "[USERNAME]" + }, + "effective_owner": { + "action": "skip", + "reason": "spec:output_only", + "remote": "[USERNAME]" + }, + "effective_value": { + "action": "skip", + "reason": "spec:output_only", + "remote": "[redacted]" + }, + "full_name": { + "action": "skip", + "reason": "spec:output_only", + "remote": "test_catalog_[UNIQUE_NAME].myschema.mysecret" + }, + "owner": { + "action": "skip", + "reason": "spec:input_only", + "remote": "[USERNAME]" + }, + "schema_name": { + "action": "update_id", + "reason": "parent_renamed", + "old": "myschema", + "new": "myschema2", + "remote": "myschema" + }, + "update_time": { + "action": "skip", + "reason": "spec:output_only", + "remote": "[SECRET_CREATE_TIME]" + }, + "value": { + "action": "skip", + "reason": "custom", + "old": "", + "new": "[redacted]", + "remote": "[redacted]" + } + } + } +} + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... +Updated schemas.schema1 +Files: 1 uploaded, 0 deleted +Resources: 0 created, 1 changed, 0 deleted, 2 unchanged + +>>> print_requests.py //unity-catalog +{ + "method": "PATCH", + "path": "/api/2.1/unity-catalog/schemas/test_catalog_[UNIQUE_NAME].myschema", + "body": { + "comment": "", + "custom_max_retention_hours": 0, + "new_name": "myschema2" + } +} + +>>> [CLI] secrets-uc get-secret test_catalog_[UNIQUE_NAME].myschema2.mysecret +{ + "full_name": "test_catalog_[UNIQUE_NAME].myschema2.mysecret", + "comment": "c1", + "create_time": "[SECRET_CREATE_TIME]" +} + +=== Rename the catalog and change the secret's comment: the update reaches the secret at its new path + +>>> update_file.py databricks.yml test_catalog_ test_catalog_renamed_ + +>>> update_file.py databricks.yml comment: c1 comment: c2 + +>>> [CLI] bundle plan --output json +{ + "resources.catalogs.catalog1": { + "action": "update_id", + "new_state": { + "value": { + "name": "test_catalog_renamed_[UNIQUE_NAME]" + } + }, + "remote_state": { + "catalog_type": "MANAGED_CATALOG", + "created_by": "[USERNAME]", + "full_name": "test_catalog_[UNIQUE_NAME]", + "name": "test_catalog_[UNIQUE_NAME]", + "owner": "[USERNAME]" + }, + "changes": { + "name": { + "action": "update_id", + "reason": "id_changes", + "old": "test_catalog_[UNIQUE_NAME]", + "new": "test_catalog_renamed_[UNIQUE_NAME]", + "remote": "test_catalog_[UNIQUE_NAME]" + } + } + }, + "resources.schemas.schema1": { + "depends_on": [ + { + "node": "resources.catalogs.catalog1", + "label": "${resources.catalogs.catalog1.name}" + } + ], + "action": "update_id", + "state_only": true, + "new_state": { + "value": { + "catalog_name": "test_catalog_renamed_[UNIQUE_NAME]", + "name": "myschema2" + } + }, + "remote_state": { + "catalog_name": "test_catalog_[UNIQUE_NAME]", + "catalog_type": "MANAGED_CATALOG", + "comment": "", + "created_by": "[USERNAME]", + "custom_max_retention_hours": 0, + "full_name": "test_catalog_[UNIQUE_NAME].myschema2", + "name": "myschema2", + "owner": "[USERNAME]", + "schema_id": "[UUID]" + }, + "changes": { + "catalog_name": { + "action": "update_id", + "reason": "parent_renamed", + "old": "test_catalog_[UNIQUE_NAME]", + "new": "test_catalog_renamed_[UNIQUE_NAME]", + "remote": "test_catalog_[UNIQUE_NAME]" + }, + "comment": { + "action": "skip", + "reason": "empty", + "remote": "" + }, + "custom_max_retention_hours": { + "action": "skip", + "reason": "empty", + "remote": 0 + }, + "schema_id": { + "action": "skip", + "reason": "internal_id", + "old": "[UUID]", + "remote": "[UUID]" + } + } + }, + "resources.secrets.secret1": { + "depends_on": [ + { + "node": "resources.catalogs.catalog1", + "label": "${resources.catalogs.catalog1.name}" + }, + { + "node": "resources.schemas.schema1", + "label": "${resources.schemas.schema1.name}" + } + ], + "action": "update_id", + "new_state": { + "value": { + "catalog_name": "test_catalog_renamed_[UNIQUE_NAME]", + "comment": "c2", + "name": "mysecret", + "schema_name": "myschema2", + "value": "secret-value" + } + }, + "remote_state": { + "catalog_name": "test_catalog_[UNIQUE_NAME]", + "comment": "c1", + "create_time": "[SECRET_CREATE_TIME]", + "created_by": "[USERNAME]", + "effective_owner": "[USERNAME]", + "effective_value": "[redacted]", + "full_name": "test_catalog_[UNIQUE_NAME].myschema2.mysecret", + "name": "mysecret", + "owner": "[USERNAME]", + "schema_name": "myschema2", + "update_time": "[SECRET_CREATE_TIME]", + "value": "[redacted]" + }, + "changes": { + "catalog_name": { + "action": "update_id", + "reason": "parent_renamed", + "old": "test_catalog_[UNIQUE_NAME]", + "new": "test_catalog_renamed_[UNIQUE_NAME]", + "remote": "test_catalog_[UNIQUE_NAME]" + }, + "comment": { + "action": "update", + "old": "c1", + "new": "c2", + "remote": "c1" + }, + "create_time": { + "action": "skip", + "reason": "spec:output_only", + "remote": "[SECRET_CREATE_TIME]" + }, + "created_by": { + "action": "skip", + "reason": "spec:output_only", + "remote": "[USERNAME]" + }, + "effective_owner": { + "action": "skip", + "reason": "spec:output_only", + "remote": "[USERNAME]" + }, + "effective_value": { + "action": "skip", + "reason": "spec:output_only", + "remote": "[redacted]" + }, + "full_name": { + "action": "skip", + "reason": "spec:output_only", + "remote": "test_catalog_[UNIQUE_NAME].myschema2.mysecret" + }, + "owner": { + "action": "skip", + "reason": "spec:input_only", + "remote": "[USERNAME]" + }, + "update_time": { + "action": "skip", + "reason": "spec:output_only", + "remote": "[SECRET_CREATE_TIME]" + }, + "value": { + "action": "skip", + "reason": "custom", + "old": "", + "new": "[redacted]", + "remote": "[redacted]" + } + } + } +} + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... +Updated catalogs.catalog1 +Updated secrets.secret1 +Files: 1 uploaded, 0 deleted +Resources: 0 created, 2 changed, 0 deleted, 1 unchanged + +>>> print_requests.py //unity-catalog +{ + "method": "PATCH", + "path": "/api/2.1/unity-catalog/catalogs/test_catalog_[UNIQUE_NAME]", + "body": { + "comment": "", + "custom_max_retention_hours": 0, + "new_name": "test_catalog_renamed_[UNIQUE_NAME]" + } +} +{ + "method": "PATCH", + "path": "/api/2.1/unity-catalog/secrets/test_catalog_renamed_[UNIQUE_NAME].myschema2.mysecret", + "q": { + "update_mask": "*" + }, + "body": { + "catalog_name": "test_catalog_renamed_[UNIQUE_NAME]", + "comment": "c2", + "name": "mysecret", + "schema_name": "myschema2", + "value": "secret-value" + } +} + +>>> [CLI] secrets-uc get-secret test_catalog_renamed_[UNIQUE_NAME].myschema2.mysecret +{ + "full_name": "test_catalog_renamed_[UNIQUE_NAME].myschema2.mysecret", + "comment": "c2", + "create_time": "[SECRET_CREATE_TIME]" +} + +=== The state tracks the new path and the plan is a no-op + +>>> [CLI] bundle summary --output json +{ + "catalogs": { + "catalog1": "test_catalog_renamed_[UNIQUE_NAME]" + }, + "schemas": { + "schema1": "test_catalog_renamed_[UNIQUE_NAME].myschema2" + }, + "secrets": { + "secret1": "test_catalog_renamed_[UNIQUE_NAME].myschema2.mysecret" + } +} + +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 3 unchanged + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.catalogs.catalog1 + delete resources.schemas.schema1 + delete resources.secrets.secret1 + +This action will result in the deletion of the following UC schemas. Any underlying data may be lost: + delete resources.schemas.schema1 + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default + +Destroy: 3 deleted diff --git a/acceptance/bundle/resources/secrets/parent_rename/script b/acceptance/bundle/resources/secrets/parent_rename/script new file mode 100644 index 00000000000..efeb9c91c31 --- /dev/null +++ b/acceptance/bundle/resources/secrets/parent_rename/script @@ -0,0 +1,39 @@ +envsubst < databricks.yml.tmpl > databricks.yml +export BUNDLE_VAR_secret_value=secret-value + +cleanup() { + trace $CLI bundle destroy --auto-approve + + # Consume the destroy's requests so none are left recorded. + print_requests.py //unity-catalog &> LOG.destroy_requests +} +trap cleanup EXIT + +# Drop the remote_state fields that differ between clouds or from the test server. +plan() { + trace $CLI bundle plan --output json | tee .databricks/plan.json | jq .plan | normalize_uc_payload.py created_at updated_at updated_by metastore_id browse_only effective_predictive_optimization_flag enable_predictive_optimization isolation_mode securable_type +} + +title "Deploy a catalog with a schema and a secret in it" +trace $CLI bundle deploy +trace print_requests.py //unity-catalog +add_repl "$($CLI secrets-uc get-secret "test_catalog_${UNIQUE_NAME}.myschema.mysecret" | jq -r .create_time)" SECRET_CREATE_TIME + +title "Rename the schema: UC moves the secret with it, so the secret is re-keyed without a request" +trace update_file.py databricks.yml "name: myschema" "name: myschema2" +plan +trace $CLI bundle deploy $(readplanarg .databricks/plan.json) +trace print_requests.py //unity-catalog +trace $CLI secrets-uc get-secret "test_catalog_${UNIQUE_NAME}.myschema2.mysecret" | jq '{full_name, comment, create_time}' + +title "Rename the catalog and change the secret's comment: the update reaches the secret at its new path" +trace update_file.py databricks.yml test_catalog_ test_catalog_renamed_ +trace update_file.py databricks.yml "comment: c1" "comment: c2" +plan +trace $CLI bundle deploy $(readplanarg .databricks/plan.json) +trace print_requests.py //unity-catalog +trace $CLI secrets-uc get-secret "test_catalog_renamed_${UNIQUE_NAME}.myschema2.mysecret" | jq '{full_name, comment, create_time}' + +title "The state tracks the new path and the plan is a no-op" +trace $CLI bundle summary --output json | jq '.resources | map_values(map_values(.id))' +trace $CLI bundle plan diff --git a/acceptance/bundle/resources/secrets/parent_rename/test.toml b/acceptance/bundle/resources/secrets/parent_rename/test.toml new file mode 100644 index 00000000000..ebd892fd9df --- /dev/null +++ b/acceptance/bundle/resources/secrets/parent_rename/test.toml @@ -0,0 +1,14 @@ +Cloud = true +EnvMatrix.READPLAN = ["", "1"] + +Ignore = [ + ".databricks", + ".gitignore", + "databricks.yml", +] + +# When READPLAN=1, "bundle deploy" is called as "bundle deploy --plan .databricks/plan.json". +# Normalize so both variants produce identical output. +[[Repls]] +Old = ' --plan .databricks/plan.json' +New = '' diff --git a/acceptance/bundle/resources/secrets/parent_rename_out_of_band/databricks.yml.tmpl b/acceptance/bundle/resources/secrets/parent_rename_out_of_band/databricks.yml.tmpl new file mode 100644 index 00000000000..ac557131422 --- /dev/null +++ b/acceptance/bundle/resources/secrets/parent_rename_out_of_band/databricks.yml.tmpl @@ -0,0 +1,18 @@ +bundle: + name: test-bundle-$UNIQUE_NAME + +variables: + secret_value: + description: The secret's value + +resources: + schemas: + schema1: + catalog_name: main + name: test-schema-$UNIQUE_NAME + secrets: + secret1: + catalog_name: main + schema_name: ${resources.schemas.schema1.name} + name: mysecret + value: ${var.secret_value} diff --git a/acceptance/bundle/resources/secrets/parent_rename_out_of_band/out.test.toml b/acceptance/bundle/resources/secrets/parent_rename_out_of_band/out.test.toml new file mode 100644 index 00000000000..0f167effef1 --- /dev/null +++ b/acceptance/bundle/resources/secrets/parent_rename_out_of_band/out.test.toml @@ -0,0 +1,2 @@ +Cloud = true +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/resources/secrets/parent_rename_out_of_band/output.txt b/acceptance/bundle/resources/secrets/parent_rename_out_of_band/output.txt new file mode 100644 index 00000000000..624b624468f --- /dev/null +++ b/acceptance/bundle/resources/secrets/parent_rename_out_of_band/output.txt @@ -0,0 +1,66 @@ + +=== Deploy a schema with a secret in it + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... +Created schemas.schema1 +Created secrets.secret1 +Files: 2 uploaded, 0 deleted +Resources: 2 created, 0 changed, 0 deleted, 0 unchanged + +=== Rename the schema out of band. UC moves the secret with it + +>>> [CLI] schemas update main.test-schema-[UNIQUE_NAME] --new-name test-schema-renamed-[UNIQUE_NAME] +{ + "full_name": "main.test-schema-renamed-[UNIQUE_NAME]" +} + +=== With the config renamed to match, the plan finds the schema by its schema_id. A secret has no such id, so the plan creates it again + +>>> update_file.py databricks.yml test-schema- test-schema-renamed- + +>>> [CLI] bundle plan +create secrets.secret1 + +Plan: 1 to add, 0 to change, 0 to delete, 1 unchanged + +=== The create fails: the secret is already there + +>>> musterr [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... +Error: cannot create resources.secrets.secret1: Secret 'main.test-schema-renamed-[UNIQUE_NAME].mysecret' already exists (409 ALREADY_EXISTS) + +Endpoint: POST [DATABRICKS_URL]/api/2.1/unity-catalog/secrets +HTTP Status: 409 Conflict +API error_code: ALREADY_EXISTS +API message: Secret 'main.test-schema-renamed-[UNIQUE_NAME].mysecret' already exists + +Files: 3 uploaded, 0 deleted + +>>> print_requests.py //unity-catalog +{ + "method": "POST", + "path": "/api/2.1/unity-catalog/secrets", + "body": { + "catalog_name": "main", + "name": "mysecret", + "schema_name": "test-schema-renamed-[UNIQUE_NAME]", + "value": "secret-value" + } +} + +>>> [CLI] secrets-uc get-secret main.test-schema-renamed-[UNIQUE_NAME].mysecret +{ + "full_name": "main.test-schema-renamed-[UNIQUE_NAME].mysecret" +} + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.schemas.schema1 + +This action will result in the deletion of the following UC schemas. Any underlying data may be lost: + delete resources.schemas.schema1 + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default + +Destroy: 1 deleted diff --git a/acceptance/bundle/resources/secrets/parent_rename_out_of_band/script b/acceptance/bundle/resources/secrets/parent_rename_out_of_band/script new file mode 100644 index 00000000000..79d15f33811 --- /dev/null +++ b/acceptance/bundle/resources/secrets/parent_rename_out_of_band/script @@ -0,0 +1,30 @@ +envsubst < databricks.yml.tmpl > databricks.yml +export BUNDLE_VAR_secret_value=secret-value + +OLD_SCHEMA="main.test-schema-${UNIQUE_NAME}" +NEW_SCHEMA="main.test-schema-renamed-${UNIQUE_NAME}" + +cleanup() { + trace $CLI bundle destroy --auto-approve + + # Consume the destroy's requests so none are left recorded. + print_requests.py //unity-catalog &> LOG.destroy_requests +} +trap cleanup EXIT + +title "Deploy a schema with a secret in it" +trace $CLI bundle deploy +print_requests.py //unity-catalog &> LOG.deploy_requests + +title "Rename the schema out of band. UC moves the secret with it" +trace $CLI schemas update "$OLD_SCHEMA" --new-name "test-schema-renamed-${UNIQUE_NAME}" | jq '{full_name}' +print_requests.py //unity-catalog &> LOG.rename_requests + +title "With the config renamed to match, the plan finds the schema by its schema_id. A secret has no such id, so the plan creates it again" +trace update_file.py databricks.yml test-schema- test-schema-renamed- +trace $CLI bundle plan + +title "The create fails: the secret is already there" +trace musterr $CLI bundle deploy +trace print_requests.py //unity-catalog +trace $CLI secrets-uc get-secret "$NEW_SCHEMA.mysecret" | jq '{full_name}' diff --git a/acceptance/bundle/resources/secrets/parent_rename_out_of_band/test.toml b/acceptance/bundle/resources/secrets/parent_rename_out_of_band/test.toml new file mode 100644 index 00000000000..b1af5f43017 --- /dev/null +++ b/acceptance/bundle/resources/secrets/parent_rename_out_of_band/test.toml @@ -0,0 +1,8 @@ +Cloud = true +Badness = "The plan does not look for a secret whose schema was renamed outside the bundle (a secret has no recorded id to find it by), so it creates it again: the create fails with ALREADY_EXISTS on every deploy. Same on main." + +Ignore = [ + ".databricks", + ".gitignore", + "databricks.yml", +] diff --git a/acceptance/bundle/resources/secrets/recreate_in_renamed_schema/databricks.yml.tmpl b/acceptance/bundle/resources/secrets/recreate_in_renamed_schema/databricks.yml.tmpl new file mode 100644 index 00000000000..30a41a92d18 --- /dev/null +++ b/acceptance/bundle/resources/secrets/recreate_in_renamed_schema/databricks.yml.tmpl @@ -0,0 +1,21 @@ +bundle: + name: test-bundle-$UNIQUE_NAME + +variables: + secret_value: + description: The secret's value + +resources: + catalogs: + catalog1: + name: test_catalog_$UNIQUE_NAME + schemas: + schema1: + catalog_name: ${resources.catalogs.catalog1.name} + name: myschema + secrets: + secret1: + catalog_name: ${resources.catalogs.catalog1.name} + schema_name: ${resources.schemas.schema1.name} + name: mysecret + value: ${var.secret_value} diff --git a/acceptance/bundle/resources/secrets/recreate_in_renamed_schema/out.test.toml b/acceptance/bundle/resources/secrets/recreate_in_renamed_schema/out.test.toml new file mode 100644 index 00000000000..7a62f726f25 --- /dev/null +++ b/acceptance/bundle/resources/secrets/recreate_in_renamed_schema/out.test.toml @@ -0,0 +1,3 @@ +Cloud = true +EnvMatrix.DMS = ["", "true"] +EnvMatrix.READPLAN = ["", "1"] diff --git a/acceptance/bundle/resources/secrets/recreate_in_renamed_schema/output.txt b/acceptance/bundle/resources/secrets/recreate_in_renamed_schema/output.txt new file mode 100644 index 00000000000..c292b659090 --- /dev/null +++ b/acceptance/bundle/resources/secrets/recreate_in_renamed_schema/output.txt @@ -0,0 +1,224 @@ + +=== Deploy a catalog with a schema and a secret in it + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... +Created catalogs.catalog1 +Created schemas.schema1 +Created secrets.secret1 +Files: 2 uploaded, 0 deleted +Resources: 3 created, 0 changed, 0 deleted, 0 unchanged + +=== Rename the schema and the secret in one deploy. UC cannot rename a secret, so the secret is recreated + +>>> update_file.py databricks.yml name: myschema name: myschema2 + +>>> update_file.py databricks.yml name: mysecret name: mysecret2 + +>>> [CLI] bundle plan --output json +{ + "resources.catalogs.catalog1": { + "action": "skip", + "remote_state": { + "catalog_type": "MANAGED_CATALOG", + "created_by": "[USERNAME]", + "full_name": "test_catalog_[UNIQUE_NAME]", + "name": "test_catalog_[UNIQUE_NAME]", + "owner": "[USERNAME]" + } + }, + "resources.schemas.schema1": { + "depends_on": [ + { + "node": "resources.catalogs.catalog1", + "label": "${resources.catalogs.catalog1.name}" + } + ], + "action": "update_id", + "new_state": { + "value": { + "catalog_name": "test_catalog_[UNIQUE_NAME]", + "name": "myschema2" + } + }, + "remote_state": { + "catalog_name": "test_catalog_[UNIQUE_NAME]", + "catalog_type": "MANAGED_CATALOG", + "created_by": "[USERNAME]", + "full_name": "test_catalog_[UNIQUE_NAME].myschema", + "name": "myschema", + "owner": "[USERNAME]", + "schema_id": "[UUID]" + }, + "changes": { + "name": { + "action": "update_id", + "reason": "id_changes", + "old": "myschema", + "new": "myschema2", + "remote": "myschema" + }, + "schema_id": { + "action": "skip", + "reason": "internal_id", + "old": "[UUID]", + "remote": "[UUID]" + } + } + }, + "resources.secrets.secret1": { + "id": "test_catalog_[UNIQUE_NAME].myschema2.mysecret", + "depends_on": [ + { + "node": "resources.catalogs.catalog1", + "label": "${resources.catalogs.catalog1.name}" + }, + { + "node": "resources.schemas.schema1", + "label": "${resources.schemas.schema1.name}" + } + ], + "action": "recreate", + "new_state": { + "value": { + "catalog_name": "test_catalog_[UNIQUE_NAME]", + "name": "mysecret2", + "schema_name": "myschema2", + "value": "secret-value" + } + }, + "remote_state": { + "catalog_name": "test_catalog_[UNIQUE_NAME]", + "create_time": "[OLD_SECRET_CREATE_TIME]", + "created_by": "[USERNAME]", + "effective_owner": "[USERNAME]", + "effective_value": "[redacted]", + "full_name": "test_catalog_[UNIQUE_NAME].myschema.mysecret", + "name": "mysecret", + "owner": "[USERNAME]", + "schema_name": "myschema", + "update_time": "[OLD_SECRET_CREATE_TIME]", + "value": "[redacted]" + }, + "changes": { + "create_time": { + "action": "skip", + "reason": "spec:output_only", + "remote": "[OLD_SECRET_CREATE_TIME]" + }, + "created_by": { + "action": "skip", + "reason": "spec:output_only", + "remote": "[USERNAME]" + }, + "effective_owner": { + "action": "skip", + "reason": "spec:output_only", + "remote": "[USERNAME]" + }, + "effective_value": { + "action": "skip", + "reason": "spec:output_only", + "remote": "[redacted]" + }, + "full_name": { + "action": "skip", + "reason": "spec:output_only", + "remote": "test_catalog_[UNIQUE_NAME].myschema.mysecret" + }, + "name": { + "action": "recreate", + "reason": "id_field", + "old": "mysecret", + "new": "mysecret2", + "remote": "mysecret" + }, + "owner": { + "action": "skip", + "reason": "spec:input_only", + "remote": "[USERNAME]" + }, + "schema_name": { + "action": "update_id", + "reason": "parent_renamed", + "old": "myschema", + "new": "myschema2", + "remote": "myschema" + }, + "update_time": { + "action": "skip", + "reason": "spec:output_only", + "remote": "[OLD_SECRET_CREATE_TIME]" + }, + "value": { + "action": "skip", + "reason": "custom", + "old": "", + "new": "[redacted]", + "remote": "[redacted]" + } + } + } +} + +>>> [CLI] bundle deploy --auto-approve +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... +Recreated secrets.secret1 +Updated schemas.schema1 +Files: 2 uploaded, 0 deleted +Resources: 1 created, 1 changed, 1 deleted, 1 unchanged + +>>> print_requests.py //unity-catalog +{ + "method": "PATCH", + "path": "/api/2.1/unity-catalog/schemas/test_catalog_[UNIQUE_NAME].myschema", + "body": { + "comment": "", + "custom_max_retention_hours": 0, + "new_name": "myschema2" + } +} +{ + "method": "DELETE", + "path": "/api/2.1/unity-catalog/secrets/test_catalog_[UNIQUE_NAME].myschema2.mysecret" +} +{ + "method": "POST", + "path": "/api/2.1/unity-catalog/secrets", + "body": { + "catalog_name": "test_catalog_[UNIQUE_NAME]", + "name": "mysecret2", + "schema_name": "myschema2", + "value": "secret-value" + } +} + +=== The new secret is in the renamed schema + +>>> [CLI] secrets-uc get-secret test_catalog_[UNIQUE_NAME].myschema2.mysecret2 +{ + "full_name": "test_catalog_[UNIQUE_NAME].myschema2.mysecret2" +} + +=== The old secret, moved with the renamed schema, was deleted there + +>>> musterr [CLI] secrets-uc get-secret test_catalog_[UNIQUE_NAME].myschema2.mysecret +Error: Secret 'test_catalog_[UNIQUE_NAME].myschema2.mysecret' does not exist. + +=== Redeploy is a no-op + +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 3 unchanged + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.catalogs.catalog1 + delete resources.schemas.schema1 + delete resources.secrets.secret1 + +This action will result in the deletion of the following UC schemas. Any underlying data may be lost: + delete resources.schemas.schema1 + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default + +Destroy: 3 deleted diff --git a/acceptance/bundle/resources/secrets/recreate_in_renamed_schema/script b/acceptance/bundle/resources/secrets/recreate_in_renamed_schema/script new file mode 100644 index 00000000000..3c3b984a7e5 --- /dev/null +++ b/acceptance/bundle/resources/secrets/recreate_in_renamed_schema/script @@ -0,0 +1,38 @@ +envsubst < databricks.yml.tmpl > databricks.yml +export BUNDLE_VAR_secret_value=secret-value + +CATALOG="test_catalog_${UNIQUE_NAME}" + +cleanup() { + trace $CLI bundle destroy --auto-approve + + # Consume the destroy's requests so none are left recorded. + print_requests.py //unity-catalog &> LOG.destroy_requests +} +trap cleanup EXIT + +# Drop the remote_state fields that differ between clouds or from the test server. +plan() { + trace $CLI bundle plan --output json | tee .databricks/plan.json | jq .plan | normalize_uc_payload.py created_at updated_at updated_by metastore_id browse_only effective_predictive_optimization_flag enable_predictive_optimization isolation_mode securable_type +} + +title "Deploy a catalog with a schema and a secret in it" +trace $CLI bundle deploy +print_requests.py //unity-catalog &> LOG.deploy_requests +add_repl "$($CLI secrets-uc get-secret "$CATALOG.myschema.mysecret" | jq -r .create_time)" OLD_SECRET_CREATE_TIME + +title "Rename the schema and the secret in one deploy. UC cannot rename a secret, so the secret is recreated" +trace update_file.py databricks.yml "name: myschema" "name: myschema2" +trace update_file.py databricks.yml "name: mysecret" "name: mysecret2" +plan +trace $CLI bundle deploy $(readplanarg .databricks/plan.json) --auto-approve +trace print_requests.py //unity-catalog + +title "The new secret is in the renamed schema" +trace $CLI secrets-uc get-secret "$CATALOG.myschema2.mysecret2" | jq '{full_name}' + +title "The old secret, moved with the renamed schema, was deleted there" +trace musterr $CLI secrets-uc get-secret "$CATALOG.myschema2.mysecret" + +title "Redeploy is a no-op" +trace $CLI bundle plan diff --git a/acceptance/bundle/resources/secrets/recreate_in_renamed_schema/test.toml b/acceptance/bundle/resources/secrets/recreate_in_renamed_schema/test.toml new file mode 100644 index 00000000000..ebd892fd9df --- /dev/null +++ b/acceptance/bundle/resources/secrets/recreate_in_renamed_schema/test.toml @@ -0,0 +1,14 @@ +Cloud = true +EnvMatrix.READPLAN = ["", "1"] + +Ignore = [ + ".databricks", + ".gitignore", + "databricks.yml", +] + +# When READPLAN=1, "bundle deploy" is called as "bundle deploy --plan .databricks/plan.json". +# Normalize so both variants produce identical output. +[[Repls]] +Old = ' --plan .databricks/plan.json' +New = '' diff --git a/bundle/direct/bundle_plan_test.go b/bundle/direct/bundle_plan_test.go index 947f5bee96d..8645ca5d648 100644 --- a/bundle/direct/bundle_plan_test.go +++ b/bundle/direct/bundle_plan_test.go @@ -307,6 +307,13 @@ func TestFollowsParentRename(t *testing.T) { old: "s_old", expectedAction: deployplan.Recreate, }, + { + name: "own name referencing the renamed parent is recreated", + resource: "secrets", + field: "name", + old: "s_old", + expectedAction: deployplan.Recreate, + }, { name: "recreate_on_changes field is recreated", resource: "volumes", diff --git a/bundle/direct/dresources/configs/secrets.yml b/bundle/direct/dresources/configs/secrets.yml index 0b5918467ed..e5d0177a7dc 100644 --- a/bundle/direct/dresources/configs/secrets.yml +++ b/bundle/direct/dresources/configs/secrets.yml @@ -3,6 +3,15 @@ parent_id_fields: reason: parent - field: schema_name reason: parent +provided_id_fields: + # The ID is the full name. UC cannot rename a secret; a renamed catalog or schema moves it (see + # parent_id_fields). + - field: catalog_name + reason: id_field + - field: schema_name + reason: id_field + - field: name + reason: id_field sensitive_fields: - field: value - field: effective_value diff --git a/bundle/direct/dresources/secret.go b/bundle/direct/dresources/secret.go index 7cc30eec4d5..38edb0265eb 100644 --- a/bundle/direct/dresources/secret.go +++ b/bundle/direct/dresources/secret.go @@ -109,6 +109,15 @@ func (*ResourceSecret) IDFromState(state *catalog.Secret) string { return state.CatalogName + "." + state.SchemaName + "." + state.Name } +// DoUpdateWithID updates a secret whose catalog or schema was renamed: UC moved the secret with its +// parent, so it is addressed at its new ID. A secret cannot be renamed itself, so this is the only +// way its ID changes. +func (r *ResourceSecret) DoUpdateWithID(ctx context.Context, _ string, state *catalog.Secret, entry *PlanEntry) (string, *catalog.Secret, error) { + newID := r.IDFromState(state) + remote, err := r.DoUpdate(ctx, newID, state, entry) + return newID, remote, err +} + // DoDelete deletes the secret. func (r *ResourceSecret) DoDelete(ctx context.Context, id string, _ *catalog.Secret) error { return r.client.SecretsUc.DeleteSecret(ctx, catalog.DeleteSecretRequest{ diff --git a/libs/testserver/uc_secrets.go b/libs/testserver/uc_secrets.go index 28bacf1fbed..1c8e6839a67 100644 --- a/libs/testserver/uc_secrets.go +++ b/libs/testserver/uc_secrets.go @@ -32,13 +32,13 @@ func (s *FakeWorkspace) SecretsUcCreateSecret(req Request) Response { inputSecret.SchemaName, inputSecret.Name) - // Check if secret already exists + // UC answers 409 ALREADY_EXISTS (verified on AWS, Azure and GCP). if _, exists := s.UCSecrets[fullName]; exists { return Response{ StatusCode: 409, Body: map[string]any{ - "error_code": "RESOURCE_ALREADY_EXISTS", - "message": fmt.Sprintf("Secret %s already exists", fullName), + "error_code": "ALREADY_EXISTS", + "message": fmt.Sprintf("Secret '%s' already exists", fullName), }, } } @@ -94,7 +94,7 @@ func (s *FakeWorkspace) SecretsUcGetSecret(req Request) Response { StatusCode: 404, Body: map[string]any{ "error_code": "RESOURCE_DOES_NOT_EXIST", - "message": fmt.Sprintf("Secret %s not found", fullName), + "message": fmt.Sprintf("Secret '%s' does not exist.", fullName), }, } } @@ -143,7 +143,7 @@ func (s *FakeWorkspace) SecretsUcUpdateSecret(req Request) Response { StatusCode: 404, Body: map[string]any{ "error_code": "RESOURCE_DOES_NOT_EXIST", - "message": fmt.Sprintf("Secret %s not found", fullName), + "message": fmt.Sprintf("Secret '%s' does not exist.", fullName), }, } } @@ -195,7 +195,7 @@ func (s *FakeWorkspace) SecretsUcDeleteSecret(req Request) Response { StatusCode: 404, Body: map[string]any{ "error_code": "RESOURCE_DOES_NOT_EXIST", - "message": fmt.Sprintf("Secret %s not found", fullName), + "message": fmt.Sprintf("Secret '%s' does not exist.", fullName), }, } }