diff --git a/.nextchanges/bundles/parent-id-fields.md b/.nextchanges/bundles/parent-id-fields.md new file mode 100644 index 00000000000..054d31bf8e5 --- /dev/null +++ b/.nextchanges/bundles/parent-id-fields.md @@ -0,0 +1 @@ +* Delete the old resource when its name follows a renamed schema. ([#7718](https://github.com/databricks/cli/pull/7718)) diff --git a/acceptance/bundle/resources/schemas/rename_referenced_by_name/databricks.yml.tmpl b/acceptance/bundle/resources/schemas/rename_referenced_by_name/databricks.yml.tmpl new file mode 100644 index 00000000000..730c34678c5 --- /dev/null +++ b/acceptance/bundle/resources/schemas/rename_referenced_by_name/databricks.yml.tmpl @@ -0,0 +1,27 @@ +bundle: + name: test-bundle-$UNIQUE_NAME + +variables: + secret_value: + description: The secret's value + +resources: + schemas: + app: + catalog_name: main + name: test_app_$UNIQUE_NAME + shared: + catalog_name: main + name: test_shared_$UNIQUE_NAME + # Both live in the shared schema and take their own name from the app schema's name. + registered_models: + model1: + catalog_name: main + schema_name: ${resources.schemas.shared.name} + name: ${resources.schemas.app.name} + secrets: + secret1: + catalog_name: main + schema_name: ${resources.schemas.shared.name} + name: ${resources.schemas.app.name} + value: ${var.secret_value} diff --git a/acceptance/bundle/resources/schemas/rename_referenced_by_name/out.test.toml b/acceptance/bundle/resources/schemas/rename_referenced_by_name/out.test.toml new file mode 100644 index 00000000000..0f167effef1 --- /dev/null +++ b/acceptance/bundle/resources/schemas/rename_referenced_by_name/out.test.toml @@ -0,0 +1,2 @@ +Cloud = true +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/resources/schemas/rename_referenced_by_name/output.txt b/acceptance/bundle/resources/schemas/rename_referenced_by_name/output.txt new file mode 100644 index 00000000000..7499d0dff65 --- /dev/null +++ b/acceptance/bundle/resources/schemas/rename_referenced_by_name/output.txt @@ -0,0 +1,102 @@ + +=== Deploy two schemas, and a model and a secret in the shared one, named after the app schema + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... +Created registered_models.model1 +Created schemas.app +Created schemas.shared +Created secrets.secret1 +Files: 2 uploaded, 0 deleted +Resources: 4 created, 0 changed, 0 deleted, 0 unchanged + +=== Rename the app schema. The model and the secret stay in the shared schema, so only their own name changes + +>>> update_file.py databricks.yml test_app_ test_app_renamed_ + +>>> [CLI] bundle plan +recreate registered_models.model1 +update schemas.app +recreate secrets.secret1 + +Plan: 2 to add, 1 to change, 2 to delete, 1 unchanged + +>>> [CLI] bundle deploy --auto-approve +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... +Recreated registered_models.model1 +Recreated secrets.secret1 +Updated schemas.app +Files: 2 uploaded, 0 deleted +Resources: 2 created, 1 changed, 2 deleted, 1 unchanged + +>>> print_requests.py --sort //unity-catalog +{ + "method": "DELETE", + "path": "/api/2.1/unity-catalog/models/main.test_shared_[UNIQUE_NAME].test_app_[UNIQUE_NAME]" +} +{ + "method": "DELETE", + "path": "/api/2.1/unity-catalog/secrets/main.test_shared_[UNIQUE_NAME].test_app_[UNIQUE_NAME]" +} +{ + "method": "PATCH", + "path": "/api/2.1/unity-catalog/schemas/main.test_app_[UNIQUE_NAME]", + "body": { + "comment": "", + "custom_max_retention_hours": 0, + "new_name": "test_app_renamed_[UNIQUE_NAME]" + } +} +{ + "method": "POST", + "path": "/api/2.1/unity-catalog/models", + "body": { + "catalog_name": "main", + "name": "test_app_renamed_[UNIQUE_NAME]", + "schema_name": "test_shared_[UNIQUE_NAME]" + } +} +{ + "method": "POST", + "path": "/api/2.1/unity-catalog/secrets", + "body": { + "catalog_name": "main", + "name": "test_app_renamed_[UNIQUE_NAME]", + "schema_name": "test_shared_[UNIQUE_NAME]", + "value": "secret-value" + } +} + +=== The old model and secret are gone; the new ones are in the shared schema +old model is gone +old secret is gone + +>>> [CLI] registered-models get main.test_shared_[UNIQUE_NAME].test_app_renamed_[UNIQUE_NAME] +{ + "full_name": "main.test_shared_[UNIQUE_NAME].test_app_renamed_[UNIQUE_NAME]" +} + +>>> [CLI] secrets-uc get-secret main.test_shared_[UNIQUE_NAME].test_app_renamed_[UNIQUE_NAME] +{ + "full_name": "main.test_shared_[UNIQUE_NAME].test_app_renamed_[UNIQUE_NAME]" +} + +=== Redeploy is a no-op + +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 4 unchanged + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.registered_models.model1 + delete resources.schemas.app + delete resources.schemas.shared + delete resources.secrets.secret1 + +This action will result in the deletion of the following UC schemas. Any underlying data may be lost: + delete resources.schemas.app + delete resources.schemas.shared + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default + +Destroy: 4 deleted diff --git a/acceptance/bundle/resources/schemas/rename_referenced_by_name/script b/acceptance/bundle/resources/schemas/rename_referenced_by_name/script new file mode 100644 index 00000000000..7cb6a142926 --- /dev/null +++ b/acceptance/bundle/resources/schemas/rename_referenced_by_name/script @@ -0,0 +1,32 @@ +envsubst < databricks.yml.tmpl > databricks.yml +export BUNDLE_VAR_secret_value=secret-value + +SHARED="main.test_shared_${UNIQUE_NAME}" + +cleanup() { + trace $CLI bundle destroy --auto-approve + + # Consume the destroy's requests so none are left recorded. + print_requests.py //unity-catalog &> LOG.destroy_requests +} +trap cleanup EXIT + +title "Deploy two schemas, and a model and a secret in the shared one, named after the app schema" +trace $CLI bundle deploy +print_requests.py //unity-catalog &> LOG.deploy_requests + +title "Rename the app schema. The model and the secret stay in the shared schema, so only their own name changes" +trace update_file.py databricks.yml test_app_ test_app_renamed_ +trace $CLI bundle plan +trace $CLI bundle deploy --auto-approve +trace print_requests.py --sort //unity-catalog + +title "The old model and secret are gone; the new ones are in the shared schema" +# Not traced: the "not found" messages differ between UC and the test server. +$CLI registered-models get "$SHARED.test_app_${UNIQUE_NAME}" &> /dev/null && echo "old model still exists" || echo "old model is gone" +$CLI secrets-uc get-secret "$SHARED.test_app_${UNIQUE_NAME}" &> /dev/null && echo "old secret still exists" || echo "old secret is gone" +trace $CLI registered-models get "$SHARED.test_app_renamed_${UNIQUE_NAME}" | jq '{full_name}' +trace $CLI secrets-uc get-secret "$SHARED.test_app_renamed_${UNIQUE_NAME}" | jq '{full_name}' + +title "Redeploy is a no-op" +trace $CLI bundle plan diff --git a/acceptance/bundle/resources/schemas/rename_referenced_by_name/test.toml b/acceptance/bundle/resources/schemas/rename_referenced_by_name/test.toml new file mode 100644 index 00000000000..4272a80cf61 --- /dev/null +++ b/acceptance/bundle/resources/schemas/rename_referenced_by_name/test.toml @@ -0,0 +1,7 @@ +Cloud = true + +Ignore = [ + ".databricks", + ".gitignore", + "databricks.yml", +] diff --git a/bundle/direct/bundle_plan.go b/bundle/direct/bundle_plan.go index 7e7a727ccc5..1011a0791d5 100644 --- a/bundle/direct/bundle_plan.go +++ b/bundle/direct/bundle_plan.go @@ -561,7 +561,7 @@ func addPerFieldActions(ctx context.Context, adapter *dresources.Adapter, change } else if action, reason, ok := classifyIDField(cfg, path, ch); ok { ch.Action = action ch.Reason = reason - if action == deployplan.Recreate && followsParentRename(adapter, parentRenames[pathString], ch) { + if action == deployplan.Recreate && followsParentRename(adapter, path, parentRenames[pathString], ch) { ch.Action = deployplan.UpdateWithID ch.Reason = deployplan.ReasonParentRenamed } @@ -831,8 +831,9 @@ func (b *DeploymentBundle) addParentMoves(plan *deployplan.Plan) error { if err != nil { return err } - // The resource's own renamable name stays its own, even when it references the parent. - if isUpdatableIDField(adapter, path) { + // Only a field naming the parent moves with it: the resource's own name stays its own, + // even when it references the parent's. + if !isParentIDField(adapter, path) { continue } if err := structaccess.Set(state, path, rename.New); err != nil { @@ -862,12 +863,11 @@ func (b *DeploymentBundle) addParentMoves(plan *deployplan.Plan) error { return nil } -// isUpdatableIDField reports whether path is one of the resource's updatable ID fields, its own -// renamable name. -func isUpdatableIDField(adapter *dresources.Adapter, path *structpath.PathNode) bool { - _, own := findMatchingRule(path, adapter.ResourceConfig().UpdatableIDFields) - _, generated := findMatchingRule(path, adapter.GeneratedResourceConfig().UpdatableIDFields) - return own || generated +// isParentIDField reports whether path is one of the resource's parent ID fields (see +// ResourceLifecycleConfig.ParentIDFields). +func isParentIDField(adapter *dresources.Adapter, path *structpath.PathNode) bool { + _, ok := findMatchingRule(path, adapter.ResourceConfig().ParentIDFields) + return ok } // recordRefTypes records, for each pure-reference field of resourceKey, the type of the resource it @@ -1077,16 +1077,17 @@ func hasUnresolvedIDField(adapter *dresources.Adapter, sv *structvar.StructVar) return false } -// followsParentRename reports whether a change to a provided ID field only follows parentChange, -// the rename of the parent field it references (see parentRenames). UC moves the children of a -// renamed catalog or schema along with it, so recreating such a child fails with "already -// exists"; it is planned as an update_id instead, which addresses it under the new parent. That -// needs IDFromState: with no other change, the update_id only re-keys the state (StateOnly). +// followsParentRename reports whether a change to a parent ID field (see isParentIDField) only +// follows parentChange, the rename of the parent field it references (see parentRenames). UC +// moves the children of a renamed catalog or schema along with it, so recreating such a child +// fails with "already exists"; it is planned as an update_id instead, which addresses it under +// the new parent. That needs IDFromState: with no other change, the update_id only re-keys the +// state (StateOnly). // // The old values must match too: a child moved to the renamed parent from somewhere else // still has to be recreated. -func followsParentRename(adapter *dresources.Adapter, parentChange, ch *deployplan.ChangeDesc) bool { - return parentChange != nil && canFollowParentRename(adapter) && structdiff.IsEqual(ch.Old, parentChange.Old) +func followsParentRename(adapter *dresources.Adapter, path *structpath.PathNode, parentChange, ch *deployplan.ChangeDesc) bool { + return parentChange != nil && isParentIDField(adapter, path) && canFollowParentRename(adapter) && structdiff.IsEqual(ch.Old, parentChange.Old) } // canFollowParentRename reports whether the resource can follow a renamed parent with an update_id: diff --git a/bundle/direct/dresources/adapter.go b/bundle/direct/dresources/adapter.go index 898eb90dbee..21ac7a02344 100644 --- a/bundle/direct/dresources/adapter.go +++ b/bundle/direct/dresources/adapter.go @@ -438,6 +438,10 @@ func (a *Adapter) validate() error { a.internalIDPath = path } + if a.resourceConfig != nil && len(a.resourceConfig.ParentIDFields) > 0 && a.idFromState == nil { + return errors.New("parent_id_fields requires IDFromState") + } + if a.doUpdateWithID != nil { validations = append(validations, "DoUpdateWithID newState", a.doUpdateWithID.InTypes[2], stateType) // DoUpdateWithID must return (string, remoteType, error) diff --git a/bundle/direct/dresources/all_test.go b/bundle/direct/dresources/all_test.go index f80fda21d14..745b48913c7 100644 --- a/bundle/direct/dresources/all_test.go +++ b/bundle/direct/dresources/all_test.go @@ -1395,6 +1395,9 @@ func validateResourceConfig(t *testing.T, stateType, remoteType reflect.Type, cf for _, p := range cfg.UpdatableIDFields { assert.NoError(t, structaccess.ValidatePattern(stateType, p.Field), "UpdatableIDFields: %s", p.Field) } + for _, p := range cfg.ParentIDFields { + assert.NoError(t, structaccess.ValidatePattern(stateType, p.Field), "ParentIDFields: %s", p.Field) + } for _, p := range cfg.IgnoreRemoteChanges { assert.NoError(t, structaccess.ValidatePattern(stateType, p.Field), "IgnoreRemoteChanges: %s", p.Field) } diff --git a/bundle/direct/dresources/config.go b/bundle/direct/dresources/config.go index b630b7e1176..45bb8115884 100644 --- a/bundle/direct/dresources/config.go +++ b/bundle/direct/dresources/config.go @@ -129,6 +129,14 @@ type ResourceLifecycleConfig struct { // the resource by its recorded ID, so it can only be backend normalization. UpdatableIDFields []FieldRule `yaml:"updatable_id_fields,omitempty"` + // ParentIDFields: the ID fields that name the resource's parent (a volume's catalog_name and + // schema_name). The backend moves the resource along with a renamed parent, so a change to one + // of these that only follows the parent's rename re-keys the resource instead of recreating it + // at its old ID (see followsParentRename and addParentMoves). Any other field, such as a name + // that references the parent's name, is the resource's own and never follows it. Requires + // IDFromState. + ParentIDFields []FieldRule `yaml:"parent_id_fields,omitempty"` + // NormalizeSlash: string field patterns the UC API strips trailing slashes from. // A change is skipped when local and remote differ only by trailing slashes. NormalizeSlash []FieldRule `yaml:"normalize_slash,omitempty"` @@ -201,6 +209,7 @@ var empty = ResourceLifecycleConfig{ RecreateOnChanges: nil, ProvidedIDFields: nil, UpdatableIDFields: nil, + ParentIDFields: nil, NormalizeSlash: nil, IgnoreRemoteAdditions: nil, BackendDefaults: nil, diff --git a/bundle/direct/dresources/configs/registered_models.yml b/bundle/direct/dresources/configs/registered_models.yml index 03bcbcc6079..6ef0dcead00 100644 --- a/bundle/direct/dresources/configs/registered_models.yml +++ b/bundle/direct/dresources/configs/registered_models.yml @@ -28,6 +28,11 @@ provided_id_fields: reason: id_field - field: schema_name reason: id_field +parent_id_fields: + - field: catalog_name + reason: parent + - field: schema_name + reason: parent recreate_on_changes: - field: storage_location reason: immutable diff --git a/bundle/direct/dresources/configs/schemas.yml b/bundle/direct/dresources/configs/schemas.yml index 5bb8f161038..51f6c69cd83 100644 --- a/bundle/direct/dresources/configs/schemas.yml +++ b/bundle/direct/dresources/configs/schemas.yml @@ -1,6 +1,9 @@ provided_id_fields: - field: catalog_name reason: id_field +parent_id_fields: + - field: catalog_name + reason: parent updatable_id_fields: # UC lowercases identifier names; remote returns "myschema" for config "MySchema". - field: name diff --git a/bundle/direct/dresources/configs/secrets.yml b/bundle/direct/dresources/configs/secrets.yml index 22ca3d7f49f..0b5918467ed 100644 --- a/bundle/direct/dresources/configs/secrets.yml +++ b/bundle/direct/dresources/configs/secrets.yml @@ -1,3 +1,8 @@ +parent_id_fields: + - field: catalog_name + reason: parent + - field: schema_name + reason: parent sensitive_fields: - field: value - field: effective_value diff --git a/bundle/direct/dresources/configs/volumes.yml b/bundle/direct/dresources/configs/volumes.yml index 9e570a1f7c2..bc001c1b773 100644 --- a/bundle/direct/dresources/configs/volumes.yml +++ b/bundle/direct/dresources/configs/volumes.yml @@ -3,6 +3,11 @@ provided_id_fields: reason: id_field - field: schema_name reason: id_field +parent_id_fields: + - field: catalog_name + reason: parent + - field: schema_name + reason: parent recreate_on_changes: - field: storage_location reason: immutable