diff --git a/.nextchanges/bundles/record-unbind-history.md b/.nextchanges/bundles/record-unbind-history.md new file mode 100644 index 00000000000..3dcb88098d6 --- /dev/null +++ b/.nextchanges/bundles/record-unbind-history.md @@ -0,0 +1 @@ +* Record deployment history for `bundle deployment unbind` when deployment history is enabled. ([#7818](https://github.com/databricks/cli/pull/7818)) diff --git a/acceptance/bundle/deployment/bind/feature_spec/out.unbind.recorded.txt b/acceptance/bundle/deployment/bind/feature_spec/out.unbind.recorded.txt index aa9e81e6253..31696588f39 100644 --- a/acceptance/bundle/deployment/bind/feature_spec/out.unbind.recorded.txt +++ b/acceptance/bundle/deployment/bind/feature_spec/out.unbind.recorded.txt @@ -1,2 +1,3 @@ >>> [CLI] bundle deployment unbind spec1 +Warn: Failed to record deployment history for unbind: failed to create deployment version: operations.resource_key must have a known resource type prefix (e.g. 'jobs.', 'pipelines.'): feature_specs.spec1 diff --git a/acceptance/bundle/dms/bind-unbind/databricks.yml.tmpl b/acceptance/bundle/dms/bind-unbind/databricks.yml.tmpl index 92a70710626..417b10b4962 100644 --- a/acceptance/bundle/dms/bind-unbind/databricks.yml.tmpl +++ b/acceptance/bundle/dms/bind-unbind/databricks.yml.tmpl @@ -4,3 +4,6 @@ resources: jobs: foo: name: foo + permissions: + - group_name: users + level: CAN_VIEW diff --git a/acceptance/bundle/dms/bind-unbind/output.txt b/acceptance/bundle/dms/bind-unbind/output.txt index d6a1f7c1de0..ebae0707424 100644 --- a/acceptance/bundle/dms/bind-unbind/output.txt +++ b/acceptance/bundle/dms/bind-unbind/output.txt @@ -54,9 +54,10 @@ Run 'bundle deploy' to deploy changes to your workspace >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-bind-unbind-[UNIQUE_NAME]/default/files... +Created jobs.foo.permissions Updated jobs.foo Files: 2 uploaded, 0 deleted -Resources: 0 created, 1 changed, 0 deleted, 0 unchanged +Resources: 1 created, 1 changed, 0 deleted, 0 unchanged >>> jq {features, resource_id: .state["resources.jobs.foo"].__id__} .databricks/bundle/default/resources.json { @@ -70,6 +71,12 @@ Resources: 0 created, 1 changed, 0 deleted, 0 unchanged >>> [CLI] bundle deployment unbind foo +>>> print_requests.py --dms //versions //jobs //permissions --oneline +{"method": "POST", "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions", "q": {"version_id": "5"}, "body": {"cli_version": "[CLI_VERSION]", "operations": [{"action_type": "OPERATION_ACTION_TYPE_UPDATE", "resource_key": "jobs.foo"}, {"action_type": "OPERATION_ACTION_TYPE_UPDATE", "resource_key": "jobs.foo.permissions"}], "previous_version_id": "4", "version_type": "VERSION_TYPE_DEPLOY"}} +{"method": "PATCH", "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions/5/operations/jobs.foo", "q": {"update_mask": "state,error_message,status"}, "body": {"error_message": "", "sequence_id": 0, "status": "OPERATION_STATUS_SUCCEEDED"}} +{"method": "PATCH", "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions/5/operations/jobs.foo.permissions", "q": {"update_mask": "state,error_message,status"}, "body": {"error_message": "", "sequence_id": 0, "status": "OPERATION_STATUS_SUCCEEDED"}} +{"method": "POST", "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions/5/complete", "body": {"completion_reason": "VERSION_COMPLETE_SUCCESS"}} + >>> jq {features, state} .databricks/bundle/default/resources.json { "features": { @@ -83,6 +90,15 @@ Resources: 0 created, 1 changed, 0 deleted, 0 unchanged "job_id": [JOB_ID] } +>>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-resources deployments/[DEPLOYMENT_ID] -o json +[] + +=== Unbinding an already unbound resource creates no history version + +>>> [CLI] bundle deployment unbind foo + +>>> print_requests.py --dms //versions + === Rebind preserves the existing shadow history marker without configuration >>> DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY= [CLI] bundle deployment bind foo [JOB_ID] --auto-approve @@ -98,16 +114,17 @@ Run 'bundle deploy' to deploy changes to your workspace } >>> print_requests.py --dms //versions --oneline -{"method": "POST", "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions", "q": {"version_id": "7"}, "body": {"cli_version": "[CLI_VERSION]", "operations": [{"action_type": "OPERATION_ACTION_TYPE_UPDATE", "resource_key": "jobs.foo"}], "previous_version_id": "4", "version_type": "VERSION_TYPE_DEPLOY"}} -{"method": "PATCH", "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions/7/operations/jobs.foo", "q": {"update_mask": "state,error_message,resource_id,status"}, "body": {"error_message": "", "resource_id": "[JOB_ID]", "sequence_id": 0, "state": "{\"state\":{\"deployment\":{\"kind\":\"BUNDLE\",\"metadata_file_path\":\"/Workspace/Users/[USERNAME]/.bundle/dms-bind-unbind-[UNIQUE_NAME]/default/state/metadata.json\"},\"edit_mode\":\"UI_LOCKED\",\"format\":\"MULTI_TASK\",\"max_concurrent_runs\":1,\"name\":\"foo\",\"queue\":{\"enabled\":true}}}", "status": "OPERATION_STATUS_SUCCEEDED"}} -{"method": "POST", "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions/7/complete", "body": {"completion_reason": "VERSION_COMPLETE_SUCCESS"}} +{"method": "POST", "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions", "q": {"version_id": "8"}, "body": {"cli_version": "[CLI_VERSION]", "operations": [{"action_type": "OPERATION_ACTION_TYPE_UPDATE", "resource_key": "jobs.foo"}], "previous_version_id": "5", "version_type": "VERSION_TYPE_DEPLOY"}} +{"method": "PATCH", "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions/8/operations/jobs.foo", "q": {"update_mask": "state,error_message,resource_id,status"}, "body": {"error_message": "", "resource_id": "[JOB_ID]", "sequence_id": 0, "state": "{\"state\":{\"deployment\":{\"kind\":\"BUNDLE\",\"metadata_file_path\":\"/Workspace/Users/[USERNAME]/.bundle/dms-bind-unbind-[UNIQUE_NAME]/default/state/metadata.json\"},\"edit_mode\":\"UI_LOCKED\",\"format\":\"MULTI_TASK\",\"max_concurrent_runs\":1,\"name\":\"foo\",\"queue\":{\"enabled\":true}}}", "status": "OPERATION_STATUS_SUCCEEDED"}} +{"method": "POST", "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions/8/complete", "body": {"completion_reason": "VERSION_COMPLETE_SUCCESS"}} === A subsequent deploy continues managing the same job >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-bind-unbind-[UNIQUE_NAME]/default/files... +Created jobs.foo.permissions Files: 0 uploaded, 0 deleted -Resources: 0 created, 0 changed, 0 deleted, 1 unchanged +Resources: 1 created, 0 changed, 0 deleted, 1 unchanged >>> jq {features, resource_id: .state["resources.jobs.foo"].__id__} .databricks/bundle/default/resources.json { diff --git a/acceptance/bundle/dms/bind-unbind/script b/acceptance/bundle/dms/bind-unbind/script index 5d101b99f27..1f42f47ce54 100644 --- a/acceptance/bundle/dms/bind-unbind/script +++ b/acceptance/bundle/dms/bind-unbind/script @@ -27,13 +27,22 @@ title "Deploy keeps the bound job ID with recorded history" trace $CLI bundle deploy trace jq '{features, resource_id: .state["resources.jobs.foo"].__id__}' .databricks/bundle/default/resources.json +rm -f "$OUT_REQUESTS" + title "Unbind preserves the shadow history marker and leaves the job intact" trace $CLI bundle deployment unbind foo +trace print_requests.py --dms //versions //jobs //permissions --oneline trace jq '{features, state}' .databricks/bundle/default/resources.json trace $CLI jobs get "$job_id" | jq '{job_id}' +trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-resources "deployments/${deployment_id}" -o json | jq -S 'map({resource_key, resource_id, last_action_type})' + rm -f "$OUT_REQUESTS" +title "Unbinding an already unbound resource creates no history version" +trace $CLI bundle deployment unbind foo +trace print_requests.py --dms //versions + title "Rebind preserves the existing shadow history marker without configuration" trace DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY= $CLI bundle deployment bind foo "$job_id" --auto-approve trace jq '{features, resource_id: .state["resources.jobs.foo"].__id__}' .databricks/bundle/default/resources.json diff --git a/acceptance/bundle/dms/suppress-transport-warnings/output.txt b/acceptance/bundle/dms/suppress-transport-warnings/output.txt index 7e4d5803615..9f13235b190 100644 --- a/acceptance/bundle/dms/suppress-transport-warnings/output.txt +++ b/acceptance/bundle/dms/suppress-transport-warnings/output.txt @@ -3,3 +3,4 @@ Warn: Could not enable deployment history for this deploy: failed to create depl Warn: Failed to complete deployment history version: Internal error Error: cannot create resources.secrets.foo: response body: stream error: stream ID 73; INTERNAL_ERROR; received from peer Warn: Failed to record deployment history for bind: operations.resource_key must have a known resource type +Warn: Failed to record deployment history for unbind: operations.resource_key must have a known resource type diff --git a/acceptance/bundle/dms/suppress-transport-warnings/script b/acceptance/bundle/dms/suppress-transport-warnings/script index 4eabace7c59..b4b994ac193 100644 --- a/acceptance/bundle/dms/suppress-transport-warnings/script +++ b/acceptance/bundle/dms/suppress-transport-warnings/script @@ -8,6 +8,7 @@ echo "Warn: Could not delete deployment history: response body: stream error: st echo "Warn: Failed to record deployment history: context deadline exceeded" echo "Warn: Failed to record deployment history for bind: unexpected EOF" +echo "Warn: Failed to record deployment history for unbind: context deadline exceeded" # Kept: errors the service returns, and transport errors outside deployment history. echo "Warn: Could not enable deployment history for this deploy: failed to create deployment: " @@ -15,3 +16,4 @@ echo "Warn: Could not enable deployment history for this deploy: failed to creat echo "Warn: Failed to complete deployment history version: Internal error" echo "Error: cannot create resources.secrets.foo: response body: stream error: stream ID 73; INTERNAL_ERROR; received from peer" echo "Warn: Failed to record deployment history for bind: operations.resource_key must have a known resource type" +echo "Warn: Failed to record deployment history for unbind: operations.resource_key must have a known resource type" diff --git a/acceptance/bundle/dms/unbind-upload-failure/databricks.yml.tmpl b/acceptance/bundle/dms/unbind-upload-failure/databricks.yml.tmpl new file mode 100644 index 00000000000..fd2636e25a5 --- /dev/null +++ b/acceptance/bundle/dms/unbind-upload-failure/databricks.yml.tmpl @@ -0,0 +1,7 @@ +bundle: + name: dms-unbind-upload-failure-$UNIQUE_NAME + +resources: + jobs: + upload_failed: + name: upload_failed diff --git a/acceptance/bundle/dms/unbind-upload-failure/out.test.toml b/acceptance/bundle/dms/unbind-upload-failure/out.test.toml new file mode 100644 index 00000000000..88e6fbd847c --- /dev/null +++ b/acceptance/bundle/dms/unbind-upload-failure/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DMS = ["true"] diff --git a/acceptance/bundle/dms/unbind-upload-failure/output.txt b/acceptance/bundle/dms/unbind-upload-failure/output.txt new file mode 100644 index 00000000000..01ed6edea7b --- /dev/null +++ b/acceptance/bundle/dms/unbind-upload-failure/output.txt @@ -0,0 +1,23 @@ + +=== A remote state upload failure fails the unbind version + +>>> fault.py POST /api/2.0/workspace-files/import-file/**resources.json 400 0 1 + +>>> musterr [CLI] bundle deployment unbind upload_failed +Error: Fault injected by test. (400 INJECTED) + +Endpoint: POST [DATABRICKS_URL]/api/2.0/workspace-files/import-file/Workspace%2FUsers%2F[USERNAME]%2F.bundle%2Fdms-unbind-upload-failure-[UNIQUE_NAME]%2Fdefault%2Fstate%2Fresources.json?overwrite=true +HTTP Status: 400 Bad Request +API error_code: INJECTED +API message: Fault injected by test. + + +>>> print_requests.py --dms //complete --oneline +{"method": "POST", "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions/4/complete", "body": {"completion_reason": "VERSION_COMPLETE_FAILURE"}} + +=== Remote state still contains the binding whose upload failed + +>>> [CLI] bundle summary -o json +{ + "upload_failed": "[JOB_ID]" +} diff --git a/acceptance/bundle/dms/unbind-upload-failure/script b/acceptance/bundle/dms/unbind-upload-failure/script new file mode 100644 index 00000000000..d7fc57c0804 --- /dev/null +++ b/acceptance/bundle/dms/unbind-upload-failure/script @@ -0,0 +1,15 @@ +envsubst < databricks.yml.tmpl > databricks.yml +job_id=$($CLI jobs create --json '{"name": "unbind-target-'$UNIQUE_NAME'"}' | jq -r '.job_id') +add_repl "$job_id" JOB_ID +$CLI bundle deployment bind upload_failed "$job_id" --auto-approve &> LOG.setup +rm -f "$OUT_REQUESTS" + +title "A remote state upload failure fails the unbind version" +trace fault.py "POST /api/2.0/workspace-files/import-file/**resources.json" 400 0 1 +trace musterr $CLI bundle deployment unbind upload_failed +trace print_requests.py --dms //complete --oneline + +rm -rf .databricks +title "Remote state still contains the binding whose upload failed" +trace $CLI bundle summary -o json | jq '.resources.jobs | map_values(.id // null)' +rm -f "$OUT_REQUESTS" diff --git a/acceptance/bundle/dms/unbind-upload-failure/test.toml b/acceptance/bundle/dms/unbind-upload-failure/test.toml new file mode 100644 index 00000000000..8ab24f1b817 --- /dev/null +++ b/acceptance/bundle/dms/unbind-upload-failure/test.toml @@ -0,0 +1 @@ +Cloud = false # Inject a remote state-upload failure. diff --git a/acceptance/bundle/dms_fault/complete_version/output.txt b/acceptance/bundle/dms_fault/complete_version/output.txt index 785d9afaafd..b67b57b006b 100644 --- a/acceptance/bundle/dms_fault/complete_version/output.txt +++ b/acceptance/bundle/dms_fault/complete_version/output.txt @@ -48,6 +48,7 @@ >>> bundle deployment unbind foo + Warn: Failed to complete deployment history version: Fault injected by test. >>> bundle generate job --existing-job-id [FOO_ID] --key foo --config-dir resources --source-dir src --force --bind Job configuration successfully saved to resources/foo.job.yml @@ -67,6 +68,7 @@ >>> bundle deployment unbind bar + Warn: Failed to complete deployment history version: Fault injected by test. >>> bundle generate pipeline --existing-pipeline-id [BAR_ID] --key bar --config-dir resources --source-dir src --force --bind File successfully saved to src/pipeline.sql diff --git a/acceptance/bundle/dms_fault/create_deployment/output.txt b/acceptance/bundle/dms_fault/create_deployment/output.txt index 7f7b781fd79..4ae0acb41f6 100644 --- a/acceptance/bundle/dms_fault/create_deployment/output.txt +++ b/acceptance/bundle/dms_fault/create_deployment/output.txt @@ -39,6 +39,7 @@ >>> bundle deployment unbind foo + Warn: Failed to record deployment history for unbind: failed to create deployment: Fault injected by test. >>> bundle generate job --existing-job-id [FOO_ID] --key foo --config-dir resources --source-dir src --force --bind Job configuration successfully saved to resources/foo.job.yml @@ -58,6 +59,7 @@ >>> bundle deployment unbind bar + Warn: Failed to record deployment history for unbind: failed to create deployment: Fault injected by test. >>> bundle generate pipeline --existing-pipeline-id [BAR_ID] --key bar --config-dir resources --source-dir src --force --bind File successfully saved to src/pipeline.sql diff --git a/acceptance/bundle/dms_fault/create_version/output.txt b/acceptance/bundle/dms_fault/create_version/output.txt index 20eb577bf16..0ef6a35c36a 100644 --- a/acceptance/bundle/dms_fault/create_version/output.txt +++ b/acceptance/bundle/dms_fault/create_version/output.txt @@ -47,6 +47,7 @@ >>> bundle deployment unbind foo + Warn: Failed to record deployment history for unbind: failed to create deployment version: Fault injected by test. >>> bundle generate job --existing-job-id [FOO_ID] --key foo --config-dir resources --source-dir src --force --bind Job configuration successfully saved to resources/foo.job.yml @@ -66,6 +67,7 @@ >>> bundle deployment unbind bar + Warn: Failed to record deployment history for unbind: failed to create deployment version: Fault injected by test. >>> bundle generate pipeline --existing-pipeline-id [BAR_ID] --key bar --config-dir resources --source-dir src --force --bind File successfully saved to src/pipeline.sql diff --git a/acceptance/bundle/dms_fault/get_deployment/output.txt b/acceptance/bundle/dms_fault/get_deployment/output.txt index 2d35346e952..44689cefc28 100644 --- a/acceptance/bundle/dms_fault/get_deployment/output.txt +++ b/acceptance/bundle/dms_fault/get_deployment/output.txt @@ -55,6 +55,9 @@ >>> bundle deployment unbind foo + Warn: retrying after 500 Internal Server Error from GET /api/2.0/bundle/deployments/[DEPLOYMENT_ID] + Warn: retrying after 500 Internal Server Error from GET /api/2.0/bundle/deployments/[DEPLOYMENT_ID] + Warn: Failed to record deployment history for unbind: Fault injected by test. >>> bundle generate job --existing-job-id [FOO_ID] --key foo --config-dir resources --source-dir src --force --bind Job configuration successfully saved to resources/foo.job.yml @@ -76,6 +79,9 @@ >>> bundle deployment unbind bar + Warn: retrying after 500 Internal Server Error from GET /api/2.0/bundle/deployments/[DEPLOYMENT_ID] + Warn: retrying after 500 Internal Server Error from GET /api/2.0/bundle/deployments/[DEPLOYMENT_ID] + Warn: Failed to record deployment history for unbind: Fault injected by test. >>> bundle generate pipeline --existing-pipeline-id [BAR_ID] --key bar --config-dir resources --source-dir src --force --bind File successfully saved to src/pipeline.sql diff --git a/acceptance/bundle/dms_fault/run.py b/acceptance/bundle/dms_fault/run.py index 52c2a0feef4..ce4eff464d9 100644 --- a/acceptance/bundle/dms_fault/run.py +++ b/acceptance/bundle/dms_fault/run.py @@ -212,6 +212,7 @@ def assert_destroyed(cwd, root, current): assert (cwd / "exported-src/pipeline.sql").read_text() == (cwd / "pipeline.sql").read_text() run(cwd, "bundle", "deployment", "unbind", key) assert resource not in state(cwd)["state"] + assert resource not in remote_state(cwd, root)["state"] generate(cwd, kind, key, current[resource], bind=True) assert ids(cwd)[resource] == current[resource] assert remote_state(cwd, root)["state"][resource]["__id__"] == current[resource] diff --git a/acceptance/bundle/dms_fault/update_operation/output.txt b/acceptance/bundle/dms_fault/update_operation/output.txt index 3db42b8ba77..0959d8794cb 100644 --- a/acceptance/bundle/dms_fault/update_operation/output.txt +++ b/acceptance/bundle/dms_fault/update_operation/output.txt @@ -48,6 +48,7 @@ >>> bundle deployment unbind foo + Warn: Failed to record deployment history for unbind: recording operation for [RESOURCE_KEY]: Fault injected by test. >>> bundle generate job --existing-job-id [FOO_ID] --key foo --config-dir resources --source-dir src --force --bind Job configuration successfully saved to resources/foo.job.yml @@ -67,6 +68,7 @@ >>> bundle deployment unbind bar + Warn: Failed to record deployment history for unbind: recording operation for [RESOURCE_KEY]: Fault injected by test. >>> bundle generate pipeline --existing-pipeline-id [BAR_ID] --key bar --config-dir resources --source-dir src --force --bind File successfully saved to src/pipeline.sql diff --git a/bundle/direct/bind.go b/bundle/direct/bind.go index 20f5095d07b..f7aaaa8275a 100644 --- a/bundle/direct/bind.go +++ b/bundle/direct/bind.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "os" + "slices" "strings" "github.com/databricks/cli/bundle/config" @@ -42,6 +43,12 @@ type DMSBindMetadata struct { ResourceState dstate.ResourceEntry } +// DMSUnbindMetadata holds removed keys to publish after the unbind commits. +type DMSUnbindMetadata struct { + Serial int + ResourceKeys []string +} + // BindResult contains the result of a bind operation including any detected changes. type BindResult struct { // DMSMetadata is nil when deployment history is disabled. @@ -269,18 +276,40 @@ func (result *BindResult) Cancel() { // Unbind removes a resource from direct engine state without deleting // the workspace resource. Also removes associated permissions/grants entries. -func (b *DeploymentBundle) Unbind(ctx context.Context, statePath, resourceKey string) error { +// It returns nil DMS metadata when deployment history is disabled. +func (b *DeploymentBundle) Unbind(ctx context.Context, configRoot *config.Root, statePath, resourceKey string) (*DMSUnbindMetadata, error) { if err := b.StateDB.Open(ctx, statePath, dstate.ReadingFile, dstate.WithRecovery(true), dstate.OpenDmsArgs{}); err != nil { - return err + return nil, err + } + var configured *bool + if configRoot.Experimental != nil { + configured = configRoot.Experimental.DeploymentHistory + } + if b.StateDB.RecordsHistory() || bundleenv.RecordsDeploymentHistory(ctx, configured) { + if b.StateDB.Data.Features == nil { + b.StateDB.Data.Features = make(map[string]struct{}) + } + b.StateDB.Data.Features[dstate.FeatureDeploymentHistoryShadow] = struct{}{} } if err := b.StateDB.UpgradeToWrite(); err != nil { - return err + return nil, err + } + var metadata *DMSUnbindMetadata + if b.StateDB.RecordsHistory() { + metadata = &DMSUnbindMetadata{Serial: b.StateDB.Data.Serial} + // Finalize commits the removals below as one existing WAL transaction. + if b.StateDB.Data.State != nil { + metadata.Serial++ + } + if _, exists := b.StateDB.GetResourceEntry(resourceKey); exists { + metadata.ResourceKeys = append(metadata.ResourceKeys, resourceKey) + } } // Delete the main resource err := b.StateDB.DeleteState(ctx, resourceKey, false) if err != nil { - return err + return nil, err } log.Infof(ctx, "Unbound %s", resourceKey) @@ -294,12 +323,18 @@ func (b *DeploymentBundle) Unbind(ctx context.Context, statePath, resourceKey st if key == permissionsKey || key == grantsKey || strings.HasPrefix(key, resourceKey+".") { err = b.StateDB.DeleteState(ctx, key, false) if err != nil { - return err + return nil, err + } + if metadata != nil { + metadata.ResourceKeys = append(metadata.ResourceKeys, key) } log.Infof(ctx, "Unbound %s", key) } } + if metadata != nil { + slices.Sort(metadata.ResourceKeys) + } _, err = b.StateDB.Finalize(ctx) - return err + return metadata, err } diff --git a/bundle/direct/bind_test.go b/bundle/direct/bind_test.go new file mode 100644 index 00000000000..7bc78a2e5e2 --- /dev/null +++ b/bundle/direct/bind_test.go @@ -0,0 +1,58 @@ +package direct_test + +import ( + "path/filepath" + "testing" + + "github.com/databricks/cli/bundle/config" + "github.com/databricks/cli/bundle/direct" + "github.com/databricks/cli/bundle/direct/dstate" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestUnbindReturnsRemovedKeys(t *testing.T) { + for _, tc := range []struct { + name string + keys []string + history bool + }{ + {"resource", []string{"resources.jobs.foo"}, true}, + {"children", []string{"resources.jobs.foo", "resources.jobs.foo.permissions", "resources.jobs.foo.grants"}, true}, + {"children_without_parent", []string{"resources.jobs.foo.permissions"}, true}, + {"missing", nil, true}, + {"history_disabled", []string{"resources.jobs.foo"}, false}, + } { + t.Run(tc.name, func(t *testing.T) { + ctx := t.Context() + statePath := filepath.Join(t.TempDir(), "resources.json") + var b direct.DeploymentBundle + require.NoError(t, b.StateDB.Open(ctx, statePath, dstate.ReadingFile, false, dstate.OpenDmsArgs{})) + require.NoError(t, b.StateDB.UpgradeToWrite()) + for _, key := range append(tc.keys, "resources.jobs.foobar") { + require.NoError(t, b.StateDB.SaveState(ctx, key, "id", map[string]string{"name": key}, nil)) + } + _, err := b.StateDB.Finalize(ctx) + require.NoError(t, err) + + root := &config.Root{Experimental: &config.Experimental{DeploymentHistory: &tc.history}} + metadata, err := b.Unbind(ctx, root, statePath, "resources.jobs.foo") + require.NoError(t, err) + if tc.history { + require.NotNil(t, metadata) + assert.ElementsMatch(t, tc.keys, metadata.ResourceKeys) + } else { + assert.Nil(t, metadata) + } + + require.NoError(t, b.StateDB.Open(ctx, statePath, dstate.ReadingFile, false, dstate.OpenDmsArgs{})) + assert.Len(t, b.StateDB.Data.State, 1) + if metadata != nil { + assert.Equal(t, b.StateDB.Data.Serial, metadata.Serial) + } + assert.Equal(t, "id", b.StateDB.GetResourceID("resources.jobs.foobar")) + _, err = b.StateDB.Finalize(ctx) + require.NoError(t, err) + }) + } +} diff --git a/bundle/phases/bind.go b/bundle/phases/bind.go index 8ea2d82b0ad..0ac53ffbb22 100644 --- a/bundle/phases/bind.go +++ b/bundle/phases/bind.go @@ -175,7 +175,10 @@ func Unbind(ctx context.Context, b *bundle.Bundle, group, resourceKey string, en return } + var history *bindingHistory + var historyErr error defer func() { + completeBindingHistory(ctx, b, history, historyErr == nil && !logdiag.HasError(ctx)) bundle.ApplyContext(ctx, b, lock.Release(lock.GoalUnbind)) }() @@ -188,12 +191,17 @@ func Unbind(ctx context.Context, b *bundle.Bundle, group, resourceKey string, en fullResourceKey := fmt.Sprintf("resources.%s.%s", group, resourceKey) _, statePath := b.StateFilenameDirect(ctx) - err := b.DeploymentBundle.Unbind(ctx, statePath, fullResourceKey) + metadata, err := b.DeploymentBundle.Unbind(ctx, &b.Config, statePath, fullResourceKey) if err != nil { logdiag.LogError(ctx, err) return } + history, historyErr = recordUnbindHistory(ctx, b, metadata) + if historyErr != nil { + log.Warnf(ctx, "Failed to record deployment history for unbind: %s", historyErr) + } + statemgmt.PushResourcesState(ctx, b) } @@ -218,6 +226,28 @@ func recordBindHistory(ctx context.Context, b *bundle.Bundle, resourceKey string return history, history.buffer.Drain(ctx) } +// recordUnbindHistory publishes committed removals when DMS metadata was requested. +func recordUnbindHistory(ctx context.Context, b *bundle.Bundle, metadata *direct.DMSUnbindMetadata) (*bindingHistory, error) { + if metadata == nil || len(metadata.ResourceKeys) == 0 { + return nil, nil + } + staged := make([]bundledeployments.StagedOperation, 0, len(metadata.ResourceKeys)) + for _, key := range metadata.ResourceKeys { + staged = append(staged, bundledeployments.StagedOperation{ + ResourceKey: strings.TrimPrefix(key, dms.StatePrefix), + ActionType: bundledeployments.OperationActionTypeOperationActionTypeUpdate, + }) + } + history, err := createBindingVersion(ctx, b, metadata.Serial, staged) + if err != nil { + return nil, err + } + for _, key := range metadata.ResourceKeys { + history.buffer.RecordOperation(ctx, key, false, "", nil) + } + return history, history.buffer.Drain(ctx) +} + // createBindingVersion claims a history version for the staged binding operations. func createBindingVersion(ctx context.Context, b *bundle.Bundle, serial int, staged []bundledeployments.StagedOperation) (*bindingHistory, error) { w := b.WorkspaceClient(ctx)