From 4833f0641916f1cbbfe66ca932ef79f35539f903 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Wed, 18 Nov 2026 12:13:36 +0000 Subject: [PATCH] Record resource unbindings through StateDB history --- .nextchanges/bundles/record-unbind-history.md | 1 + .../bind/feature_spec/out.unbind.recorded.txt | 1 + .../dms/bind-unbind/databricks.yml.tmpl | 3 + acceptance/bundle/dms/bind-unbind/output.txt | 27 +++- acceptance/bundle/dms/bind-unbind/script | 9 ++ .../databricks.yml.tmpl | 13 ++ .../dms/unbind-history-failures/out.test.toml | 2 + .../dms/unbind-history-failures/output.txt | 86 ++++++++++++ .../bundle/dms/unbind-history-failures/script | 37 +++++ .../dms/unbind-history-failures/test.toml | 1 + .../dms_fault/complete_version/output.txt | 2 + .../dms_fault/create_deployment/output.txt | 2 + .../dms_fault/create_version/output.txt | 2 + .../dms_fault/get_deployment/output.txt | 6 + .../dms_fault/update_operation/output.txt | 2 + bundle/direct/bind.go | 50 +++---- bundle/direct/bind_test.go | 46 +++++++ bundle/phases/bind.go | 64 +++++++-- bundle/phases/bind_test.go | 130 +++++++++++------- 19 files changed, 387 insertions(+), 97 deletions(-) create mode 100644 .nextchanges/bundles/record-unbind-history.md create mode 100644 acceptance/bundle/dms/unbind-history-failures/databricks.yml.tmpl create mode 100644 acceptance/bundle/dms/unbind-history-failures/out.test.toml create mode 100644 acceptance/bundle/dms/unbind-history-failures/output.txt create mode 100644 acceptance/bundle/dms/unbind-history-failures/script create mode 100644 acceptance/bundle/dms/unbind-history-failures/test.toml diff --git a/.nextchanges/bundles/record-unbind-history.md b/.nextchanges/bundles/record-unbind-history.md new file mode 100644 index 00000000000..643336c5f4f --- /dev/null +++ b/.nextchanges/bundles/record-unbind-history.md @@ -0,0 +1 @@ +* Record a deployment history version for `bundle deployment unbind` when deployment history is enabled. ([#7401](https://github.com/databricks/cli/pull/7401)) diff --git a/acceptance/bundle/deployment/bind/feature_spec/out.unbind.recorded.txt b/acceptance/bundle/deployment/bind/feature_spec/out.unbind.recorded.txt index aa9e81e6253..ad910fb1f4f 100644 --- a/acceptance/bundle/deployment/bind/feature_spec/out.unbind.recorded.txt +++ b/acceptance/bundle/deployment/bind/feature_spec/out.unbind.recorded.txt @@ -1,2 +1,3 @@ >>> [CLI] bundle deployment unbind spec1 +Warn: Failed to record unbind history: failed to create deployment version: operations.resource_key must have a known resource type prefix (e.g. 'jobs.', 'pipelines.'): feature_specs.spec1 diff --git a/acceptance/bundle/dms/bind-unbind/databricks.yml.tmpl b/acceptance/bundle/dms/bind-unbind/databricks.yml.tmpl index 92a70710626..417b10b4962 100644 --- a/acceptance/bundle/dms/bind-unbind/databricks.yml.tmpl +++ b/acceptance/bundle/dms/bind-unbind/databricks.yml.tmpl @@ -4,3 +4,6 @@ resources: jobs: foo: name: foo + permissions: + - group_name: users + level: CAN_VIEW diff --git a/acceptance/bundle/dms/bind-unbind/output.txt b/acceptance/bundle/dms/bind-unbind/output.txt index d3bffd7d8b5..1ceefd4ff71 100644 --- a/acceptance/bundle/dms/bind-unbind/output.txt +++ b/acceptance/bundle/dms/bind-unbind/output.txt @@ -54,9 +54,10 @@ Run 'bundle deploy' to deploy changes to your workspace >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-bind-unbind-[UNIQUE_NAME]/default/files... +Created jobs.foo.permissions Updated jobs.foo Files: 2 uploaded, 0 deleted -Resources: 0 created, 1 changed, 0 deleted, 0 unchanged +Resources: 1 created, 1 changed, 0 deleted, 0 unchanged >>> jq {features, resource_id: .state["resources.jobs.foo"].__id__} .databricks/bundle/default/resources.json { @@ -70,6 +71,12 @@ Resources: 0 created, 1 changed, 0 deleted, 0 unchanged >>> [CLI] bundle deployment unbind foo +>>> print_requests.py --dms //versions //jobs //permissions --oneline +{"method": "POST", "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions", "q": {"version_id": "3"}, "body": {"cli_version": "[CLI_VERSION]", "operations": [{"action_type": "OPERATION_ACTION_TYPE_UPDATE", "resource_key": "jobs.foo"}, {"action_type": "OPERATION_ACTION_TYPE_UPDATE", "resource_key": "jobs.foo.permissions"}], "previous_version_id": "2", "version_type": "VERSION_TYPE_DEPLOY"}} +{"method": "PATCH", "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions/3/operations/jobs.foo", "q": {"update_mask": "state,error_message,status"}, "body": {"error_message": "", "sequence_id": 0, "status": "OPERATION_STATUS_SUCCEEDED"}} +{"method": "PATCH", "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions/3/operations/jobs.foo.permissions", "q": {"update_mask": "state,error_message,status"}, "body": {"error_message": "", "sequence_id": 0, "status": "OPERATION_STATUS_SUCCEEDED"}} +{"method": "POST", "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions/3/complete", "body": {"completion_reason": "VERSION_COMPLETE_SUCCESS"}} + >>> jq {features, state} .databricks/bundle/default/resources.json { "features": { @@ -83,6 +90,15 @@ Resources: 0 created, 1 changed, 0 deleted, 0 unchanged "job_id": [JOB_ID] } +>>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-resources deployments/[DEPLOYMENT_ID] -o json +[] + +=== Unbinding an already unbound resource creates no history version + +>>> [CLI] bundle deployment unbind foo + +>>> print_requests.py --dms //versions + === Rebind preserves the existing shadow history marker without configuration >>> DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY= [CLI] bundle deployment bind foo [JOB_ID] --auto-approve @@ -98,16 +114,17 @@ Run 'bundle deploy' to deploy changes to your workspace } >>> print_requests.py --dms //versions --oneline -{"method": "POST", "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions", "q": {"version_id": "4"}, "body": {"cli_version": "[CLI_VERSION]", "operations": [{"action_type": "OPERATION_ACTION_TYPE_UPDATE", "resource_key": "jobs.foo"}], "previous_version_id": "2", "version_type": "VERSION_TYPE_DEPLOY"}} -{"method": "PATCH", "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions/4/operations/jobs.foo", "q": {"update_mask": "state,error_message,resource_id,status"}, "body": {"error_message": "", "resource_id": "[JOB_ID]", "sequence_id": 0, "state": "{\"state\":{\"deployment\":{\"kind\":\"BUNDLE\",\"metadata_file_path\":\"/Workspace/Users/[USERNAME]/.bundle/dms-bind-unbind-[UNIQUE_NAME]/default/state/metadata.json\"},\"edit_mode\":\"UI_LOCKED\",\"format\":\"MULTI_TASK\",\"max_concurrent_runs\":1,\"name\":\"foo\",\"queue\":{\"enabled\":true}}}", "status": "OPERATION_STATUS_SUCCEEDED"}} -{"method": "POST", "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions/4/complete", "body": {"completion_reason": "VERSION_COMPLETE_SUCCESS"}} +{"method": "POST", "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions", "q": {"version_id": "5"}, "body": {"cli_version": "[CLI_VERSION]", "operations": [{"action_type": "OPERATION_ACTION_TYPE_UPDATE", "resource_key": "jobs.foo"}], "previous_version_id": "3", "version_type": "VERSION_TYPE_DEPLOY"}} +{"method": "PATCH", "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions/5/operations/jobs.foo", "q": {"update_mask": "state,error_message,resource_id,status"}, "body": {"error_message": "", "resource_id": "[JOB_ID]", "sequence_id": 0, "state": "{\"state\":{\"deployment\":{\"kind\":\"BUNDLE\",\"metadata_file_path\":\"/Workspace/Users/[USERNAME]/.bundle/dms-bind-unbind-[UNIQUE_NAME]/default/state/metadata.json\"},\"edit_mode\":\"UI_LOCKED\",\"format\":\"MULTI_TASK\",\"max_concurrent_runs\":1,\"name\":\"foo\",\"queue\":{\"enabled\":true}}}", "status": "OPERATION_STATUS_SUCCEEDED"}} +{"method": "POST", "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions/5/complete", "body": {"completion_reason": "VERSION_COMPLETE_SUCCESS"}} === A subsequent deploy continues managing the same job >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-bind-unbind-[UNIQUE_NAME]/default/files... +Created jobs.foo.permissions Files: 0 uploaded, 0 deleted -Resources: 0 created, 0 changed, 0 deleted, 1 unchanged +Resources: 1 created, 0 changed, 0 deleted, 1 unchanged >>> jq {features, resource_id: .state["resources.jobs.foo"].__id__} .databricks/bundle/default/resources.json { diff --git a/acceptance/bundle/dms/bind-unbind/script b/acceptance/bundle/dms/bind-unbind/script index 5d101b99f27..1f42f47ce54 100644 --- a/acceptance/bundle/dms/bind-unbind/script +++ b/acceptance/bundle/dms/bind-unbind/script @@ -27,13 +27,22 @@ title "Deploy keeps the bound job ID with recorded history" trace $CLI bundle deploy trace jq '{features, resource_id: .state["resources.jobs.foo"].__id__}' .databricks/bundle/default/resources.json +rm -f "$OUT_REQUESTS" + title "Unbind preserves the shadow history marker and leaves the job intact" trace $CLI bundle deployment unbind foo +trace print_requests.py --dms //versions //jobs //permissions --oneline trace jq '{features, state}' .databricks/bundle/default/resources.json trace $CLI jobs get "$job_id" | jq '{job_id}' +trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-resources "deployments/${deployment_id}" -o json | jq -S 'map({resource_key, resource_id, last_action_type})' + rm -f "$OUT_REQUESTS" +title "Unbinding an already unbound resource creates no history version" +trace $CLI bundle deployment unbind foo +trace print_requests.py --dms //versions + title "Rebind preserves the existing shadow history marker without configuration" trace DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY= $CLI bundle deployment bind foo "$job_id" --auto-approve trace jq '{features, resource_id: .state["resources.jobs.foo"].__id__}' .databricks/bundle/default/resources.json diff --git a/acceptance/bundle/dms/unbind-history-failures/databricks.yml.tmpl b/acceptance/bundle/dms/unbind-history-failures/databricks.yml.tmpl new file mode 100644 index 00000000000..a30c578aa55 --- /dev/null +++ b/acceptance/bundle/dms/unbind-history-failures/databricks.yml.tmpl @@ -0,0 +1,13 @@ +bundle: + name: dms-unbind-history-failures-$UNIQUE_NAME + +resources: + jobs: + create_failed: + name: create_failed + record_failed: + name: record_failed + complete_failed: + name: complete_failed + upload_failed: + name: upload_failed diff --git a/acceptance/bundle/dms/unbind-history-failures/out.test.toml b/acceptance/bundle/dms/unbind-history-failures/out.test.toml new file mode 100644 index 00000000000..88e6fbd847c --- /dev/null +++ b/acceptance/bundle/dms/unbind-history-failures/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DMS = ["true"] diff --git a/acceptance/bundle/dms/unbind-history-failures/output.txt b/acceptance/bundle/dms/unbind-history-failures/output.txt new file mode 100644 index 00000000000..1480f227969 --- /dev/null +++ b/acceptance/bundle/dms/unbind-history-failures/output.txt @@ -0,0 +1,86 @@ + +=== A version creation failure does not prevent unbinding + +>>> fault.py POST /api/2.0/bundle/deployments/*/versions 400 0 1 + +>>> [CLI] bundle deployment unbind create_failed +Warn: Failed to record unbind history: failed to create deployment version: Fault injected by test. + +=== An operation recording failure does not prevent unbinding + +>>> fault.py PATCH /api/2.0/bundle/** 400 0 1 + +>>> [CLI] bundle deployment unbind record_failed +Warn: Failed to record deployment history: recording operation for resources.jobs.record_failed: Fault injected by test. + +=== A version completion failure does not prevent unbinding + +>>> fault.py POST /api/2.0/bundle/deployments/*/versions/*/complete 400 0 1 + +>>> [CLI] bundle deployment unbind complete_failed +Warn: Failed to complete deployment history version: Fault injected by test. + +=== Successful unbindings survived the remote state round trips and the job still exists + +>>> jq .state .databricks/bundle/default/resources.json +{ + "resources.jobs.upload_failed": { + "__id__": "[JOB_ID]", + "state": { + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-unbind-history-failures-[UNIQUE_NAME]/default/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "upload_failed", + "queue": { + "enabled": true + } + } + } +} + +>>> [CLI] jobs get [JOB_ID] +{ + "job_id": [JOB_ID], + "name": "unbind-target-[UNIQUE_NAME]" +} + +>>> print_requests.py --dms //versions --oneline +{"method": "POST", "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions", "q": {"version_id": "5"}, "body": {"cli_version": "[CLI_VERSION]", "operations": [{"action_type": "OPERATION_ACTION_TYPE_UPDATE", "resource_key": "jobs.create_failed"}], "previous_version_id": "4", "version_type": "VERSION_TYPE_DEPLOY"}} +{"method": "POST", "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions", "q": {"version_id": "6"}, "body": {"cli_version": "[CLI_VERSION]", "operations": [{"action_type": "OPERATION_ACTION_TYPE_UPDATE", "resource_key": "jobs.record_failed"}], "previous_version_id": "4", "version_type": "VERSION_TYPE_DEPLOY"}} +{"method": "PATCH", "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions/6/operations/jobs.record_failed", "q": {"update_mask": "state,error_message,status"}, "body": {"error_message": "", "sequence_id": 0, "status": "OPERATION_STATUS_SUCCEEDED"}} +{"method": "POST", "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions/6/complete", "body": {"completion_reason": "VERSION_COMPLETE_FAILURE"}} +{"method": "POST", "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions", "q": {"version_id": "7"}, "body": {"cli_version": "[CLI_VERSION]", "operations": [{"action_type": "OPERATION_ACTION_TYPE_UPDATE", "resource_key": "jobs.complete_failed"}], "previous_version_id": "6", "version_type": "VERSION_TYPE_DEPLOY"}} +{"method": "PATCH", "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions/7/operations/jobs.complete_failed", "q": {"update_mask": "state,error_message,status"}, "body": {"error_message": "", "sequence_id": 0, "status": "OPERATION_STATUS_SUCCEEDED"}} +{"method": "POST", "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions/7/complete", "body": {"completion_reason": "VERSION_COMPLETE_SUCCESS"}} + +=== A remote state upload failure fails the unbind version + +>>> fault.py POST /api/2.0/workspace-files/import-file/**resources.json 400 0 1 + +>>> musterr [CLI] bundle deployment unbind upload_failed +Error: Fault injected by test. (400 INJECTED) + +Endpoint: POST [DATABRICKS_URL]/api/2.0/workspace-files/import-file/Workspace%2FUsers%2F[USERNAME]%2F.bundle%2Fdms-unbind-history-failures-[UNIQUE_NAME]%2Fdefault%2Fstate%2Fresources.json?overwrite=true +HTTP Status: 400 Bad Request +API error_code: INJECTED +API message: Fault injected by test. + + +>>> print_requests.py --dms //versions --oneline +{"method": "POST", "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions", "q": {"version_id": "8"}, "body": {"cli_version": "[CLI_VERSION]", "operations": [{"action_type": "OPERATION_ACTION_TYPE_UPDATE", "resource_key": "jobs.upload_failed"}], "previous_version_id": "7", "version_type": "VERSION_TYPE_DEPLOY"}} +{"method": "PATCH", "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions/8/operations/jobs.upload_failed", "q": {"update_mask": "state,error_message,status"}, "body": {"error_message": "", "sequence_id": 0, "status": "OPERATION_STATUS_SUCCEEDED"}} +{"method": "POST", "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions/8/complete", "body": {"completion_reason": "VERSION_COMPLETE_FAILURE"}} + +=== Remote state still contains the binding whose upload failed + +>>> [CLI] bundle summary -o json +{ + "complete_failed": null, + "create_failed": null, + "record_failed": null, + "upload_failed": "[JOB_ID]" +} diff --git a/acceptance/bundle/dms/unbind-history-failures/script b/acceptance/bundle/dms/unbind-history-failures/script new file mode 100644 index 00000000000..ef0101567b2 --- /dev/null +++ b/acceptance/bundle/dms/unbind-history-failures/script @@ -0,0 +1,37 @@ +envsubst < databricks.yml.tmpl > databricks.yml +job_id=$($CLI jobs create --json '{"name": "unbind-target-'$UNIQUE_NAME'"}' | jq -r '.job_id') +add_repl "$job_id" JOB_ID +for key in create_failed record_failed complete_failed upload_failed; do + $CLI bundle deployment bind "$key" "$job_id" --auto-approve >> LOG.setup 2>&1 +done +rm -f "$OUT_REQUESTS" + +title "A version creation failure does not prevent unbinding" +trace fault.py "POST /api/2.0/bundle/deployments/*/versions" 400 0 1 +trace $CLI bundle deployment unbind create_failed +rm -rf .databricks + +title "An operation recording failure does not prevent unbinding" +trace fault.py "PATCH /api/2.0/bundle/**" 400 0 1 +trace $CLI bundle deployment unbind record_failed +rm -rf .databricks + +title "A version completion failure does not prevent unbinding" +trace fault.py "POST /api/2.0/bundle/deployments/*/versions/*/complete" 400 0 1 +trace $CLI bundle deployment unbind complete_failed + +title "Successful unbindings survived the remote state round trips and the job still exists" +trace jq '.state' .databricks/bundle/default/resources.json +trace $CLI jobs get "$job_id" | jq '{job_id, name: .settings.name}' +trace print_requests.py --dms //versions --oneline + + +title "A remote state upload failure fails the unbind version" +trace fault.py "POST /api/2.0/workspace-files/import-file/**resources.json" 400 0 1 +trace musterr $CLI bundle deployment unbind upload_failed +trace print_requests.py --dms //versions --oneline + +rm -rf .databricks +title "Remote state still contains the binding whose upload failed" +trace $CLI bundle summary -o json | jq '.resources.jobs | map_values(.id // null)' +rm -f "$OUT_REQUESTS" diff --git a/acceptance/bundle/dms/unbind-history-failures/test.toml b/acceptance/bundle/dms/unbind-history-failures/test.toml new file mode 100644 index 00000000000..0e4c9337799 --- /dev/null +++ b/acceptance/bundle/dms/unbind-history-failures/test.toml @@ -0,0 +1 @@ +Cloud = false # Inject failures into the deployment history service. diff --git a/acceptance/bundle/dms_fault/complete_version/output.txt b/acceptance/bundle/dms_fault/complete_version/output.txt index 2d6b9c60c94..ffe8b045fc7 100644 --- a/acceptance/bundle/dms_fault/complete_version/output.txt +++ b/acceptance/bundle/dms_fault/complete_version/output.txt @@ -48,6 +48,7 @@ >>> bundle deployment unbind foo + Warn: Failed to complete deployment history version: Fault injected by test. >>> bundle generate job --existing-job-id [FOO_ID] --key foo --config-dir resources --source-dir src --force --bind Job configuration successfully saved to resources/foo.job.yml @@ -67,6 +68,7 @@ >>> bundle deployment unbind bar + Warn: Failed to complete deployment history version: Fault injected by test. >>> bundle generate pipeline --existing-pipeline-id [BAR_ID] --key bar --config-dir resources --source-dir src --force --bind File successfully saved to src/pipeline.sql diff --git a/acceptance/bundle/dms_fault/create_deployment/output.txt b/acceptance/bundle/dms_fault/create_deployment/output.txt index 3f346874945..bcaf6e57381 100644 --- a/acceptance/bundle/dms_fault/create_deployment/output.txt +++ b/acceptance/bundle/dms_fault/create_deployment/output.txt @@ -39,6 +39,7 @@ >>> bundle deployment unbind foo + Warn: Failed to record unbind history: failed to create deployment: Fault injected by test. >>> bundle generate job --existing-job-id [FOO_ID] --key foo --config-dir resources --source-dir src --force --bind Job configuration successfully saved to resources/foo.job.yml @@ -58,6 +59,7 @@ >>> bundle deployment unbind bar + Warn: Failed to record unbind history: failed to create deployment: Fault injected by test. >>> bundle generate pipeline --existing-pipeline-id [BAR_ID] --key bar --config-dir resources --source-dir src --force --bind File successfully saved to src/pipeline.sql diff --git a/acceptance/bundle/dms_fault/create_version/output.txt b/acceptance/bundle/dms_fault/create_version/output.txt index 89e7730c044..065383757c8 100644 --- a/acceptance/bundle/dms_fault/create_version/output.txt +++ b/acceptance/bundle/dms_fault/create_version/output.txt @@ -47,6 +47,7 @@ >>> bundle deployment unbind foo + Warn: Failed to record unbind history: failed to create deployment version: Fault injected by test. >>> bundle generate job --existing-job-id [FOO_ID] --key foo --config-dir resources --source-dir src --force --bind Job configuration successfully saved to resources/foo.job.yml @@ -66,6 +67,7 @@ >>> bundle deployment unbind bar + Warn: Failed to record unbind history: failed to create deployment version: Fault injected by test. >>> bundle generate pipeline --existing-pipeline-id [BAR_ID] --key bar --config-dir resources --source-dir src --force --bind File successfully saved to src/pipeline.sql diff --git a/acceptance/bundle/dms_fault/get_deployment/output.txt b/acceptance/bundle/dms_fault/get_deployment/output.txt index e7603c56e18..0cd9844b781 100644 --- a/acceptance/bundle/dms_fault/get_deployment/output.txt +++ b/acceptance/bundle/dms_fault/get_deployment/output.txt @@ -55,6 +55,9 @@ >>> bundle deployment unbind foo + Warn: retrying after 500 Internal Server Error from GET /api/2.0/bundle/deployments/[DEPLOYMENT_ID] + Warn: retrying after 500 Internal Server Error from GET /api/2.0/bundle/deployments/[DEPLOYMENT_ID] + Warn: Failed to record unbind history: Fault injected by test. >>> bundle generate job --existing-job-id [FOO_ID] --key foo --config-dir resources --source-dir src --force --bind Job configuration successfully saved to resources/foo.job.yml @@ -76,6 +79,9 @@ >>> bundle deployment unbind bar + Warn: retrying after 500 Internal Server Error from GET /api/2.0/bundle/deployments/[DEPLOYMENT_ID] + Warn: retrying after 500 Internal Server Error from GET /api/2.0/bundle/deployments/[DEPLOYMENT_ID] + Warn: Failed to record unbind history: Fault injected by test. >>> bundle generate pipeline --existing-pipeline-id [BAR_ID] --key bar --config-dir resources --source-dir src --force --bind File successfully saved to src/pipeline.sql diff --git a/acceptance/bundle/dms_fault/update_operation/output.txt b/acceptance/bundle/dms_fault/update_operation/output.txt index eb9be000cfb..523fdd02816 100644 --- a/acceptance/bundle/dms_fault/update_operation/output.txt +++ b/acceptance/bundle/dms_fault/update_operation/output.txt @@ -48,6 +48,7 @@ >>> bundle deployment unbind foo + Warn: Failed to record deployment history: recording operation for [RESOURCE_KEY]: Fault injected by test. >>> bundle generate job --existing-job-id [FOO_ID] --key foo --config-dir resources --source-dir src --force --bind Job configuration successfully saved to resources/foo.job.yml @@ -67,6 +68,7 @@ >>> bundle deployment unbind bar + Warn: Failed to record deployment history: recording operation for [RESOURCE_KEY]: Fault injected by test. >>> bundle generate pipeline --existing-pipeline-id [BAR_ID] --key bar --config-dir resources --source-dir src --force --bind File successfully saved to src/pipeline.sql diff --git a/bundle/direct/bind.go b/bundle/direct/bind.go index bd0e2cdbdc0..2853498a5e4 100644 --- a/bundle/direct/bind.go +++ b/bundle/direct/bind.go @@ -147,39 +147,31 @@ func (b *DeploymentBundle) Bind(ctx context.Context, client *databricks.Workspac return result, nil } -// Unbind removes a resource from direct engine state without deleting -// the workspace resource. Also removes associated permissions/grants entries. -func (b *DeploymentBundle) Unbind(ctx context.Context, statePath, resourceKey string) error { - if err := b.StateDB.Open(ctx, statePath, dstate.ReadingFile, dstate.WithRecovery(true), dstate.OpenDmsArgs{}); err != nil { - return err - } - if err := b.StateDB.UpgradeToWrite(); err != nil { - return err - } - - // Delete the main resource - err := b.StateDB.DeleteState(ctx, resourceKey, false) - if err != nil { - return err - } - - log.Infof(ctx, "Unbound %s", resourceKey) - +// UnbindResourceKeys returns the sorted resource and child keys present in the open state. +func (b *DeploymentBundle) UnbindResourceKeys(resourceKey string) []string { // Also delete associated permissions/grants if they exist // These are stored as "resources.jobs.foo.permissions" or "resources.schemas.bar.grants" - permissionsKey := resourceKey + ".permissions" - grantsKey := resourceKey + ".grants" - + var keys []string for key := range b.StateDB.Data.State { - if key == permissionsKey || key == grantsKey || strings.HasPrefix(key, resourceKey+".") { - err = b.StateDB.DeleteState(ctx, key, false) - if err != nil { - return err - } - log.Infof(ctx, "Unbound %s", key) + if key == resourceKey || strings.HasPrefix(key, resourceKey+".") { + keys = append(keys, key) } } + slices.Sort(keys) + return keys +} - _, err = b.StateDB.Finalize(ctx) - return err +// Unbind removes the selected keys from open writable state without deleting workspace resources. +func (b *DeploymentBundle) Unbind(ctx context.Context, resourceKey string, keys []string) error { + if len(keys) == 0 { + // Preserve the existing local transaction when the resource is already absent. + keys = []string{resourceKey} + } + for _, key := range keys { + if err := b.StateDB.DeleteState(ctx, key, false); err != nil { + return err + } + log.Infof(ctx, "Unbound %s", key) + } + return nil } diff --git a/bundle/direct/bind_test.go b/bundle/direct/bind_test.go index 08a32db807e..ab5cc9cefa8 100644 --- a/bundle/direct/bind_test.go +++ b/bundle/direct/bind_test.go @@ -100,3 +100,49 @@ func TestBindPreparesIsolatedState(t *testing.T) { assert.True(t, prepared.HasChanges) assert.Equal(t, deployplan.Update, prepared.Action) } + +func TestUnbindResourceKeys(t *testing.T) { + for _, tc := range []struct { + name string + keys []string + history bool + }{ + {"resource", []string{"resources.jobs.foo"}, true}, + {"children", []string{"resources.jobs.foo", "resources.jobs.foo.grants", "resources.jobs.foo.permissions", "resources.jobs.foo.permissions.child"}, true}, + {"children_without_parent", []string{"resources.jobs.foo.permissions"}, true}, + {"missing", nil, true}, + {"history_disabled", []string{"resources.jobs.foo"}, false}, + } { + t.Run(tc.name, func(t *testing.T) { + ctx := t.Context() + statePath := filepath.Join(t.TempDir(), "resources.json") + var b direct.DeploymentBundle + require.NoError(t, b.StateDB.Open(ctx, statePath, dstate.ReadingFile, false, dstate.OpenDmsArgs{})) + require.NoError(t, b.StateDB.UpgradeToWrite()) + for _, key := range append(tc.keys, "resources.jobs.foobar") { + require.NoError(t, b.StateDB.SaveState(ctx, key, "id", map[string]string{"name": key}, nil)) + } + _, err := b.StateDB.Finalize(ctx) + require.NoError(t, err) + + require.NoError(t, b.StateDB.Open(ctx, statePath, dstate.ReadingFile, false, dstate.OpenDmsArgs{})) + keys := b.UnbindResourceKeys("resources.jobs.foo") + assert.Equal(t, tc.keys, keys) + assert.Len(t, b.StateDB.Data.State, len(tc.keys)+1) + if tc.history { + b.StateDB.Data.Features = map[string]struct{}{dstate.FeatureDeploymentHistoryShadow: {}} + } + require.NoError(t, b.StateDB.UpgradeToWrite()) + require.NoError(t, b.Unbind(ctx, "resources.jobs.foo", keys)) + _, err = b.StateDB.Finalize(ctx) + require.NoError(t, err) + + require.NoError(t, b.StateDB.Open(ctx, statePath, dstate.ReadingFile, false, dstate.OpenDmsArgs{})) + assert.Len(t, b.StateDB.Data.State, 1) + assert.Equal(t, 2, b.StateDB.Data.Serial) + assert.Equal(t, "id", b.StateDB.GetResourceID("resources.jobs.foobar")) + _, err = b.StateDB.Finalize(ctx) + require.NoError(t, err) + }) + } +} diff --git a/bundle/phases/bind.go b/bundle/phases/bind.go index 293d842dd78..3984528bcb1 100644 --- a/bundle/phases/bind.go +++ b/bundle/phases/bind.go @@ -34,7 +34,7 @@ type BindOptions struct { ResourceId string } -// createBindingDeployment creates a deployment record for the first recorded bind, without a deployed file path. +// createBindingDeployment creates a deployment record for the first recorded binding change, without a deployed file path. func createBindingDeployment(ctx context.Context, b *bundle.Bundle) (string, error) { // Bind does not upload bundle files, so it cannot report a deployed file path. metadata := bundleMetadata(b) @@ -152,7 +152,7 @@ func Bind(ctx context.Context, b *bundle.Bundle, opts *BindOptions, stateDesc *s return } if db.RecordsHistory() { - if err := prepareBindHistory(ctx, b, resourceKey); err != nil { + if err := prepareBindingHistory(ctx, b, []string{resourceKey}); err != nil { log.Warnf(ctx, "Failed to record bind history: %s", err) } } @@ -185,7 +185,17 @@ func Unbind(ctx context.Context, b *bundle.Bundle, group, resourceKey string, en return } + if !cmdctx.HasWorkspaceClient(ctx) { + ctx = cmdctx.SetWorkspaceClient(ctx, b.WorkspaceClient(ctx)) + } + db := &b.DeploymentBundle.StateDB defer func() { + if _, err := db.Finalize(ctx); err != nil { + logdiag.LogError(ctx, err) + } + if _, err := db.CompleteVersion(ctx, !logdiag.HasError(ctx)); err != nil { + log.Warnf(ctx, "Failed to complete deployment history version: %s", err) + } bundle.ApplyContext(ctx, b, lock.Release(lock.GoalUnbind)) }() @@ -198,8 +208,35 @@ func Unbind(ctx context.Context, b *bundle.Bundle, group, resourceKey string, en fullResourceKey := fmt.Sprintf("resources.%s.%s", group, resourceKey) _, statePath := b.StateFilenameDirect(ctx) - err := b.DeploymentBundle.Unbind(ctx, statePath, fullResourceKey) - if err != nil { + if err := db.Open(ctx, statePath, dstate.ReadingFile, dstate.WithRecovery(true), dstate.OpenDmsArgs{}); err != nil { + logdiag.LogError(ctx, err) + return + } + keys := b.DeploymentBundle.UnbindResourceKeys(fullResourceKey) + var configured *bool + if b.Config.Experimental != nil { + configured = b.Config.Experimental.DeploymentHistory + } + if db.RecordsHistory() || bundleenv.RecordsDeploymentHistory(ctx, configured) { + if db.Data.Features == nil { + db.Data.Features = make(map[string]struct{}) + } + db.Data.Features[dstate.FeatureDeploymentHistoryShadow] = struct{}{} + } + if err := db.UpgradeToWrite(); err != nil { + logdiag.LogError(ctx, err) + return + } + if db.RecordsHistory() { + if err := prepareBindingHistory(ctx, b, keys); err != nil { + log.Warnf(ctx, "Failed to record unbind history: %s", err) + } + } + if err := b.DeploymentBundle.Unbind(ctx, fullResourceKey, keys); err != nil { + logdiag.LogError(ctx, err) + return + } + if _, err := db.Finalize(ctx); err != nil { logdiag.LogError(ctx, err) return } @@ -207,8 +244,11 @@ func Unbind(ctx context.Context, b *bundle.Bundle, group, resourceKey string, en statemgmt.PushResourcesState(ctx, b) } -// prepareBindHistory stages only the approved binding, preserving deployed file metadata. -func prepareBindHistory(ctx context.Context, b *bundle.Bundle, resourceKey string) error { +// prepareBindingHistory stages only binding changes, preserving deployed file metadata. +func prepareBindingHistory(ctx context.Context, b *bundle.Bundle, resourceKeys []string) error { + if len(resourceKeys) == 0 { + return nil + } deploymentID, current, err := dms.FetchDeployment(ctx, b.WorkspaceClient(ctx), b.Config.Workspace.StatePath) if err != nil { return err @@ -226,8 +266,12 @@ func prepareBindHistory(ctx context.Context, b *bundle.Bundle, resourceKey strin } db := &b.DeploymentBundle.StateDB db.DeploymentID, db.VersionID = deploymentID, previous - return startVersion(ctx, b, dms.VersionTypeDeploy, []bundledeployments.StagedOperation{{ - ResourceKey: strings.TrimPrefix(resourceKey, dms.StatePrefix), - ActionType: bundledeployments.OperationActionTypeOperationActionTypeUpdate, - }}) + staged := make([]bundledeployments.StagedOperation, 0, len(resourceKeys)) + for _, key := range resourceKeys { + staged = append(staged, bundledeployments.StagedOperation{ + ResourceKey: strings.TrimPrefix(key, dms.StatePrefix), + ActionType: bundledeployments.OperationActionTypeOperationActionTypeUpdate, + }) + } + return startVersion(ctx, b, dms.VersionTypeDeploy, staged) } diff --git a/bundle/phases/bind_test.go b/bundle/phases/bind_test.go index aadfd009a1e..1a2ee87ad1b 100644 --- a/bundle/phases/bind_test.go +++ b/bundle/phases/bind_test.go @@ -3,6 +3,7 @@ package phases_test import ( "encoding/json" "os" + "path/filepath" "testing" "github.com/databricks/cli/bundle" @@ -22,11 +23,13 @@ import ( "github.com/stretchr/testify/require" ) -func TestBindLocalCommitFailureCompletesVersion(t *testing.T) { - ctx, _ := cmdio.NewTestContextWithStderr(t.Context()) - ctx = logdiag.InitContext(ctx) - b := &bundle.Bundle{BundleRootPath: t.TempDir()} - root, diags := config.LoadFromBytes("databricks.yml", []byte(`bundle: +func TestBindingLocalCommitFailureCompletesVersion(t *testing.T) { + for _, operation := range []string{"bind", "unbind"} { + t.Run(operation, func(t *testing.T) { + ctx, _ := cmdio.NewTestContextWithStderr(t.Context()) + ctx = logdiag.InitContext(ctx) + b := &bundle.Bundle{BundleRootPath: t.TempDir()} + root, diags := config.LoadFromBytes("databricks.yml", []byte(`bundle: name: test target: default deployment: @@ -42,56 +45,77 @@ resources: bound: name: configured `)) - require.Empty(t, diags) - b.Config = *root - _, statePath := b.StateFilenameDirect(ctx) + require.Empty(t, diags) + b.Config = *root + _, statePath := b.StateFilenameDirect(ctx) + if operation == "unbind" { + data := dstate.NewDatabase("lineage", 0) + data.State["resources.jobs.bound"] = dstate.ResourceEntry{ID: "20", State: json.RawMessage(`{"name":"configured"}`)} + state, err := json.Marshal(data) + require.NoError(t, err) + require.NoError(t, os.MkdirAll(filepath.Dir(statePath), 0o700)) + require.NoError(t, os.WriteFile(statePath, state, 0o600)) + } - server := testserver.New(t) - server.Handle("GET", "/api/2.2/jobs/get", func(req testserver.Request) any { - return jobs.Job{JobId: 20, Settings: &jobs.JobSettings{Name: "remote"}} - }) - server.Handle("GET", "/api/2.0/workspace/get-status", func(req testserver.Request) any { - return map[string]any{"object_id": 123, "object_type": "FILE"} - }) - server.Handle("GET", "/api/2.0/bundle/deployments/123", func(req testserver.Request) any { - return bundledeployments.Deployment{Name: "deployments/123"} - }) - server.Handle("POST", "/api/2.0/bundle/deployments/123/versions", func(req testserver.Request) any { - // The WAL is already open. Make the atomic state replacement fail after SaveState. - require.FileExists(t, statePath+".wal") - require.NoError(t, os.Mkdir(statePath, 0o700)) - return bundledeployments.Version{Name: "deployments/123/versions/1", VersionId: "1"} - }) - server.Handle("PATCH", "/api/2.0/bundle/deployments/123/versions/1/operations/jobs.bound", func(req testserver.Request) any { - return bundledeployments.Operation{} - }) - var completion bundledeployments.CompleteVersionRequest - server.Handle("POST", "/api/2.0/bundle/deployments/123/versions/1/complete", func(req testserver.Request) any { - require.NoError(t, json.Unmarshal(req.Body, &completion)) - return bundledeployments.Version{} - }) - testserver.AddDefaultHandlers(server) - client, err := databricks.NewWorkspaceClient(&databricks.Config{Host: server.URL, Token: "testtoken"}) - require.NoError(t, err) - b.SetWorkpaceClient(client) - ctx = cmdctx.SetWorkspaceClient(ctx, client) + server := testserver.New(t) + server.Handle("GET", "/api/2.2/jobs/get", func(req testserver.Request) any { + return jobs.Job{JobId: 20, Settings: &jobs.JobSettings{Name: "remote"}} + }) + server.Handle("GET", "/api/2.0/workspace/get-status", func(req testserver.Request) any { + return map[string]any{"object_id": 123, "object_type": "FILE"} + }) + server.Handle("GET", "/api/2.0/bundle/deployments/123", func(req testserver.Request) any { + return bundledeployments.Deployment{Name: "deployments/123"} + }) + server.Handle("POST", "/api/2.0/bundle/deployments/123/versions", func(req testserver.Request) any { + // The WAL is already open. Make the atomic state replacement fail after the state write. + require.FileExists(t, statePath+".wal") + if operation == "unbind" { + require.NoError(t, os.Remove(statePath)) + } + require.NoError(t, os.Mkdir(statePath, 0o700)) + return bundledeployments.Version{Name: "deployments/123/versions/1", VersionId: "1"} + }) + server.Handle("PATCH", "/api/2.0/bundle/deployments/123/versions/1/operations/jobs.bound", func(req testserver.Request) any { + return bundledeployments.Operation{} + }) + var completion bundledeployments.CompleteVersionRequest + server.Handle("POST", "/api/2.0/bundle/deployments/123/versions/1/complete", func(req testserver.Request) any { + require.NoError(t, json.Unmarshal(req.Body, &completion)) + return bundledeployments.Version{} + }) + testserver.AddDefaultHandlers(server) + client, err := databricks.NewWorkspaceClient(&databricks.Config{Host: server.URL, Token: "testtoken"}) + require.NoError(t, err) + b.SetWorkpaceClient(client) + ctx = cmdctx.SetWorkspaceClient(ctx, client) - phases.Bind(ctx, b, &phases.BindOptions{Group: "jobs", ResourceKey: "bound", ResourceId: "20", AutoApprove: true}, &statemgmt.StateDesc{Engine: engine.EngineDirect}) + if operation == "unbind" { + phases.Unbind(ctx, b, "jobs", "bound", engine.EngineDirect) + } else { + phases.Bind(ctx, b, &phases.BindOptions{Group: "jobs", ResourceKey: "bound", ResourceId: "20", AutoApprove: true}, &statemgmt.StateDesc{Engine: engine.EngineDirect}) + } - assert.True(t, logdiag.HasError(ctx)) - assert.False(t, b.DeploymentBundle.StateDB.IsOpen()) - assert.Equal(t, bundledeployments.VersionCompleteVersionCompleteFailure, completion.CompletionReason) - // Completion releases the operation buffer while retaining its final measurements. - require.NotNil(t, b.DeploymentBundle.StateDB.OperationBufferStats()) - completed, err := b.DeploymentBundle.StateDB.CompleteVersion(ctx, true) - require.NoError(t, err) - assert.False(t, completed) + assert.True(t, logdiag.HasError(ctx)) + assert.False(t, b.DeploymentBundle.StateDB.IsOpen()) + assert.Equal(t, bundledeployments.VersionCompleteVersionCompleteFailure, completion.CompletionReason) + // Completion releases the operation buffer while retaining its final measurements. + require.NotNil(t, b.DeploymentBundle.StateDB.OperationBufferStats()) + completed, err := b.DeploymentBundle.StateDB.CompleteVersion(ctx, true) + require.NoError(t, err) + assert.False(t, completed) - // Finalize closed the WAL even though persistence failed, so ordinary recovery can retry it. - require.NoError(t, os.Remove(statePath)) - var recovered dstate.DeploymentState - require.NoError(t, recovered.Open(ctx, statePath, dstate.ReadingFile, dstate.WithRecovery(true), dstate.OpenDmsArgs{})) - assert.Equal(t, "20", recovered.GetResourceID("resources.jobs.bound")) - _, err = recovered.Finalize(ctx) - require.NoError(t, err) + // Finalize closed the WAL even though persistence failed, so ordinary recovery can retry it. + require.NoError(t, os.Remove(statePath)) + var recovered dstate.DeploymentState + require.NoError(t, recovered.Open(ctx, statePath, dstate.ReadingFile, dstate.WithRecovery(true), dstate.OpenDmsArgs{})) + if operation == "unbind" { + assert.Empty(t, recovered.GetResourceID("resources.jobs.bound")) + } else { + assert.Equal(t, "20", recovered.GetResourceID("resources.jobs.bound")) + } + _, err = recovered.Finalize(ctx) + require.NoError(t, err) + }) + } }