Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .nextchanges/bundles/7708.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
* Retry transient server errors when deleting resources in the direct deployment engine. ([#7708](https://github.com/databricks/cli/pull/7708))
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
bundle:
name: test-bundle

sync:
paths: []

resources:
postgres_projects:
my_project:
project_id: test-pg-proj
display_name: "Test Postgres Project"
pg_version: 16
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
Cloud = false
EnvMatrix.DMS = ["", "true"]
EnvMatrix.READPLAN = []
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@

>>> [CLI] bundle deploy
Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files...
Created postgres_projects.my_project
Files: 0 uploaded, 0 deleted
Resources: 1 created, 0 changed, 0 deleted, 0 unchanged

>>> [CLI] bundle destroy --auto-approve
The following resources will be deleted:
delete resources.postgres_projects.my_project

This action will result in the deletion of the following Lakebase projects along with
all their branches, databases, and endpoints. All data stored in them will be permanently lost:
delete resources.postgres_projects.my_project

All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle/default

Warn: destroying resources.postgres_projects.my_project: retrying after 500 Internal Server Error from DELETE /api/2.0/postgres/projects/test-pg-proj
Destroy: 1 deleted

>>> print_requests.py --method DELETE --summary //postgres
DELETE /api/2.0/postgres/projects/test-pg-proj
DELETE /api/2.0/postgres/projects/test-pg-proj
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
trace $CLI bundle deploy

# The delete lands but the response is a transient 500. The retry gets a NotFound, which counts as
# deleted, so the destroy succeeds.
fault.py --after-handler "DELETE /api/2.0/postgres/projects/*" 500 0 1 INTERNAL_ERROR

trace $CLI bundle destroy --auto-approve

# Two delete requests: the one that landed and the retry.
trace print_requests.py --method DELETE --summary //postgres
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
Cloud = false
RecordRequests = true
EnvMatrix.READPLAN = []
Env.DATABRICKS_BUNDLE_ENGINE = "direct"
Env.DATABRICKS_BUNDLE_RETRY_INTERVAL_MS = "100"
SuppressLines.warn_retry = ""
5 changes: 4 additions & 1 deletion bundle/direct/apply.go
Original file line number Diff line number Diff line change
Expand Up @@ -299,7 +299,10 @@ func (d *DeploymentUnit) Delete(ctx context.Context, db *dstate.DeploymentState,
return err
}

err = d.Adapter.DoDelete(ctx, oldID, oldState)
// Delete is idempotent: if a failed attempt did delete the resource, the retry gets a NotFound.
_, err = retry.With(ctx, retry.IsTransient, func() (struct{}, error) {
Comment thread
denik marked this conversation as resolved.
Resolved
return struct{}{}, d.Adapter.DoDelete(ctx, oldID, oldState)
})
if err != nil && !apierr.IsMissing(err) && !isManagedByParent(err) {
// Rather than failing delete and requiring user to unbind, we perform unbind automatically there.
// Some services, e.g. jobs, return 403 for missing resources if caller did not have permissions to it when job existed.
Expand Down
Loading