From 80a499c56b515d1601d0f9301cda0286f63b9ace Mon Sep 17 00:00:00 2001 From: "denis.bilenko" Date: Tue, 17 Nov 2026 07:33:07 +0000 Subject: [PATCH 1/2] direct: retry delete on transient server errors bundle destroy (DeploymentUnit.Delete) now retries DoDelete on transient 5xx/408 errors, as Recreate, Update and Resize already do. gcp: TestAccept/bundle/resources/postgres_catalogs/recreate TestAccept/bundle/resources/postgres_synced_tables/recreate Co-authored-by: Isaac --- .../delete_transient_500/databricks.yml | 12 ++++++++++ .../delete_transient_500/out.test.toml | 3 +++ .../delete_transient_500/output.txt | 23 +++++++++++++++++++ .../delete_transient_500/script | 9 ++++++++ .../delete_transient_500/test.toml | 6 +++++ bundle/direct/apply.go | 5 +++- 6 files changed, 57 insertions(+), 1 deletion(-) create mode 100644 acceptance/bundle/resources/postgres_projects/delete_transient_500/databricks.yml create mode 100644 acceptance/bundle/resources/postgres_projects/delete_transient_500/out.test.toml create mode 100644 acceptance/bundle/resources/postgres_projects/delete_transient_500/output.txt create mode 100644 acceptance/bundle/resources/postgres_projects/delete_transient_500/script create mode 100644 acceptance/bundle/resources/postgres_projects/delete_transient_500/test.toml diff --git a/acceptance/bundle/resources/postgres_projects/delete_transient_500/databricks.yml b/acceptance/bundle/resources/postgres_projects/delete_transient_500/databricks.yml new file mode 100644 index 00000000000..b95ebe35d2c --- /dev/null +++ b/acceptance/bundle/resources/postgres_projects/delete_transient_500/databricks.yml @@ -0,0 +1,12 @@ +bundle: + name: test-bundle + +sync: + paths: [] + +resources: + postgres_projects: + my_project: + project_id: test-pg-proj + display_name: "Test Postgres Project" + pg_version: 16 diff --git a/acceptance/bundle/resources/postgres_projects/delete_transient_500/out.test.toml b/acceptance/bundle/resources/postgres_projects/delete_transient_500/out.test.toml new file mode 100644 index 00000000000..15c32f324ac --- /dev/null +++ b/acceptance/bundle/resources/postgres_projects/delete_transient_500/out.test.toml @@ -0,0 +1,3 @@ +Cloud = false +EnvMatrix.DMS = ["", "true"] +EnvMatrix.READPLAN = [] diff --git a/acceptance/bundle/resources/postgres_projects/delete_transient_500/output.txt b/acceptance/bundle/resources/postgres_projects/delete_transient_500/output.txt new file mode 100644 index 00000000000..6ac1d974165 --- /dev/null +++ b/acceptance/bundle/resources/postgres_projects/delete_transient_500/output.txt @@ -0,0 +1,23 @@ + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Created postgres_projects.my_project +Files: 0 uploaded, 0 deleted +Resources: 1 created, 0 changed, 0 deleted, 0 unchanged + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.postgres_projects.my_project + +This action will result in the deletion of the following Lakebase projects along with +all their branches, databases, and endpoints. All data stored in them will be permanently lost: + delete resources.postgres_projects.my_project + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle/default + +Warn: destroying resources.postgres_projects.my_project: retrying after 500 Internal Server Error from DELETE /api/2.0/postgres/projects/test-pg-proj +Destroy: 1 deleted + +>>> print_requests.py --get //postgres +DELETE /api/2.0/postgres/projects/test-pg-proj +DELETE /api/2.0/postgres/projects/test-pg-proj diff --git a/acceptance/bundle/resources/postgres_projects/delete_transient_500/script b/acceptance/bundle/resources/postgres_projects/delete_transient_500/script new file mode 100644 index 00000000000..bc0ed27b292 --- /dev/null +++ b/acceptance/bundle/resources/postgres_projects/delete_transient_500/script @@ -0,0 +1,9 @@ +trace $CLI bundle deploy + +# One transient 500 on the delete request; it is retried and the destroy succeeds. +fault.py "DELETE /api/2.0/postgres/projects/*" 500 0 1 INTERNAL_ERROR + +trace $CLI bundle destroy --auto-approve + +# Two delete requests: the 500 and the retry. +trace print_requests.py --get //postgres | jq -r 'select(.method == "DELETE") | .method + " " + .path' diff --git a/acceptance/bundle/resources/postgres_projects/delete_transient_500/test.toml b/acceptance/bundle/resources/postgres_projects/delete_transient_500/test.toml new file mode 100644 index 00000000000..ab3f407bfa7 --- /dev/null +++ b/acceptance/bundle/resources/postgres_projects/delete_transient_500/test.toml @@ -0,0 +1,6 @@ +Cloud = false +RecordRequests = true +EnvMatrix.READPLAN = [] +Env.DATABRICKS_BUNDLE_ENGINE = "direct" +Env.DATABRICKS_BUNDLE_RETRY_INTERVAL_MS = "100" +SuppressLines.warn_retry = "" diff --git a/bundle/direct/apply.go b/bundle/direct/apply.go index 5b093a61d4c..2683c6c1532 100644 --- a/bundle/direct/apply.go +++ b/bundle/direct/apply.go @@ -299,7 +299,10 @@ func (d *DeploymentUnit) Delete(ctx context.Context, db *dstate.DeploymentState, return err } - err = d.Adapter.DoDelete(ctx, oldID, oldState) + // Delete is idempotent: if a failed attempt did delete the resource, the retry gets a NotFound. + _, err = retry.With(ctx, retry.IsTransient, func() (struct{}, error) { + return struct{}{}, d.Adapter.DoDelete(ctx, oldID, oldState) + }) if err != nil && !apierr.IsMissing(err) && !isManagedByParent(err) { // Rather than failing delete and requiring user to unbind, we perform unbind automatically there. // Some services, e.g. jobs, return 403 for missing resources if caller did not have permissions to it when job existed. From ad3fd0078d3c3d5fbb51667e7162c897b4a2b004 Mon Sep 17 00:00:00 2001 From: "denis.bilenko" Date: Tue, 17 Nov 2026 12:15:26 +0000 Subject: [PATCH 2/2] acc: test delete retry when the delete landed; add changelog fragment The fault now fires after the handler, so the retried DELETE gets a NotFound, which Delete treats as success. Use print_requests.py --method DELETE --summary. gcp: TestAccept/bundle/resources/postgres_catalogs/recreate TestAccept/bundle/resources/postgres_synced_tables/recreate Co-authored-by: Isaac --- .nextchanges/bundles/7708.md | 1 + .../postgres_projects/delete_transient_500/output.txt | 2 +- .../postgres_projects/delete_transient_500/script | 9 +++++---- 3 files changed, 7 insertions(+), 5 deletions(-) create mode 100644 .nextchanges/bundles/7708.md diff --git a/.nextchanges/bundles/7708.md b/.nextchanges/bundles/7708.md new file mode 100644 index 00000000000..036355b3c8d --- /dev/null +++ b/.nextchanges/bundles/7708.md @@ -0,0 +1 @@ +* Retry transient server errors when deleting resources in the direct deployment engine. ([#7708](https://github.com/databricks/cli/pull/7708)) diff --git a/acceptance/bundle/resources/postgres_projects/delete_transient_500/output.txt b/acceptance/bundle/resources/postgres_projects/delete_transient_500/output.txt index 6ac1d974165..9fb3c8fbaf2 100644 --- a/acceptance/bundle/resources/postgres_projects/delete_transient_500/output.txt +++ b/acceptance/bundle/resources/postgres_projects/delete_transient_500/output.txt @@ -18,6 +18,6 @@ All files and directories at the following location will be deleted: /Workspace/ Warn: destroying resources.postgres_projects.my_project: retrying after 500 Internal Server Error from DELETE /api/2.0/postgres/projects/test-pg-proj Destroy: 1 deleted ->>> print_requests.py --get //postgres +>>> print_requests.py --method DELETE --summary //postgres DELETE /api/2.0/postgres/projects/test-pg-proj DELETE /api/2.0/postgres/projects/test-pg-proj diff --git a/acceptance/bundle/resources/postgres_projects/delete_transient_500/script b/acceptance/bundle/resources/postgres_projects/delete_transient_500/script index bc0ed27b292..16b8da255eb 100644 --- a/acceptance/bundle/resources/postgres_projects/delete_transient_500/script +++ b/acceptance/bundle/resources/postgres_projects/delete_transient_500/script @@ -1,9 +1,10 @@ trace $CLI bundle deploy -# One transient 500 on the delete request; it is retried and the destroy succeeds. -fault.py "DELETE /api/2.0/postgres/projects/*" 500 0 1 INTERNAL_ERROR +# The delete lands but the response is a transient 500. The retry gets a NotFound, which counts as +# deleted, so the destroy succeeds. +fault.py --after-handler "DELETE /api/2.0/postgres/projects/*" 500 0 1 INTERNAL_ERROR trace $CLI bundle destroy --auto-approve -# Two delete requests: the 500 and the retry. -trace print_requests.py --get //postgres | jq -r 'select(.method == "DELETE") | .method + " " + .path' +# Two delete requests: the one that landed and the retry. +trace print_requests.py --method DELETE --summary //postgres