diff --git a/.nextchanges/bundles/7708.md b/.nextchanges/bundles/7708.md new file mode 100644 index 00000000000..036355b3c8d --- /dev/null +++ b/.nextchanges/bundles/7708.md @@ -0,0 +1 @@ +* Retry transient server errors when deleting resources in the direct deployment engine. ([#7708](https://github.com/databricks/cli/pull/7708)) diff --git a/acceptance/bundle/resources/postgres_projects/delete_transient_500/databricks.yml b/acceptance/bundle/resources/postgres_projects/delete_transient_500/databricks.yml new file mode 100644 index 00000000000..b95ebe35d2c --- /dev/null +++ b/acceptance/bundle/resources/postgres_projects/delete_transient_500/databricks.yml @@ -0,0 +1,12 @@ +bundle: + name: test-bundle + +sync: + paths: [] + +resources: + postgres_projects: + my_project: + project_id: test-pg-proj + display_name: "Test Postgres Project" + pg_version: 16 diff --git a/acceptance/bundle/resources/postgres_projects/delete_transient_500/out.test.toml b/acceptance/bundle/resources/postgres_projects/delete_transient_500/out.test.toml new file mode 100644 index 00000000000..15c32f324ac --- /dev/null +++ b/acceptance/bundle/resources/postgres_projects/delete_transient_500/out.test.toml @@ -0,0 +1,3 @@ +Cloud = false +EnvMatrix.DMS = ["", "true"] +EnvMatrix.READPLAN = [] diff --git a/acceptance/bundle/resources/postgres_projects/delete_transient_500/output.txt b/acceptance/bundle/resources/postgres_projects/delete_transient_500/output.txt new file mode 100644 index 00000000000..9fb3c8fbaf2 --- /dev/null +++ b/acceptance/bundle/resources/postgres_projects/delete_transient_500/output.txt @@ -0,0 +1,23 @@ + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Created postgres_projects.my_project +Files: 0 uploaded, 0 deleted +Resources: 1 created, 0 changed, 0 deleted, 0 unchanged + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.postgres_projects.my_project + +This action will result in the deletion of the following Lakebase projects along with +all their branches, databases, and endpoints. All data stored in them will be permanently lost: + delete resources.postgres_projects.my_project + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle/default + +Warn: destroying resources.postgres_projects.my_project: retrying after 500 Internal Server Error from DELETE /api/2.0/postgres/projects/test-pg-proj +Destroy: 1 deleted + +>>> print_requests.py --method DELETE --summary //postgres +DELETE /api/2.0/postgres/projects/test-pg-proj +DELETE /api/2.0/postgres/projects/test-pg-proj diff --git a/acceptance/bundle/resources/postgres_projects/delete_transient_500/script b/acceptance/bundle/resources/postgres_projects/delete_transient_500/script new file mode 100644 index 00000000000..16b8da255eb --- /dev/null +++ b/acceptance/bundle/resources/postgres_projects/delete_transient_500/script @@ -0,0 +1,10 @@ +trace $CLI bundle deploy + +# The delete lands but the response is a transient 500. The retry gets a NotFound, which counts as +# deleted, so the destroy succeeds. +fault.py --after-handler "DELETE /api/2.0/postgres/projects/*" 500 0 1 INTERNAL_ERROR + +trace $CLI bundle destroy --auto-approve + +# Two delete requests: the one that landed and the retry. +trace print_requests.py --method DELETE --summary //postgres diff --git a/acceptance/bundle/resources/postgres_projects/delete_transient_500/test.toml b/acceptance/bundle/resources/postgres_projects/delete_transient_500/test.toml new file mode 100644 index 00000000000..ab3f407bfa7 --- /dev/null +++ b/acceptance/bundle/resources/postgres_projects/delete_transient_500/test.toml @@ -0,0 +1,6 @@ +Cloud = false +RecordRequests = true +EnvMatrix.READPLAN = [] +Env.DATABRICKS_BUNDLE_ENGINE = "direct" +Env.DATABRICKS_BUNDLE_RETRY_INTERVAL_MS = "100" +SuppressLines.warn_retry = "" diff --git a/bundle/direct/apply.go b/bundle/direct/apply.go index 5b093a61d4c..2683c6c1532 100644 --- a/bundle/direct/apply.go +++ b/bundle/direct/apply.go @@ -299,7 +299,10 @@ func (d *DeploymentUnit) Delete(ctx context.Context, db *dstate.DeploymentState, return err } - err = d.Adapter.DoDelete(ctx, oldID, oldState) + // Delete is idempotent: if a failed attempt did delete the resource, the retry gets a NotFound. + _, err = retry.With(ctx, retry.IsTransient, func() (struct{}, error) { + return struct{}{}, d.Adapter.DoDelete(ctx, oldID, oldState) + }) if err != nil && !apierr.IsMissing(err) && !isManagedByParent(err) { // Rather than failing delete and requiring user to unbind, we perform unbind automatically there. // Some services, e.g. jobs, return 403 for missing resources if caller did not have permissions to it when job existed.