From ddef46e7eb2e4a33f4d60248bfe281dd265abb5e Mon Sep 17 00:00:00 2001 From: Pieter Noordhuis Date: Thu, 4 Jun 2026 16:29:24 +0200 Subject: [PATCH] acceptance: make generate/*_and_deploy bundle names unique Append -$UNIQUE_NAME to the bundle name in generate/pipeline_and_deploy and python_job_and_deploy (via databricks.yml.tmpl + envsubst). The static bundle name made all parallel cloud variants deploy to the same path, causing deploy-lock contention where all but one variant failed. Co-authored-by: Isaac --- acceptance/bundle/generate/pipeline_and_deploy/databricks.yml | 2 -- .../bundle/generate/pipeline_and_deploy/databricks.yml.tmpl | 2 ++ acceptance/bundle/generate/pipeline_and_deploy/output.txt | 4 ++-- acceptance/bundle/generate/pipeline_and_deploy/script | 2 ++ .../bundle/generate/python_job_and_deploy/databricks.yml | 2 -- .../bundle/generate/python_job_and_deploy/databricks.yml.tmpl | 2 ++ acceptance/bundle/generate/python_job_and_deploy/output.txt | 4 ++-- acceptance/bundle/generate/python_job_and_deploy/script | 2 ++ 8 files changed, 12 insertions(+), 8 deletions(-) delete mode 100644 acceptance/bundle/generate/pipeline_and_deploy/databricks.yml create mode 100644 acceptance/bundle/generate/pipeline_and_deploy/databricks.yml.tmpl delete mode 100644 acceptance/bundle/generate/python_job_and_deploy/databricks.yml create mode 100644 acceptance/bundle/generate/python_job_and_deploy/databricks.yml.tmpl diff --git a/acceptance/bundle/generate/pipeline_and_deploy/databricks.yml b/acceptance/bundle/generate/pipeline_and_deploy/databricks.yml deleted file mode 100644 index 15b90fec1bb..00000000000 --- a/acceptance/bundle/generate/pipeline_and_deploy/databricks.yml +++ /dev/null @@ -1,2 +0,0 @@ -bundle: - name: pipeline_and_deploy diff --git a/acceptance/bundle/generate/pipeline_and_deploy/databricks.yml.tmpl b/acceptance/bundle/generate/pipeline_and_deploy/databricks.yml.tmpl new file mode 100644 index 00000000000..487b54402f8 --- /dev/null +++ b/acceptance/bundle/generate/pipeline_and_deploy/databricks.yml.tmpl @@ -0,0 +1,2 @@ +bundle: + name: pipeline_and_deploy-$UNIQUE_NAME diff --git a/acceptance/bundle/generate/pipeline_and_deploy/output.txt b/acceptance/bundle/generate/pipeline_and_deploy/output.txt index 25943c80bcf..41c92d9ac10 100644 --- a/acceptance/bundle/generate/pipeline_and_deploy/output.txt +++ b/acceptance/bundle/generate/pipeline_and_deploy/output.txt @@ -13,13 +13,13 @@ Pipeline configuration successfully saved to resources/out.pipeline.yml === Verify generated yaml has expected fields === Deploy the generated bundle >>> [CLI] bundle deploy -Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/pipeline_and_deploy/default/files... +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/pipeline_and_deploy-[UNIQUE_NAME]/default/files... Deploying resources... Deployment complete! === Destroy the deployed bundle >>> [CLI] bundle destroy --auto-approve -All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/pipeline_and_deploy/default +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/pipeline_and_deploy-[UNIQUE_NAME]/default Deleting files... Destroy complete! diff --git a/acceptance/bundle/generate/pipeline_and_deploy/script b/acceptance/bundle/generate/pipeline_and_deploy/script index cd047231113..85d9e044d5f 100644 --- a/acceptance/bundle/generate/pipeline_and_deploy/script +++ b/acceptance/bundle/generate/pipeline_and_deploy/script @@ -1,3 +1,5 @@ +envsubst < databricks.yml.tmpl > databricks.yml + title "Upload files to workspace" trace $CLI workspace import "/Workspace/Users/${CURRENT_USER_NAME}/notebook.py" --file notebook.py --format AUTO --overwrite trace $CLI workspace import "/Workspace/Users/${CURRENT_USER_NAME}/test.py" --file test.py --format AUTO --overwrite diff --git a/acceptance/bundle/generate/python_job_and_deploy/databricks.yml b/acceptance/bundle/generate/python_job_and_deploy/databricks.yml deleted file mode 100644 index d7f9b4f9454..00000000000 --- a/acceptance/bundle/generate/python_job_and_deploy/databricks.yml +++ /dev/null @@ -1,2 +0,0 @@ -bundle: - name: python_job_and_deploy diff --git a/acceptance/bundle/generate/python_job_and_deploy/databricks.yml.tmpl b/acceptance/bundle/generate/python_job_and_deploy/databricks.yml.tmpl new file mode 100644 index 00000000000..9d3cc69cdec --- /dev/null +++ b/acceptance/bundle/generate/python_job_and_deploy/databricks.yml.tmpl @@ -0,0 +1,2 @@ +bundle: + name: python_job_and_deploy-$UNIQUE_NAME diff --git a/acceptance/bundle/generate/python_job_and_deploy/output.txt b/acceptance/bundle/generate/python_job_and_deploy/output.txt index 418e008c535..aaa39b42448 100644 --- a/acceptance/bundle/generate/python_job_and_deploy/output.txt +++ b/acceptance/bundle/generate/python_job_and_deploy/output.txt @@ -12,13 +12,13 @@ Job configuration successfully saved to resources/out.job.yml === Verify generated yaml has expected fields === Deploy the generated bundle >>> [CLI] bundle deploy -Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/python_job_and_deploy/default/files... +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/python_job_and_deploy-[UNIQUE_NAME]/default/files... Deploying resources... Deployment complete! === Destroy the deployed bundle >>> [CLI] bundle destroy --auto-approve -All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/python_job_and_deploy/default +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/python_job_and_deploy-[UNIQUE_NAME]/default Deleting files... Destroy complete! diff --git a/acceptance/bundle/generate/python_job_and_deploy/script b/acceptance/bundle/generate/python_job_and_deploy/script index 1ae0e2142f2..8037948ed69 100644 --- a/acceptance/bundle/generate/python_job_and_deploy/script +++ b/acceptance/bundle/generate/python_job_and_deploy/script @@ -1,3 +1,5 @@ +envsubst < databricks.yml.tmpl > databricks.yml + title "Upload notebook to a workspace path" trace $CLI workspace import "/Workspace/Users/${CURRENT_USER_NAME}/test_notebook.py" --file test_notebook.py --format AUTO --overwrite