Create deployment / version when acquiring a lock on a deployment

acceptance/bundle/dms/enable-after-deploy/databricks.yml

@@ -0,0 +1,7 @@
+bundle:
+  name: dms-enable-after-deploy
+
+resources:
+  jobs:
+    foo:
+      name: foo

acceptance/bundle/dms/enable-after-deploy/output.txt

@@ -0,0 +1,88 @@
+
+=== Deploy without the feature: no DMS calls are made
+>>> [CLI] bundle deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-enable-after-deploy/default/files...
+Deploying resources...
+Updating deployment state...
+Deployment complete!
+
+>>> print_requests.py //api/2.0/bundle --keep
+
+>>> print_requests.py //deploy.lock
+{
+  "method": "POST",
+  "path": "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/dms-enable-after-deploy/default/state/deploy.lock",
+  "q": {
+    "overwrite": "false"
+  },
+  "body": {
+    "ID": "[UUID]",
+    "AcquisitionTime": "[TIMESTAMP]",
+    "IsForced": false,
+    "User": "[USERNAME]"
+  }
+}
+
+=== Enable experimental.record_deployment_history
+=== Delete local cache (.databricks) so the next deploy cannot rely on it
+=== Next deploy creates a DMS deployment + version 1 (lineage recovered from remote state, not local cache)
+>>> [CLI] bundle deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-enable-after-deploy/default/files...
+Deploying resources...
+Updating deployment state...
+Deployment complete!
+
+>>> print_requests.py //api/2.0/bundle --keep
+{
+  "method": "POST",
+  "path": "/api/2.0/bundle/deployments",
+  "q": {
+    "deployment_id": "[UUID]"
+  },
+  "body": {
+    "target_name": "default"
+  }
+}
+{
+  "method": "POST",
+  "path": "/api/2.0/bundle/deployments/[UUID]/versions",
+  "q": {
+    "version_id": "1"
+  },
+  "body": {
+    "cli_version": "[DEV_VERSION]",
+    "target_name": "default",
+    "version_type": "VERSION_TYPE_DEPLOY"
+  }
+}
+{
+  "method": "POST",
+  "path": "/api/2.0/bundle/deployments/[UUID]/versions/1/complete",
+  "body": {
+    "completion_reason": "VERSION_COMPLETE_SUCCESS"
+  }
+}
+
+>>> print_requests.py //deploy.lock
+{
+  "method": "POST",
+  "path": "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/dms-enable-after-deploy/default/state/deploy.lock",
+  "q": {
+    "overwrite": "false"
+  },
+  "body": {
+    "ID": "[UUID]",
+    "AcquisitionTime": "[TIMESTAMP]",
+    "IsForced": false,
+    "User": "[USERNAME]"
+  }
+}
+
+>>> [CLI] bundle destroy --auto-approve
+The following resources will be deleted:
+  delete resources.jobs.foo
+
+All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/dms-enable-after-deploy/default
+
+Deleting files...
+Destroy complete!

acceptance/bundle/dms/enable-after-deploy/script

@@ -0,0 +1,25 @@
+cleanup() {
+    trace $CLI bundle destroy --auto-approve
+    rm -f out.requests.txt
+}
+trap cleanup EXIT
+
+title "Deploy without the feature: no DMS calls are made"
+trace $CLI bundle deploy
+trace print_requests.py //api/2.0/bundle --keep
+trace print_requests.py //deploy.lock
+
+title "Enable experimental.record_deployment_history"
+cat >> databricks.yml <<'YAML'
+
+experimental:
+  record_deployment_history: true
+YAML
+
+title "Delete local cache (.databricks) so the next deploy cannot rely on it"
+rm -rf .databricks
+
+title "Next deploy creates a DMS deployment + version 1 (lineage recovered from remote state, not local cache)"
+trace $CLI bundle deploy
+trace print_requests.py //api/2.0/bundle --keep
+trace print_requests.py //deploy.lock

acceptance/bundle/dms/record/databricks.yml

@@ -0,0 +1,10 @@
+bundle:
+  name: dms-record
+
+experimental:
+  record_deployment_history: true
+
+resources:
+  jobs:
+    foo:
+      name: foo

acceptance/bundle/dms/record/output.txt

@@ -0,0 +1,148 @@
+
+=== Deploy: a DMS deployment + version are recorded while the file lock is held
+>>> [CLI] bundle deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-record/default/files...
+Deploying resources...
+Updating deployment state...
+Deployment complete!
+
+=== DMS API calls made during deploy (create deployment, create + complete version)
+>>> print_requests.py //api/2.0/bundle --keep
+{
+  "method": "POST",
+  "path": "/api/2.0/bundle/deployments",
+  "q": {
+    "deployment_id": "[UUID]"
+  },
+  "body": {
+    "target_name": "default"
+  }
+}
+{
+  "method": "POST",
+  "path": "/api/2.0/bundle/deployments/[UUID]/versions",
+  "q": {
+    "version_id": "1"
+  },
+  "body": {
+    "cli_version": "[DEV_VERSION]",
+    "target_name": "default",
+    "version_type": "VERSION_TYPE_DEPLOY"
+  }
+}
+{
+  "method": "POST",
+  "path": "/api/2.0/bundle/deployments/[UUID]/versions/1/complete",
+  "body": {
+    "completion_reason": "VERSION_COMPLETE_SUCCESS"
+  }
+}
+
+=== The workspace-filesystem lock is applied alongside DMS (deploy.lock written)
+>>> print_requests.py //deploy.lock
+{
+  "method": "POST",
+  "path": "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/dms-record/default/state/deploy.lock",
+  "q": {
+    "overwrite": "false"
+  },
+  "body": {
+    "ID": "[UUID]",
+    "AcquisitionTime": "[TIMESTAMP]",
+    "IsForced": false,
+    "User": "[USERNAME]"
+  }
+}
+
+=== Redeploy after deleting local cache (.databricks): the lineage is recovered from remote state, so the same deployment is reused and the version increments (no new CreateDeployment)
+>>> [CLI] bundle deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-record/default/files...
+Deploying resources...
+Updating deployment state...
+Deployment complete!
+
+>>> print_requests.py //api/2.0/bundle --keep
+{
+  "method": "POST",
+  "path": "/api/2.0/bundle/deployments/[UUID]/versions",
+  "q": {
+    "version_id": "2"
+  },
+  "body": {
+    "cli_version": "[DEV_VERSION]",
+    "target_name": "default",
+    "version_type": "VERSION_TYPE_DEPLOY"
+  }
+}
+{
+  "method": "POST",
+  "path": "/api/2.0/bundle/deployments/[UUID]/versions/2/complete",
+  "body": {
+    "completion_reason": "VERSION_COMPLETE_SUCCESS"
+  }
+}
+
+>>> print_requests.py //deploy.lock
+{
+  "method": "POST",
+  "path": "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/dms-record/default/state/deploy.lock",
+  "q": {
+    "overwrite": "false"
+  },
+  "body": {
+    "ID": "[UUID]",
+    "AcquisitionTime": "[TIMESTAMP]",
+    "IsForced": false,
+    "User": "[USERNAME]"
+  }
+}
+
+=== Destroy: a DMS version is recorded while the file lock is held
+>>> [CLI] bundle destroy --auto-approve
+The following resources will be deleted:
+  delete resources.jobs.foo
+
+All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/dms-record/default
+
+Deleting files...
+Destroy complete!
+
+>>> print_requests.py //api/2.0/bundle --keep
+{
+  "method": "POST",
+  "path": "/api/2.0/bundle/deployments/[UUID]/versions",
+  "q": {
+    "version_id": "3"
+  },
+  "body": {
+    "cli_version": "[DEV_VERSION]",
+    "target_name": "default",
+    "version_type": "VERSION_TYPE_DESTROY"
+  }
+}
+{
+  "method": "POST",
+  "path": "/api/2.0/bundle/deployments/[UUID]/versions/3/complete",
+  "body": {
+    "completion_reason": "VERSION_COMPLETE_SUCCESS"
+  }
+}
+{
+  "method": "DELETE",
+  "path": "/api/2.0/bundle/deployments/[UUID]"
+}
+
+>>> print_requests.py //deploy.lock
+{
+  "method": "POST",
+  "path": "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/dms-record/default/state/deploy.lock",
+  "q": {
+    "overwrite": "false"
+  },
+  "body": {
+    "ID": "[UUID]",
+    "AcquisitionTime": "[TIMESTAMP]",
+    "IsForced": false,
+    "User": "[USERNAME]"
+  }
+}

acceptance/bundle/dms/record/script

@@ -0,0 +1,23 @@
+cleanup() {
+    title "Destroy: a DMS version is recorded while the file lock is held"
+    trace $CLI bundle destroy --auto-approve
+    trace print_requests.py //api/2.0/bundle --keep
+    trace print_requests.py //deploy.lock
+    rm -f out.requests.txt
+}
+trap cleanup EXIT
+
+title "Deploy: a DMS deployment + version are recorded while the file lock is held"
+trace $CLI bundle deploy
+
+title "DMS API calls made during deploy (create deployment, create + complete version)"
+trace print_requests.py //api/2.0/bundle --keep
+
+title "The workspace-filesystem lock is applied alongside DMS (deploy.lock written)"
+trace print_requests.py //deploy.lock
+
+title "Redeploy after deleting local cache (.databricks): the lineage is recovered from remote state, so the same deployment is reused and the version increments (no new CreateDeployment)"
+rm -rf .databricks
+trace $CLI bundle deploy
+trace print_requests.py //api/2.0/bundle --keep
+trace print_requests.py //deploy.lock

acceptance/bundle/dms/test.toml

@@ -0,0 +1,5 @@
+# Deployment Metadata Service (DMS) recording is only meaningful in the direct
+# engine, where the deployment ID is derived from the state lineage.
+EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"]
+
+RecordRequests = true