Integrate deployment metadata service for locking and state

acceptance/bin/print_requests.py

@@ -168,7 +168,7 @@ def main():
     if not requests_file.exists():
         sys.exit(f"File {requests_file.as_posix()} not found")
 
-    with open(requests_file) as fobj:
+    with open(requests_file, encoding="utf-8") as fobj:
         data = fobj.read()
 
     if not data:

acceptance/bundle/dms/add-resources/databricks.yml

@@ -0,0 +1,7 @@
+bundle:
+  name: add-resources-test
+
+resources:
+  jobs:
+    job_a:
+      name: job-a

acceptance/bundle/dms/add-resources/output.txt

@@ -0,0 +1,149 @@
+
+>>> [CLI] bundle deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/add-resources-test/default/files...
+Deploying resources...
+Deployment complete!
+
+>>> [CLI] bundle deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/add-resources-test/default/files...
+Deploying resources...
+Deployment complete!
+
+>>> [CLI] bundle plan
+update jobs.job_a
+update jobs.job_b
+
+Plan: 0 to add, 2 to change, 0 to delete, 0 unchanged
+
+>>> print_requests.py --get --sort //bundle ^//workspace-files ^//import-file
+{
+  "method": "GET",
+  "path": "/api/2.0/bundle/deployments/[UUID]"
+}
+{
+  "method": "GET",
+  "path": "/api/2.0/bundle/deployments/[UUID]/resources",
+  "q": {
+    "page_size": "1000"
+  },
+  "body": null
+}
+{
+  "method": "GET",
+  "path": "/api/2.0/bundle/deployments/[UUID]/resources",
+  "q": {
+    "page_size": "1000"
+  },
+  "body": null
+}
+{
+  "method": "POST",
+  "path": "/api/2.0/bundle/deployments",
+  "q": {
+    "deployment_id": "[UUID]"
+  },
+  "body": {
+    "display_name": "add-resources-test",
+    "target_name": "default"
+  }
+}
+{
+  "method": "POST",
+  "path": "/api/2.0/bundle/deployments/[UUID]/versions",
+  "q": {
+    "version_id": "1"
+  },
+  "body": {
+    "cli_version": "[DEV_VERSION]",
+    "version_type": "VERSION_TYPE_DEPLOY",
+    "display_name": "add-resources-test",
+    "target_name": "default",
+    "git_info": {}
+  }
+}
+{
+  "method": "POST",
+  "path": "/api/2.0/bundle/deployments/[UUID]/versions",
+  "q": {
+    "version_id": "2"
+  },
+  "body": {
+    "cli_version": "[DEV_VERSION]",
+    "version_type": "VERSION_TYPE_DEPLOY",
+    "display_name": "add-resources-test",
+    "target_name": "default",
+    "git_info": {}
+  }
+}
+{
+  "method": "POST",
+  "path": "/api/2.0/bundle/deployments/[UUID]/versions/1/complete",
+  "body": {
+    "name": "deployments/[UUID]/versions/1",
+    "completion_reason": "VERSION_COMPLETE_SUCCESS"
+  }
+}
+{
+  "method": "POST",
+  "path": "/api/2.0/bundle/deployments/[UUID]/versions/1/operations",
+  "q": {
+    "resource_key": "jobs.job_a"
+  },
+  "body": {
+    "resource_key": "jobs.job_a",
+    "action_type": "OPERATION_ACTION_TYPE_CREATE",
+    "state": {
+      "deployment": {
+        "deployment_id": "[UUID]",
+        "kind": "BUNDLE",
+        "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/add-resources-test/default/state/metadata.json",
+        "version_id": "1"
+      },
+      "edit_mode": "UI_LOCKED",
+      "format": "MULTI_TASK",
+      "max_concurrent_runs": 1,
+      "name": "job-a",
+      "queue": {
+        "enabled": true
+      }
+    },
+    "resource_id": "[NUMID]",
+    "status": "OPERATION_STATUS_SUCCEEDED"
+  }
+}
+{
+  "method": "POST",
+  "path": "/api/2.0/bundle/deployments/[UUID]/versions/2/complete",
+  "body": {
+    "name": "deployments/[UUID]/versions/2",
+    "completion_reason": "VERSION_COMPLETE_SUCCESS"
+  }
+}
+{
+  "method": "POST",
+  "path": "/api/2.0/bundle/deployments/[UUID]/versions/2/operations",
+  "q": {
+    "resource_key": "jobs.job_b"
+  },
+  "body": {
+    "resource_key": "jobs.job_b",
+    "action_type": "OPERATION_ACTION_TYPE_CREATE",
+    "state": {
+      "deployment": {
+        "deployment_id": "[UUID]",
+        "kind": "BUNDLE",
+        "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/add-resources-test/default/state/metadata.json",
+        "version_id": "2"
+      },
+      "edit_mode": "UI_LOCKED",
+      "format": "MULTI_TASK",
+      "max_concurrent_runs": 1,
+      "name": "job-b",
+      "queue": {
+        "enabled": true
+      }
+    },
+    "resource_id": "[NUMID]",
+    "status": "OPERATION_STATUS_SUCCEEDED"
+  }
+}

acceptance/bundle/dms/add-resources/script

@@ -0,0 +1,34 @@
+# Deploy with one job.
+trace $CLI bundle deploy
+
+# Delete local cache to force reading state from DMS.
+rm -rf .databricks
+
+# Add a second job and deploy again.
+cat > databricks.yml << 'EOF'
+bundle:
+  name: add-resources-test
+
+resources:
+  jobs:
+    job_a:
+      name: job-a
+    job_b:
+      name: job-b
+EOF
+trace $CLI bundle deploy
+
+# Delete local cache again and run plan — should show no changes.
+rm -rf .databricks
+trace $CLI bundle plan
+
+# Print metadata service requests. Should show:
+# - Deploy 1: CREATE for job_a
+# - Deploy 2: ListResources + CREATE for job_b (job_a is unchanged)
+# - Plan: ListResources (no operations)
+trace print_requests.py --get --sort //bundle ^//workspace-files ^//import-file
+
+# Clean up.
+rm -rf .databricks
+$CLI bundle destroy --auto-approve > /dev/null 2>&1
+print_requests.py --get > /dev/null 2>&1 || true

acceptance/bundle/dms/add-resources/test.toml

@@ -0,0 +1 @@
+Ignore = [".databricks"]

acceptance/bundle/dms/deploy-error/databricks.yml

@@ -0,0 +1,7 @@
+bundle:
+  name: metadata-service-error-test
+
+resources:
+  jobs:
+    test_job:
+      name: test-job

acceptance/bundle/dms/deploy-error/output.txt

@@ -0,0 +1,74 @@
+
+>>> musterr [CLI] bundle deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/metadata-service-error-test/default/files...
+Deploying resources...
+Error: cannot create resources.jobs.test_job: Invalid job configuration. (400 INVALID_PARAMETER_VALUE)
+
+Endpoint: POST [DATABRICKS_URL]/api/2.2/jobs/create
+HTTP Status: 400 Bad Request
+API error_code: INVALID_PARAMETER_VALUE
+API message: Invalid job configuration.
+
+
+>>> print_requests.py --get //bundle ^//workspace-files ^//import-file
+{
+  "method": "POST",
+  "path": "/api/2.0/bundle/deployments",
+  "q": {
+    "deployment_id": "[UUID]"
+  },
+  "body": {
+    "display_name": "metadata-service-error-test",
+    "target_name": "default"
+  }
+}
+{
+  "method": "POST",
+  "path": "/api/2.0/bundle/deployments/[UUID]/versions",
+  "q": {
+    "version_id": "1"
+  },
+  "body": {
+    "cli_version": "[DEV_VERSION]",
+    "version_type": "VERSION_TYPE_DEPLOY",
+    "display_name": "metadata-service-error-test",
+    "target_name": "default",
+    "git_info": {}
+  }
+}
+{
+  "method": "POST",
+  "path": "/api/2.0/bundle/deployments/[UUID]/versions/1/operations",
+  "q": {
+    "resource_key": "jobs.test_job"
+  },
+  "body": {
+    "resource_key": "jobs.test_job",
+    "action_type": "OPERATION_ACTION_TYPE_CREATE",
+    "state": {
+      "deployment": {
+        "deployment_id": "[UUID]",
+        "kind": "BUNDLE",
+        "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/metadata-service-error-test/default/state/metadata.json",
+        "version_id": "1"
+      },
+      "edit_mode": "UI_LOCKED",
+      "format": "MULTI_TASK",
+      "max_concurrent_runs": 1,
+      "name": "test-job",
+      "queue": {
+        "enabled": true
+      }
+    },
+    "status": "OPERATION_STATUS_FAILED",
+    "error_message": "Invalid job configuration."
+  }
+}
+{
+  "method": "POST",
+  "path": "/api/2.0/bundle/deployments/[UUID]/versions/1/complete",
+  "body": {
+    "name": "deployments/[UUID]/versions/1",
+    "completion_reason": "VERSION_COMPLETE_FAILURE"
+  }
+}

acceptance/bundle/dms/deploy-error/script

@@ -0,0 +1,6 @@
+# Deploy with the metadata service enabled, expecting a resource creation failure.
+trace musterr $CLI bundle deploy
+
+# Print the metadata service requests to verify the failed operation is reported.
+trace print_requests.py --get //bundle ^//workspace-files ^//import-file
+print_requests.py --get > /dev/null 2>&1 || true

acceptance/bundle/dms/deploy-error/test.toml

@@ -0,0 +1,9 @@
+Ignore = [".databricks"]
+EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"]
+EnvMatrix.DATABRICKS_BUNDLE_MANAGED_STATE = ["true"]
+RecordRequests = true
+
+[[Server]]
+Pattern = "POST /api/2.2/jobs/create"
+Response.StatusCode = 400
+Response.Body = '{"error_code": "INVALID_PARAMETER_VALUE", "message": "Invalid job configuration."}'

acceptance/bundle/dms/plan-and-summary/databricks.yml

@@ -0,0 +1,7 @@
+bundle:
+  name: plan-summary-test
+
+resources:
+  jobs:
+    test_job:
+      name: test-job