Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .nextchanges/bundles/record-bind-history.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
* Record a deployment history version for `bundle deployment bind` when deployment history is enabled. ([#7400](https://github.com/databricks/cli/pull/7400))
13 changes: 12 additions & 1 deletion acceptance/bin/nostamp.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,13 +7,14 @@
is a 2-space indent, keys in input order, <>& left unescaped, and integers at full precision.
Tests under bundle/dms assert the stamp itself and must not use it.

Removed in three shapes plus the plan header:
Removed in these shapes:
1. deployment_id/version_id nested in a deployment block, recognized by the
neighbouring "kind" and "metadata_file_path", which are kept; version_id
== "" is kept (a Terraform state dump carries that for an unstamped job).
2. changes entries keyed "deployment.deployment_id" / "deployment.version_id".
3. a changes object emptied by (2), or already empty.
4. the recorded plan header field features.
5. deployment_id/latest_version_id in bundle.deployment; other settings are kept.
"""

import argparse
Expand All @@ -23,6 +24,7 @@
_STAMP_KEYS = ("deployment_id", "version_id")
_CHANGE_KEYS = ("deployment.deployment_id", "deployment.version_id")
_PLAN_HEADER_KEYS = ("features",)
_BUNDLE_STAMP_KEYS = ("deployment_id", "latest_version_id")


def scrub(node):
Expand Down Expand Up @@ -52,6 +54,15 @@ def render(data, indent):
if isinstance(data, dict):
for k in _PLAN_HEADER_KEYS:
data.pop(k, None) # 4. plan header, present only at the root
# 5. Strip the summary's DMS IDs, keeping deployment settings such as locks.

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Running the DMS=true variants of the three scripts this PR pipes through nostamp (bind/pipelines/update, bind/pipelines/recreate, bind/dashboard/recreation) with the scripts at their base revision passes locally at this HEAD: after a first bind there is no marker, so summary and the state dump never carry bundle.deployment IDs. What difference is shape 5 and these three pipes guarding against, a cloud run or a later PR in the stack? If the latter, can they move there?

bundle = data.get("bundle")
if isinstance(bundle, dict):
deployment = bundle.get("deployment")
if isinstance(deployment, dict) and any(k in deployment for k in _BUNDLE_STAMP_KEYS):
for k in _BUNDLE_STAMP_KEYS:
deployment.pop(k, None)
if not deployment:
bundle.pop("deployment")
return json.dumps(data, indent=indent, ensure_ascii=False, separators=(",", ": "))


Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ cleanupRemoveDashboard() {
trap cleanupRemoveDashboard EXIT

trace $CLI bundle deployment bind dashboard1 "${DASHBOARD_ID}" --auto-approve
trace print_state.py --no-dms > out.state_after_bind.json
trace print_state.py --no-dms | nostamp.py --indent 1 > out.state_after_bind.json
ETAG=$(jq '.state["resources.dashboards.dashboard1"].state.etag // .resources[0].instances[0].attributes.etag // "no-etag"' < out.state_after_bind.json)
echo $ETAG
add_repl $ETAG ETAG
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@

>>> [CLI] bundle deployment bind spec1 main.fspec_[UNIQUE_NAME].boundspec --auto-approve
Successfully bound feature_spec with an id 'main.fspec_[UNIQUE_NAME].boundspec'
Run 'bundle deploy' to deploy changes to your workspace
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@

>>> [CLI] bundle deployment bind spec1 main.fspec_[UNIQUE_NAME].boundspec --auto-approve
Warn: Failed to record bind history: failed to create deployment version: operations.resource_key must have a known resource type prefix (e.g. 'jobs.', 'pipelines.'): feature_specs.spec1
Successfully bound feature_spec with an id 'main.fspec_[UNIQUE_NAME].boundspec'
Run 'bundle deploy' to deploy changes to your workspace
Original file line number Diff line number Diff line change
@@ -1,2 +1,3 @@
Cloud = true
EnvMatrix.DMS = ["", "true"]
EnvMatrix.OUTPUT_MODE = ["plain", "recorded"]
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@

>>> [CLI] bundle deployment unbind spec1
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@

>>> [CLI] bundle deployment unbind spec1
6 changes: 0 additions & 6 deletions acceptance/bundle/deployment/bind/feature_spec/output.txt
Original file line number Diff line number Diff line change
Expand Up @@ -16,10 +16,6 @@
"name": "main.fspec_[UNIQUE_NAME].boundspec"
}

>>> [CLI] bundle deployment bind spec1 main.fspec_[UNIQUE_NAME].boundspec --auto-approve
Successfully bound feature_spec with an id 'main.fspec_[UNIQUE_NAME].boundspec'
Run 'bundle deploy' to deploy changes to your workspace

>>> [CLI] bundle summary
Name: test-bind-feature-spec-[UNIQUE_NAME]
Target: default
Expand All @@ -37,8 +33,6 @@ Resources:
>>> [CLI] bundle plan
Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged

>>> [CLI] bundle deployment unbind spec1

>>> [CLI] bundle summary
Name: test-bind-feature-spec-[UNIQUE_NAME]
Target: default
Expand Down
4 changes: 2 additions & 2 deletions acceptance/bundle/deployment/bind/feature_spec/script
Original file line number Diff line number Diff line change
Expand Up @@ -6,13 +6,13 @@ trace $CLI feature-engineering create-feature-spec --json "{
\"features\": [{\"feature_lookup\": {\"table\": \"$SCHEMA_FULL_NAME.user_features\", \"lookup_keys\": [\"user_id\"], \"feature_names\": [\"age\"]}}]
}" | jq '{name}'

trace $CLI bundle deployment bind spec1 $SCHEMA_FULL_NAME.boundspec --auto-approve
trace $CLI bundle deployment bind spec1 $SCHEMA_FULL_NAME.boundspec --auto-approve &> out.bind.$OUTPUT_MODE.txt
trace $CLI bundle summary

title "The bound feature spec matches the config, so there is nothing to change"
trace $CLI bundle plan

trace $CLI bundle deployment unbind spec1
trace $CLI bundle deployment unbind spec1 &> out.unbind.$OUTPUT_MODE.txt
trace $CLI bundle summary

trace $CLI feature-engineering delete-feature-spec $SCHEMA_FULL_NAME.boundspec
6 changes: 6 additions & 0 deletions acceptance/bundle/deployment/bind/feature_spec/test.toml
Original file line number Diff line number Diff line change
@@ -1,3 +1,9 @@
# Keep history warnings in separate outputs.
EnvMatrix.OUTPUT_MODE = ["plain", "recorded"]
EnvMatrixExclude.plain_history = ["DMS=true", "OUTPUT_MODE=plain"]
EnvMatrixExclude.recorded_without_history = ["DMS=", "OUTPUT_MODE=recorded"]
EnvVaryOutput = "OUTPUT_MODE"

Ignore = [
".databricks",
]
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ rm -f "$OUT_REQUESTS"
trace $CLI bundle deployment bind foo $NEW_PIPELINE_ID --auto-approve &> out.bind-success.txt
trace print_requests.py '^//import-file/' '^//workspace/'

trace $CLI bundle summary -o json | jq .resources > out.summary.json

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

added because now the summary contains deployment and version ID

trace $CLI bundle summary -o json | jq .resources | nostamp.py > out.summary.json
trace $CLI bundle plan

trace musterr $CLI bundle deploy
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ rm -f "$OUT_REQUESTS"
trace $CLI bundle deployment bind foo $NEW_PIPELINE_ID --auto-approve &> out.bind-success.txt
print_requests.py '^//import-file/' '^//workspace/delete'

trace $CLI bundle summary -o json > out.summary.json
trace $CLI bundle summary -o json | nostamp.py > out.summary.json

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

updated nostamp.py to scrub the deployment ID from the summary output as well.

trace $CLI bundle plan

rm -f "$OUT_REQUESTS"
Expand Down
13 changes: 13 additions & 0 deletions acceptance/bundle/dms/bind-history-failures/databricks.yml.tmpl
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
bundle:
name: dms-bind-history-failures-$UNIQUE_NAME

resources:
jobs:
create_failed:
name: create_failed
record_failed:
name: record_failed
complete_failed:
name: complete_failed
upload_failed:
name: upload_failed
2 changes: 2 additions & 0 deletions acceptance/bundle/dms/bind-history-failures/out.test.toml
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
Cloud = false
EnvMatrix.DMS = ["true"]
79 changes: 79 additions & 0 deletions acceptance/bundle/dms/bind-history-failures/output.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,79 @@

=== A version creation failure does not prevent binding

>>> fault.py POST /api/2.0/bundle/deployments/*/versions 400 0 1

>>> [CLI] bundle deployment bind create_failed [JOB_ID] --auto-approve
Warn: Failed to record bind history: failed to create deployment version: Fault injected by test.
Successfully bound job with an id '[JOB_ID]'
Run 'bundle deploy' to deploy changes to your workspace

=== An operation recording failure does not prevent binding

>>> fault.py PATCH /api/2.0/bundle/** 400 0 1

>>> [CLI] bundle deployment bind record_failed [JOB_ID] --auto-approve
Warn: Failed to record deployment history: recording operation for resources.jobs.record_failed: Fault injected by test.
Successfully bound job with an id '[JOB_ID]'
Run 'bundle deploy' to deploy changes to your workspace

=== A version completion failure does not prevent binding

>>> fault.py POST /api/2.0/bundle/deployments/*/versions/*/complete 400 0 1

>>> [CLI] bundle deployment bind complete_failed [JOB_ID] --auto-approve
Warn: Failed to complete deployment history version: Fault injected by test.
Successfully bound job with an id '[JOB_ID]'
Run 'bundle deploy' to deploy changes to your workspace

=== All bindings survived the remote state round trips

>>> jq .state | map_values(.__id__) .databricks/bundle/default/resources.json
{
"resources.jobs.complete_failed": "[JOB_ID]",
"resources.jobs.create_failed": "[JOB_ID]",
"resources.jobs.record_failed": "[JOB_ID]"
}

>>> [CLI] jobs get [JOB_ID]
{
"job_id": [JOB_ID],
"name": "bind-target-[UNIQUE_NAME]"
}

>>> print_requests.py --dms //versions --oneline
{"method": "POST", "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions", "q": {"version_id": "1"}, "body": {"cli_version": "[CLI_VERSION]", "operations": [{"action_type": "OPERATION_ACTION_TYPE_UPDATE", "resource_key": "jobs.create_failed"}], "version_type": "VERSION_TYPE_DEPLOY"}}
{"method": "POST", "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions", "q": {"version_id": "2"}, "body": {"cli_version": "[CLI_VERSION]", "operations": [{"action_type": "OPERATION_ACTION_TYPE_UPDATE", "resource_key": "jobs.record_failed"}], "version_type": "VERSION_TYPE_DEPLOY"}}
{"method": "PATCH", "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions/2/operations/jobs.record_failed", "q": {"update_mask": "state,error_message,resource_id,status"}, "body": {"error_message": "", "resource_id": "[JOB_ID]", "sequence_id": 0, "state": "{\"state\":{\"deployment\":{\"kind\":\"BUNDLE\",\"metadata_file_path\":\"/Workspace/Users/[USERNAME]/.bundle/dms-bind-history-failures-[UNIQUE_NAME]/default/state/metadata.json\"},\"edit_mode\":\"UI_LOCKED\",\"format\":\"MULTI_TASK\",\"max_concurrent_runs\":1,\"name\":\"record_failed\",\"queue\":{\"enabled\":true}}}", "status": "OPERATION_STATUS_SUCCEEDED"}}
{"method": "POST", "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions/2/complete", "body": {"completion_reason": "VERSION_COMPLETE_FAILURE"}}
{"method": "POST", "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions", "q": {"version_id": "3"}, "body": {"cli_version": "[CLI_VERSION]", "operations": [{"action_type": "OPERATION_ACTION_TYPE_UPDATE", "resource_key": "jobs.complete_failed"}], "previous_version_id": "2", "version_type": "VERSION_TYPE_DEPLOY"}}
{"method": "PATCH", "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions/3/operations/jobs.complete_failed", "q": {"update_mask": "state,error_message,resource_id,status"}, "body": {"error_message": "", "resource_id": "[JOB_ID]", "sequence_id": 0, "state": "{\"state\":{\"deployment\":{\"kind\":\"BUNDLE\",\"metadata_file_path\":\"/Workspace/Users/[USERNAME]/.bundle/dms-bind-history-failures-[UNIQUE_NAME]/default/state/metadata.json\"},\"edit_mode\":\"UI_LOCKED\",\"format\":\"MULTI_TASK\",\"max_concurrent_runs\":1,\"name\":\"complete_failed\",\"queue\":{\"enabled\":true}}}", "status": "OPERATION_STATUS_SUCCEEDED"}}
{"method": "POST", "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions/3/complete", "body": {"completion_reason": "VERSION_COMPLETE_SUCCESS"}}

=== A remote state upload failure fails the bind version

>>> fault.py POST /api/2.0/workspace-files/import-file/**resources.json 400 0 1

>>> musterr [CLI] bundle deployment bind upload_failed [JOB_ID] --auto-approve
Error: Fault injected by test. (400 INJECTED)

Endpoint: POST [DATABRICKS_URL]/api/2.0/workspace-files/import-file/Workspace%2FUsers%2F[USERNAME]%2F.bundle%2Fdms-bind-history-failures-[UNIQUE_NAME]%2Fdefault%2Fstate%2Fresources.json?overwrite=true
HTTP Status: 400 Bad Request
API error_code: INJECTED
API message: Fault injected by test.


>>> print_requests.py --dms //versions --oneline
{"method": "POST", "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions", "q": {"version_id": "4"}, "body": {"cli_version": "[CLI_VERSION]", "operations": [{"action_type": "OPERATION_ACTION_TYPE_UPDATE", "resource_key": "jobs.upload_failed"}], "previous_version_id": "3", "version_type": "VERSION_TYPE_DEPLOY"}}
{"method": "PATCH", "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions/4/operations/jobs.upload_failed", "q": {"update_mask": "state,error_message,resource_id,status"}, "body": {"error_message": "", "resource_id": "[JOB_ID]", "sequence_id": 0, "state": "{\"state\":{\"deployment\":{\"kind\":\"BUNDLE\",\"metadata_file_path\":\"/Workspace/Users/[USERNAME]/.bundle/dms-bind-history-failures-[UNIQUE_NAME]/default/state/metadata.json\"},\"edit_mode\":\"UI_LOCKED\",\"format\":\"MULTI_TASK\",\"max_concurrent_runs\":1,\"name\":\"upload_failed\",\"queue\":{\"enabled\":true}}}", "status": "OPERATION_STATUS_SUCCEEDED"}}
{"method": "POST", "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions/4/complete", "body": {"completion_reason": "VERSION_COMPLETE_FAILURE"}}

=== Remote state still contains only the successful bindings

>>> [CLI] bundle summary -o json
{
"complete_failed": "[JOB_ID]",
"create_failed": "[JOB_ID]",
"record_failed": "[JOB_ID]",
"upload_failed": null
}
33 changes: 33 additions & 0 deletions acceptance/bundle/dms/bind-history-failures/script
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
envsubst < databricks.yml.tmpl > databricks.yml
job_id=$($CLI jobs create --json '{"name": "bind-target-'$UNIQUE_NAME'"}' | jq -r '.job_id')
add_repl "$job_id" JOB_ID

title "A version creation failure does not prevent binding"
trace fault.py "POST /api/2.0/bundle/deployments/*/versions" 400 0 1
trace $CLI bundle deployment bind create_failed "$job_id" --auto-approve
rm -rf .databricks

title "An operation recording failure does not prevent binding"
trace fault.py "PATCH /api/2.0/bundle/**" 400 0 1
trace $CLI bundle deployment bind record_failed "$job_id" --auto-approve
rm -rf .databricks

title "A version completion failure does not prevent binding"
trace fault.py "POST /api/2.0/bundle/deployments/*/versions/*/complete" 400 0 1
trace $CLI bundle deployment bind complete_failed "$job_id" --auto-approve

title "All bindings survived the remote state round trips"
trace jq '.state | map_values(.__id__)' .databricks/bundle/default/resources.json
trace $CLI jobs get "$job_id" | jq '{job_id, name: .settings.name}'
trace print_requests.py --dms //versions --oneline


title "A remote state upload failure fails the bind version"
trace fault.py "POST /api/2.0/workspace-files/import-file/**resources.json" 400 0 1
trace musterr $CLI bundle deployment bind upload_failed "$job_id" --auto-approve
trace print_requests.py --dms //versions --oneline

rm -rf .databricks
title "Remote state still contains only the successful bindings"
trace $CLI bundle summary -o json | jq '.resources.jobs | map_values(.id // null)'
rm -f "$OUT_REQUESTS"
1 change: 1 addition & 0 deletions acceptance/bundle/dms/bind-history-failures/test.toml
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
Cloud = false # Inject failures into the deployment history service.
8 changes: 8 additions & 0 deletions acceptance/bundle/dms/bind-immutable/databricks.yml.tmpl
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
bundle:
name: dms-bind-immutable-$UNIQUE_NAME
workspace:
sync_mode: immutable
resources:
jobs:
foo:
name: foo
2 changes: 2 additions & 0 deletions acceptance/bundle/dms/bind-immutable/out.test.toml
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
Cloud = false
EnvMatrix.DMS = ["true"]
13 changes: 13 additions & 0 deletions acceptance/bundle/dms/bind-immutable/output.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@

>>> [CLI] bundle deployment bind foo [JOB_ID] --auto-approve
Successfully bound job with an id '[JOB_ID]'
Run 'bundle deploy' to deploy changes to your workspace

>>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-resources deployments/[DEPLOYMENT_ID] -o json
[
{
"last_action_type": "OPERATION_ACTION_TYPE_UPDATE",
"resource_id": "[JOB_ID]",
"resource_key": "jobs.foo"
}
]
8 changes: 8 additions & 0 deletions acceptance/bundle/dms/bind-immutable/script
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
envsubst < databricks.yml.tmpl > databricks.yml
job_id=$($CLI jobs create --json '{"name": "bind-target-'$UNIQUE_NAME'"}' | jq -r '.job_id')
add_repl "$job_id" JOB_ID

trace $CLI bundle deployment bind foo "$job_id" --auto-approve
deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-bind-immutable-${UNIQUE_NAME}/default/state/resources.deployment.json" -o json | jq -r '.object_id')
trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-resources "deployments/${deployment_id}" -o json | jq -S 'map({resource_key, resource_id, last_action_type})'
rm -f "$OUT_REQUESTS"
4 changes: 4 additions & 0 deletions acceptance/bundle/dms/bind-immutable/test.toml
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
# This test ensures that deployment history works with bind and immutable folders. This case is interesting
# because by default we record the snapshot path in deploy, but that is not available during bind. So it's important
# to ensure that we don't do that during deployment bind, and recording history continues to work.
Cloud = false
2 changes: 1 addition & 1 deletion acceptance/bundle/dms/bind-unbind/out.test.toml
Original file line number Diff line number Diff line change
@@ -1,2 +1,2 @@
Cloud = false
Cloud = true
EnvMatrix.DMS = ["true"]
Loading
Loading