Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 8 additions & 2 deletions acceptance/bundle/migrate/script.prepare
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,13 @@ print_migration_telemetry() {
print_telemetry_bool_values | grep '^direct_' || true
}

# The replay fixtures and the generated out.* files live in the bundle root but are not bundle
# The replay fixtures and the generated out.* and LOG.* files live in the bundle root but are not bundle
# files, so keep them out of `bundle deploy`'s upload. The list excludes .gitignore itself so it is
# not uploaded either; test.toml adds .gitignore to Ignore so the harness does not flag it.
printf '%s\n' '.gitignore' 'out.*' 'terraform*.tfstate' 'terraform*-requests.json' > .gitignore
printf '%s\n' '.gitignore' 'out.*' 'LOG.*' 'terraform*.tfstate' 'terraform*-requests.json' > .gitignore

# Per request: the engine tags in the User-Agent (MISSING if none), method and path.
# The sed strips uniq -c's padding, which differs between GNU and BSD (macOS).
print_engines() {
print_requests.py --get --oneline | jq -r 'select(.path | startswith("/__testserver/") | not) | ((.headers["User-Agent"][0] | [scan("engine/[a-z]+")] | join(",")) as $e | (if $e == "" then "MISSING" else $e end)) + " " + .method + " " + .path' | sort | uniq -c | sed -E 's/^ +//'
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
bundle:
name: test-bundle

resources:
jobs:
test_job:
name: "Test Auto-Migrate Plan Failure"
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
Cloud = false
EnvMatrix.DMS = [""]
35 changes: 35 additions & 0 deletions acceptance/bundle/migrate/user-agent/bind-unbind/output.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,35 @@

=== Replay a prior terraform deployment

>>> replay_tfstate.py [TESTROOT]/bundle/migrate/user-agent/bind-unbind/../terraform-requests.json [TESTROOT]/bundle/migrate/user-agent/bind-unbind/../terraform.tfstate

=== bind on a terraform state

>>> musterr [CLI] bundle deployment bind test_job [TEST_JOB_ID] --auto-approve
Error: this deployment's state uses the Terraform engine, which has been removed in Databricks CLI v1.20.0

Run "databricks bundle deploy" to migrate the state to the direct engine, or install Databricks CLI v1.19.x


>>> print_engines
2 MISSING GET /api/2.0/workspace/export
2 engine/terraform GET /api/2.0/workspace/export
3 engine/terraform GET /api/2.0/workspace/get-status
1 engine/terraform GET /api/2.2/jobs/get
1 engine/terraform POST /api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deploy.lock
1 engine/terraform POST /api/2.0/workspace/delete

=== unbind on a terraform state

>>> musterr [CLI] bundle deployment unbind test_job
Error: this deployment's state uses the Terraform engine, which has been removed in Databricks CLI v1.20.0

Run "databricks bundle deploy" to migrate the state to the direct engine, or install Databricks CLI v1.19.x


>>> print_engines
2 MISSING GET /api/2.0/workspace/export
2 engine/terraform GET /api/2.0/workspace/export
3 engine/terraform GET /api/2.0/workspace/get-status
1 engine/terraform POST /api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deploy.lock
1 engine/terraform POST /api/2.0/workspace/delete
16 changes: 16 additions & 0 deletions acceptance/bundle/migrate/user-agent/bind-unbind/script
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
export DATABRICKS_BUNDLE_ENGINE=

title "Replay a prior terraform deployment"
trace replay_tfstate.py "$TESTDIR/../terraform-requests.json" "$TESTDIR/../terraform.tfstate"
replace_ids.py
rm -f "$OUT_REQUESTS"
JOB_ID=$(read_id.py test_job)

# bind and unbind only pull the state, so a terraform state is not migrated and they reject it.
title "bind on a terraform state"
trace musterr $CLI bundle deployment bind test_job "$JOB_ID" --auto-approve
trace print_engines

title "unbind on a terraform state"
trace musterr $CLI bundle deployment unbind test_job
trace print_engines
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
bundle:
name: test-bundle

resources:
jobs:
test_job:
name: "Test Auto-Migrate Plan Failure"
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
Cloud = false
EnvMatrix.DMS = [""]
43 changes: 43 additions & 0 deletions acceptance/bundle/migrate/user-agent/engine-pinned/output.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,43 @@

=== Replay a prior terraform deployment

>>> replay_tfstate.py [TESTROOT]/bundle/migrate/user-agent/engine-pinned/../terraform-requests.json [TESTROOT]/bundle/migrate/user-agent/engine-pinned/../terraform.tfstate

=== deploy with DATABRICKS_BUNDLE_ENGINE=terraform

>>> DATABRICKS_BUNDLE_ENGINE=terraform musterr [CLI] bundle deploy
Error: the Terraform deployment engine has been removed in Databricks CLI v1.20.0

Unset the DATABRICKS_BUNDLE_ENGINE environment variable (or set it to "direct") to deploy with the direct engine; existing Terraform state is migrated automatically. To keep using Terraform, revert to Databricks CLI v1.19.x.
See https://docs.databricks.com/dev-tools/bundles/direct for details

>>> print_engines
1 MISSING POST /telemetry-ext

=== deploy with bundle.engine: terraform

>>> musterr [CLI] bundle deploy
Error: the Terraform deployment engine has been removed in Databricks CLI v1.20.0
in databricks.yml:3:11

Remove the "bundle.engine" setting (or set it to "direct") to deploy with the direct engine; existing Terraform state is migrated automatically. To keep using Terraform, revert to Databricks CLI v1.19.x.
See https://docs.databricks.com/dev-tools/bundles/direct for details


>>> print_engines
1 MISSING POST /telemetry-ext

=== summary with bundle.engine: terraform

>>> [CLI] bundle summary
Warning: the Terraform deployment engine has been removed in Databricks CLI v1.20.0
in databricks.yml:3:11

Remove the "bundle.engine" setting (or set it to "direct") to deploy with the direct engine; existing Terraform state is migrated automatically. To keep using Terraform, revert to Databricks CLI v1.19.x.
See https://docs.databricks.com/dev-tools/bundles/direct for details

Notice: migrating your bundle to direct deployment engine (https://github.com/databricks/cli/issues/6765).

>>> print_engines
1 MISSING GET /api/2.2/jobs/get
1 engine/direct GET /api/2.0/preview/scim/v2/Me
21 changes: 21 additions & 0 deletions acceptance/bundle/migrate/user-agent/engine-pinned/script
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
export DATABRICKS_BUNDLE_ENGINE=

title "Replay a prior terraform deployment"
trace replay_tfstate.py "$TESTDIR/../terraform-requests.json" "$TESTDIR/../terraform.tfstate"
replace_ids.py
rm -f "$OUT_REQUESTS"

title "deploy with DATABRICKS_BUNDLE_ENGINE=terraform"
trace DATABRICKS_BUNDLE_ENGINE=terraform musterr $CLI bundle deploy
trace print_engines

title "deploy with bundle.engine: terraform"
update_file.py databricks.yml ' name: test-bundle' ' name: test-bundle
engine: terraform'
trace musterr $CLI bundle deploy
trace print_engines

# summary allows a terraform bundle.engine (warning only) and migrates the state in memory.
title "summary with bundle.engine: terraform"
trace $CLI bundle summary > LOG.summary
trace print_engines
13 changes: 13 additions & 0 deletions acceptance/bundle/migrate/user-agent/race/databricks.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
bundle:
name: test-bundle

experimental:
scripts:
# Simulates a concurrent deploy migrating the state: postbuild runs after the state pull and
# before the auto-migration, so the migration finds no local terraform state.
postbuild: rm -f .databricks/bundle/default/terraform/terraform.tfstate

resources:
jobs:
test_job:
name: "Test Auto-Migrate Plan Failure"
2 changes: 2 additions & 0 deletions acceptance/bundle/migrate/user-agent/race/out.test.toml
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
Cloud = false
EnvMatrix.DMS = [""]
16 changes: 16 additions & 0 deletions acceptance/bundle/migrate/user-agent/race/output.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@

=== Replay a prior terraform deployment

>>> replay_tfstate.py [TESTROOT]/bundle/migrate/user-agent/race/../terraform-requests.json [TESTROOT]/bundle/migrate/user-agent/race/../terraform.tfstate

=== Deploy where the local terraform state disappears between the pull and the migration

>>> print_engines
2 MISSING GET /api/2.0/workspace/export
3 engine/terraform GET /api/2.0/workspace/export
4 engine/terraform GET /api/2.0/workspace/get-status
1 engine/terraform POST /api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/databricks.yml
1 engine/terraform POST /api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deploy.lock
1 engine/terraform POST /api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deployment.json
2 engine/terraform POST /api/2.0/workspace/delete
1 engine/terraform POST /api/2.0/workspace/mkdirs
11 changes: 11 additions & 0 deletions acceptance/bundle/migrate/user-agent/race/script
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
export DATABRICKS_BUNDLE_ENGINE=

title "Replay a prior terraform deployment"
trace replay_tfstate.py "$TESTDIR/../terraform-requests.json" "$TESTDIR/../terraform.tfstate"
replace_ids.py
rm -f "$OUT_REQUESTS"

title "Deploy where the local terraform state disappears between the pull and the migration"
musterr $CLI bundle deploy &> LOG.deploy
contains.py 'Panic Payload: internal error: DeploymentState must be opened first' < LOG.deploy > /dev/null
trace print_engines
19 changes: 19 additions & 0 deletions acceptance/bundle/migrate/user-agent/terraform-requests.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
[
{
"method": "POST",
"path": "/api/2.2/jobs/create",
"body": {
"deployment": {
"kind": "BUNDLE",
"metadata_file_path": "/Workspace/Users/tester@databricks.com/.bundle/test-bundle/default/state/metadata.json"
},
"edit_mode": "UI_LOCKED",
"format": "MULTI_TASK",
"max_concurrent_runs": 1,
"name": "Test Auto-Migrate Plan Failure",
"queue": {
"enabled": true
}
}
}
]
110 changes: 110 additions & 0 deletions acceptance/bundle/migrate/user-agent/terraform.tfstate
Original file line number Diff line number Diff line change
@@ -0,0 +1,110 @@
{

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

terraform.tfstate (110 lines) and terraform-requests.json (19 lines) are byte-for-byte identical in all three subdirectories. They could live once in the user-agent/ parent and each script would reference ../terraform-requests.json ../terraform.tfstate. That keeps them in sync and drops ~260 lines; replay_tfstate.py takes plain relative paths so it just works. If a test later needs a different fixture it can fork at that point.

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Done in 2f9d1c9: the fixtures now live once in acceptance/bundle/migrate/user-agent/. One correction: tests run in a temp copy of their directory, so a bare ../ doesn't resolve. The scripts read them via $TESTDIR/../, like bundle/validate/empty_resources.

This reply was written by Isaac.

"version": 4,
"terraform_version": "1.5.5",
"serial": 1,
"lineage": "115ec518-3f72-3614-4e56-01411863558a",
"outputs": {},
"resources": [
{
"mode": "managed",
"type": "databricks_job",
"name": "test_job",
"provider": "provider[\"registry.terraform.io/databricks/databricks\"]",
"instances": [
{
"schema_version": 2,
"attributes": {
"always_running": false,
"budget_policy_id": null,
"continuous": [],
"control_run_state": false,
"dbt_task": [],
"deployment": [
{
"deployment_id": "",
"kind": "BUNDLE",
"metadata_file_path": "/Workspace/Users/tester@databricks.com/.bundle/test-bundle/default/state/metadata.json",
"version_id": ""
}
],
"description": null,
"edit_mode": "UI_LOCKED",
"email_notifications": [
{
"no_alert_for_skipped_runs": false,
"on_duration_warning_threshold_exceeded": [],
"on_failure": [],
"on_start": [],
"on_streaming_backlog_exceeded": [],
"on_success": []
}
],
"environment": [],
"existing_cluster_id": null,
"format": "MULTI_TASK",
"git_source": [],
"health": [],
"id": "8790613902942848000",
"job_cluster": [],
"library": [],
"max_concurrent_runs": 1,
"max_retries": 0,
"min_retry_interval_millis": 0,
"name": "Test Auto-Migrate Plan Failure",
"new_cluster": [],
"notebook_task": [],
"notification_settings": [],
"parameter": [],
"parent_path": null,
"performance_target": null,
"pipeline_task": [],
"provider_config": [
{
"workspace_id": "900800700600"
}
],
"python_wheel_task": [],
"queue": [
{
"enabled": true
}
],
"retry_on_timeout": false,
"run_as": [
{
"group_name": "",
"service_principal_name": "",
"user_name": "tester@databricks.com"
}
],
"run_job_task": [],
"schedule": [],
"spark_jar_task": [],
"spark_python_task": [],
"spark_submit_task": [],
"tags": null,
"task": [],
"timeout_seconds": 0,
"timeouts": null,
"trigger": [],
"triggers": [],
"url": "[DATABRICKS_URL]/#job/8790613902942848000",
"usage_policy_id": null,
"webhook_notifications": [
{
"on_duration_warning_threshold_exceeded": [],
"on_failure": [],
"on_start": [],
"on_streaming_backlog_exceeded": [],
"on_success": []
}
]
},
"sensitive_attributes": [],
"private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjoxODAwMDAwMDAwMDAwLCJ1cGRhdGUiOjE4MDAwMDAwMDAwMDB9LCJzY2hlbWFfdmVyc2lvbiI6IjIifQ=="
}
]
}
],
"check_results": null
}
Loading