diff --git a/acceptance/bundle/resources/pipelines/ingestion-defaults/databricks.yml.tmpl b/acceptance/bundle/resources/pipelines/ingestion-defaults/databricks.yml.tmpl new file mode 100644 index 00000000000..17ebdffeefd --- /dev/null +++ b/acceptance/bundle/resources/pipelines/ingestion-defaults/databricks.yml.tmpl @@ -0,0 +1,16 @@ +bundle: + name: test-ingestion-defaults-$UNIQUE_NAME + +resources: + pipelines: + ingestion: + name: test-ingestion-$UNIQUE_NAME + catalog: main + target: default + ingestion_definition: + connection_name: $INGESTION_CONNECTION_NAME + objects: [] + +targets: + dev: + default: true diff --git a/acceptance/bundle/resources/pipelines/ingestion-defaults/out.test.toml b/acceptance/bundle/resources/pipelines/ingestion-defaults/out.test.toml new file mode 100644 index 00000000000..b4b1fb39d67 --- /dev/null +++ b/acceptance/bundle/resources/pipelines/ingestion-defaults/out.test.toml @@ -0,0 +1,4 @@ +Local = false +Cloud = true +CloudEnvs.azure = true +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/pipelines/ingestion-defaults/output.txt b/acceptance/bundle/resources/pipelines/ingestion-defaults/output.txt new file mode 100644 index 00000000000..bf202db9226 --- /dev/null +++ b/acceptance/bundle/resources/pipelines/ingestion-defaults/output.txt @@ -0,0 +1,13 @@ + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-ingestion-defaults-[UNIQUE_NAME]/dev/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> jq -s .[] | select(.path=="/api/2.0/pipelines" and .method=="POST") | .body | {name, edition, channel} out.requests.txt +{ + "name": "test-ingestion-[UNIQUE_NAME]", + "edition": null, + "channel": null +} diff --git a/acceptance/bundle/resources/pipelines/ingestion-defaults/script b/acceptance/bundle/resources/pipelines/ingestion-defaults/script new file mode 100644 index 00000000000..3d9a068be18 --- /dev/null +++ b/acceptance/bundle/resources/pipelines/ingestion-defaults/script @@ -0,0 +1,15 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +cleanup() { + trace $CLI bundle destroy --auto-approve 2>/dev/null || true + rm -f out.requests.txt +} +trap cleanup EXIT + +# Deploy the ingestion pipeline. Verifies that: +# 1. Deployment succeeds without "cluster settings" errors +# 2. edition and channel are absent from the created pipeline's spec +trace $CLI bundle deploy + +trace jq -s '.[] | select(.path=="/api/2.0/pipelines" and .method=="POST") | .body | {name, edition, channel}' out.requests.txt +rm -f out.requests.txt diff --git a/acceptance/bundle/resources/pipelines/ingestion-defaults/test.toml b/acceptance/bundle/resources/pipelines/ingestion-defaults/test.toml new file mode 100644 index 00000000000..c287a956cf9 --- /dev/null +++ b/acceptance/bundle/resources/pipelines/ingestion-defaults/test.toml @@ -0,0 +1,16 @@ +Local = false +Cloud = true +RecordRequests = true + +# The Terraform engine still injects edition/channel at provider level (SetDefault in +# resource_pipeline.go:253); companion fix: terraform-provider-databricks#5783. +# Only test with direct engine here so we cleanly validate the DABs-side fix. +[EnvMatrix] +DATABRICKS_BUNDLE_ENGINE = ["direct"] + +[Env] +# MySQL connection present in the azure test workspace. +INGESTION_CONNECTION_NAME = "name-sbggefbaaidkb" + +[CloudEnvs] +azure = true diff --git a/acceptance/bundle/resources/pipelines/recreate-keys/change-ingestion-definition/out.pipeline_get.direct.json b/acceptance/bundle/resources/pipelines/recreate-keys/change-ingestion-definition/out.pipeline_get.direct.json new file mode 100644 index 00000000000..c7e9105acbd --- /dev/null +++ b/acceptance/bundle/resources/pipelines/recreate-keys/change-ingestion-definition/out.pipeline_get.direct.json @@ -0,0 +1,31 @@ +{ + "creator_user_name": "[USERNAME]", + "effective_publishing_mode": "DEFAULT_PUBLISHING_MODE", + "last_modified": [UNIX_TIME_MILLIS], + "name": "test-pipeline-[UNIQUE_NAME]", + "pipeline_id": "[MY_ID_2]", + "run_as_user_name": "[USERNAME]", + "spec": { + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/acc-[UNIQUE_NAME]/default/state/metadata.json" + }, + "id": "[MY_ID_2]", + "ingestion_definition": { + "connection_name": "my_new_connection", + "objects": [ + {} + ] + }, + "libraries": [ + { + "file": { + "path": "/Workspace/Users/[USERNAME]/.bundle/acc-[UNIQUE_NAME]/default/files/foo.py" + } + } + ], + "name": "test-pipeline-[UNIQUE_NAME]", + "storage": "dbfs:/pipelines/[MY_ID_2]" + }, + "state": "IDLE" +} diff --git a/acceptance/bundle/resources/pipelines/recreate-keys/change-ingestion-definition/out.pipeline_get.terraform.json b/acceptance/bundle/resources/pipelines/recreate-keys/change-ingestion-definition/out.pipeline_get.terraform.json new file mode 100644 index 00000000000..c1f35c46790 --- /dev/null +++ b/acceptance/bundle/resources/pipelines/recreate-keys/change-ingestion-definition/out.pipeline_get.terraform.json @@ -0,0 +1,33 @@ +{ + "creator_user_name": "[USERNAME]", + "effective_publishing_mode": "DEFAULT_PUBLISHING_MODE", + "last_modified": [UNIX_TIME_MILLIS], + "name": "test-pipeline-[UNIQUE_NAME]", + "pipeline_id": "[MY_ID_2]", + "run_as_user_name": "[USERNAME]", + "spec": { + "channel": "CURRENT", + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/acc-[UNIQUE_NAME]/default/state/metadata.json" + }, + "edition": "ADVANCED", + "id": "[MY_ID_2]", + "ingestion_definition": { + "connection_name": "my_new_connection", + "objects": [ + {} + ] + }, + "libraries": [ + { + "file": { + "path": "/Workspace/Users/[USERNAME]/.bundle/acc-[UNIQUE_NAME]/default/files/foo.py" + } + } + ], + "name": "test-pipeline-[UNIQUE_NAME]", + "storage": "dbfs:/pipelines/[MY_ID_2]" + }, + "state": "IDLE" +} diff --git a/acceptance/bundle/resources/pipelines/recreate-keys/change-ingestion-definition/out.plan_create.direct.json b/acceptance/bundle/resources/pipelines/recreate-keys/change-ingestion-definition/out.plan_create.direct.json index bd540374432..5f4b9774b2c 100644 --- a/acceptance/bundle/resources/pipelines/recreate-keys/change-ingestion-definition/out.plan_create.direct.json +++ b/acceptance/bundle/resources/pipelines/recreate-keys/change-ingestion-definition/out.plan_create.direct.json @@ -6,12 +6,10 @@ "action": "create", "new_state": { "value": { - "channel": "CURRENT", "deployment": { "kind": "BUNDLE", "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/acc-[UNIQUE_NAME]/default/state/metadata.json" }, - "edition": "ADVANCED", "ingestion_definition": { "connection_name": "my_connection", "objects": [ diff --git a/acceptance/bundle/resources/pipelines/recreate-keys/change-ingestion-definition/out.plan_recreate.direct.json b/acceptance/bundle/resources/pipelines/recreate-keys/change-ingestion-definition/out.plan_recreate.direct.json index ee6b7940b47..3cc42d243fe 100644 --- a/acceptance/bundle/resources/pipelines/recreate-keys/change-ingestion-definition/out.plan_recreate.direct.json +++ b/acceptance/bundle/resources/pipelines/recreate-keys/change-ingestion-definition/out.plan_recreate.direct.json @@ -8,12 +8,10 @@ "action": "recreate", "new_state": { "value": { - "channel": "CURRENT", "deployment": { "kind": "BUNDLE", "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/acc-[UNIQUE_NAME]/default/state/metadata.json" }, - "edition": "ADVANCED", "ingestion_definition": { "connection_name": "my_new_connection", "objects": [ @@ -31,13 +29,11 @@ } }, "remote_state": { - "channel": "CURRENT", "creator_user_name": "[USERNAME]", "deployment": { "kind": "BUNDLE", "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/acc-[UNIQUE_NAME]/default/state/metadata.json" }, - "edition": "ADVANCED", "effective_publishing_mode": "DEFAULT_PUBLISHING_MODE", "id": "[MY_ID]", "ingestion_definition": { diff --git a/acceptance/bundle/resources/pipelines/recreate-keys/change-ingestion-definition/out.requests.direct.json b/acceptance/bundle/resources/pipelines/recreate-keys/change-ingestion-definition/out.requests.direct.json new file mode 100644 index 00000000000..ab7669a5739 --- /dev/null +++ b/acceptance/bundle/resources/pipelines/recreate-keys/change-ingestion-definition/out.requests.direct.json @@ -0,0 +1,4 @@ +{ + "method": "DELETE", + "path": "/api/2.0/pipelines/[MY_ID_2]" +} diff --git a/acceptance/bundle/resources/pipelines/recreate-keys/change-ingestion-definition/out.requests.terraform.json b/acceptance/bundle/resources/pipelines/recreate-keys/change-ingestion-definition/out.requests.terraform.json new file mode 100644 index 00000000000..ab7669a5739 --- /dev/null +++ b/acceptance/bundle/resources/pipelines/recreate-keys/change-ingestion-definition/out.requests.terraform.json @@ -0,0 +1,4 @@ +{ + "method": "DELETE", + "path": "/api/2.0/pipelines/[MY_ID_2]" +} diff --git a/acceptance/bundle/resources/pipelines/recreate-keys/change-ingestion-definition/output.txt b/acceptance/bundle/resources/pipelines/recreate-keys/change-ingestion-definition/output.txt index 77bfdc09d32..ddfedf361dc 100644 --- a/acceptance/bundle/resources/pipelines/recreate-keys/change-ingestion-definition/output.txt +++ b/acceptance/bundle/resources/pipelines/recreate-keys/change-ingestion-definition/output.txt @@ -25,34 +25,6 @@ Deploying resources... Updating deployment state... Deployment complete! ->>> print_requests -{ - "body": { - "channel": "CURRENT", - "deployment": { - "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/acc-[UNIQUE_NAME]/default/state/metadata.json" - }, - "edition": "ADVANCED", - "ingestion_definition": { - "connection_name": "my_connection", - "objects": [ - {} - ] - }, - "libraries": [ - { - "file": { - "path": "/Workspace/Users/[USERNAME]/.bundle/acc-[UNIQUE_NAME]/default/files/foo.py" - } - } - ], - "name": "test-pipeline-[UNIQUE_NAME]" - }, - "method": "POST", - "path": "/api/2.0/pipelines" -} - >>> update_file.py databricks.yml my_connection my_new_connection >>> [CLI] bundle plan @@ -72,74 +44,7 @@ Deploying resources... Updating deployment state... Deployment complete! ->>> print_requests -{ - "method": "DELETE", - "path": "/api/2.0/pipelines/[MY_ID]" -} -{ - "body": { - "channel": "CURRENT", - "deployment": { - "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/acc-[UNIQUE_NAME]/default/state/metadata.json" - }, - "edition": "ADVANCED", - "ingestion_definition": { - "connection_name": "my_new_connection", - "objects": [ - {} - ] - }, - "libraries": [ - { - "file": { - "path": "/Workspace/Users/[USERNAME]/.bundle/acc-[UNIQUE_NAME]/default/files/foo.py" - } - } - ], - "name": "test-pipeline-[UNIQUE_NAME]" - }, - "method": "POST", - "path": "/api/2.0/pipelines" -} - === Fetch pipeline ID and verify remote state ->>> [CLI] pipelines get [MY_ID_2] -{ - "creator_user_name": "[USERNAME]", - "effective_publishing_mode": "DEFAULT_PUBLISHING_MODE", - "last_modified": [UNIX_TIME_MILLIS], - "name": "test-pipeline-[UNIQUE_NAME]", - "pipeline_id": "[MY_ID_2]", - "run_as_user_name": "[USERNAME]", - "spec": { - "channel": "CURRENT", - "deployment": { - "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/acc-[UNIQUE_NAME]/default/state/metadata.json" - }, - "edition": "ADVANCED", - "id": "[MY_ID_2]", - "ingestion_definition": { - "connection_name": "my_new_connection", - "objects": [ - {} - ] - }, - "libraries": [ - { - "file": { - "path": "/Workspace/Users/[USERNAME]/.bundle/acc-[UNIQUE_NAME]/default/files/foo.py" - } - } - ], - "name": "test-pipeline-[UNIQUE_NAME]", - "storage": "dbfs:/pipelines/[MY_ID_2]" - }, - "state": "IDLE" -} - === Verify that original pipeline is gone >>> musterr [CLI] pipelines get [MY_ID] Error: The specified pipeline [MY_ID] was not found. @@ -158,11 +63,5 @@ All files and directories at the following location will be deleted: /Workspace/ Deleting files... Destroy complete! ->>> print_requests -{ - "method": "DELETE", - "path": "/api/2.0/pipelines/[MY_ID_2]" -} - >>> musterr [CLI] pipelines get [MY_ID_2] Error: The specified pipeline [MY_ID_2] was not found. diff --git a/acceptance/bundle/resources/pipelines/recreate-keys/change-ingestion-definition/script b/acceptance/bundle/resources/pipelines/recreate-keys/change-ingestion-definition/script index 82de4115550..d2ef70cb663 100644 --- a/acceptance/bundle/resources/pipelines/recreate-keys/change-ingestion-definition/script +++ b/acceptance/bundle/resources/pipelines/recreate-keys/change-ingestion-definition/script @@ -1,2 +1,38 @@ envsubst < $TESTDIR/../databricks.yml.tmpl > databricks.yml -source $TESTDIR/../_script + +trace cat databricks.yml +touch foo.py +touch bar.py +trace $CLI bundle plan # should show 'create' +$CLI bundle plan -o json > out.plan_create.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle deploy + +ppid1=`read_id.py my` + +print_requests() { + jq --sort-keys 'select(.method != "GET" and (.path | contains("/pipelines")))' < out.requests.txt > out.requests.$DATABRICKS_BUNDLE_ENGINE.json + rm -f out.requests.txt +} + +print_requests + +trace update_file.py databricks.yml $CONFIG_UPDATE +trace $CLI bundle plan # should show 'recreate' +$CLI bundle plan -o json > out.plan_recreate.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle deploy --auto-approve +print_requests + +title "Fetch pipeline ID and verify remote state" + +ppid2=`read_id.py my` +$CLI pipelines get $ppid2 > out.pipeline_get.$DATABRICKS_BUNDLE_ENGINE.json + +title "Verify that original pipeline is gone" +trace musterr $CLI pipelines get $ppid1 + +title "Destroy the pipeline and verify that it's removed from the state and from remote" +trace $CLI bundle destroy --auto-approve + +print_requests +trace musterr $CLI pipelines get $ppid2 +rm -f out.requests.txt diff --git a/acceptance/bundle/validate/pipeline_defaults/databricks.yml b/acceptance/bundle/validate/pipeline_defaults/databricks.yml new file mode 100644 index 00000000000..58b04c996e7 --- /dev/null +++ b/acceptance/bundle/validate/pipeline_defaults/databricks.yml @@ -0,0 +1,13 @@ +bundle: + name: test-bundle + +resources: + pipelines: + regular: + name: regular-pipeline + + ingestion: + name: ingestion-pipeline + ingestion_definition: + connection_name: myconn + objects: [{}] diff --git a/acceptance/bundle/validate/pipeline_defaults/out.test.toml b/acceptance/bundle/validate/pipeline_defaults/out.test.toml new file mode 100644 index 00000000000..f784a183258 --- /dev/null +++ b/acceptance/bundle/validate/pipeline_defaults/out.test.toml @@ -0,0 +1,3 @@ +Local = true +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/validate/pipeline_defaults/output.txt b/acceptance/bundle/validate/pipeline_defaults/output.txt new file mode 100644 index 00000000000..cf157159348 --- /dev/null +++ b/acceptance/bundle/validate/pipeline_defaults/output.txt @@ -0,0 +1,10 @@ +{ + "regular": { + "edition": "ADVANCED", + "channel": "CURRENT" + }, + "ingestion": { + "edition": null, + "channel": null + } +} diff --git a/acceptance/bundle/validate/pipeline_defaults/script b/acceptance/bundle/validate/pipeline_defaults/script new file mode 100644 index 00000000000..5806946a066 --- /dev/null +++ b/acceptance/bundle/validate/pipeline_defaults/script @@ -0,0 +1,2 @@ +# edition and channel defaults should be set for regular pipelines but not for ingestion pipelines +$CLI bundle validate -o json | jq '{regular: .resources.pipelines.regular | {edition, channel}, ingestion: .resources.pipelines.ingestion | {edition, channel}}' diff --git a/bundle/config/mutator/resourcemutator/resource_mutator.go b/bundle/config/mutator/resourcemutator/resource_mutator.go index 209bbcb06a0..90887bfa00a 100644 --- a/bundle/config/mutator/resourcemutator/resource_mutator.go +++ b/bundle/config/mutator/resourcemutator/resource_mutator.go @@ -80,11 +80,6 @@ func applyInitializeMutators(ctx context.Context, b *bundle.Bundle) { {"resources.jobs.*.job_clusters[*].new_cluster.workload_type.clients.notebooks", true}, {"resources.jobs.*.job_clusters[*].new_cluster.workload_type.clients.jobs", true}, - // Pipelines (same as terraform) - // https://github.com/databricks/terraform-provider-databricks/blob/v1.75.0/pipelines/resource_pipeline.go#L253 - {"resources.pipelines.*.edition", "ADVANCED"}, - {"resources.pipelines.*.channel", "CURRENT"}, - // SqlWarehouses (same as terraform) // https://github.com/databricks/terraform-provider-databricks/blob/v1.75.0/sql/resource_sql_endpoint.go#L59 {"resources.sql_warehouses.*.auto_stop_mins", 120}, @@ -109,6 +104,24 @@ func applyInitializeMutators(ctx context.Context, b *bundle.Bundle) { } } + // Apply edition and channel defaults only to non-ingestion pipelines. + // Ingestion pipelines use serverless compute by default; the API rejects + // these fields as incompatible cluster settings in that mode. + // https://github.com/databricks/terraform-provider-databricks/blob/v1.75.0/pipelines/resource_pipeline.go#L253 + for name, p := range b.Config.Resources.Pipelines { + if p == nil || p.IngestionDefinition != nil { + continue + } + bundle.SetDefault(ctx, b, "resources.pipelines."+name+".edition", "ADVANCED") + if logdiag.HasError(ctx) { + return + } + bundle.SetDefault(ctx, b, "resources.pipelines."+name+".channel", "CURRENT") + if logdiag.HasError(ctx) { + return + } + } + bundle.ApplySeqContext(ctx, b, // Reads (typed): b.Config.Resources.Dashboards (checks dashboard configurations) // Updates (typed): b.Config.Resources.Dashboards[].ParentPath (ensures /Workspace prefix is present)