From 78f77fb5559fa902e1753deedf4ce6d16ad2fea3 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Wed, 3 Jun 2026 00:04:01 +0200 Subject: [PATCH 1/4] bundle: skip edition/channel defaults for ingestion pipelines Ingestion pipelines (those with `ingestion_definition`) use serverless compute by default. The Databricks API rejects `edition` and `channel` fields as "cluster settings" when a pipeline runs on serverless compute. Previously, DABs unconditionally set `edition: ADVANCED` and `channel: CURRENT` as defaults for all pipeline types. This caused creation of LakeFlow Connect ingestion pipelines to fail with: "cannot create pipeline: You cannot provide cluster settings when using serverless compute" The fix conditionally applies these defaults only to pipelines without `ingestion_definition`. The test `change-ingestion-definition` is restructured to use per-engine output files since the TF provider independently adds these defaults for all pipeline types. Co-authored-by: Shreyas Goenka --- .../out.pipeline_get.direct.json | 31 ++++++ .../out.pipeline_get.terraform.json | 33 ++++++ .../out.plan_create.direct.json | 2 - .../out.plan_recreate.direct.json | 4 - .../out.requests.direct.json | 4 + .../out.requests.terraform.json | 4 + .../change-ingestion-definition/output.txt | 101 ------------------ .../change-ingestion-definition/script | 38 ++++++- .../validate/pipeline_defaults/databricks.yml | 13 +++ .../validate/pipeline_defaults/out.test.toml | 3 + .../validate/pipeline_defaults/output.txt | 10 ++ .../bundle/validate/pipeline_defaults/script | 2 + .../resourcemutator/resource_mutator.go | 39 ++++++- 13 files changed, 171 insertions(+), 113 deletions(-) create mode 100644 acceptance/bundle/resources/pipelines/recreate-keys/change-ingestion-definition/out.pipeline_get.direct.json create mode 100644 acceptance/bundle/resources/pipelines/recreate-keys/change-ingestion-definition/out.pipeline_get.terraform.json create mode 100644 acceptance/bundle/resources/pipelines/recreate-keys/change-ingestion-definition/out.requests.direct.json create mode 100644 acceptance/bundle/resources/pipelines/recreate-keys/change-ingestion-definition/out.requests.terraform.json create mode 100644 acceptance/bundle/validate/pipeline_defaults/databricks.yml create mode 100644 acceptance/bundle/validate/pipeline_defaults/out.test.toml create mode 100644 acceptance/bundle/validate/pipeline_defaults/output.txt create mode 100644 acceptance/bundle/validate/pipeline_defaults/script diff --git a/acceptance/bundle/resources/pipelines/recreate-keys/change-ingestion-definition/out.pipeline_get.direct.json b/acceptance/bundle/resources/pipelines/recreate-keys/change-ingestion-definition/out.pipeline_get.direct.json new file mode 100644 index 00000000000..c7e9105acbd --- /dev/null +++ b/acceptance/bundle/resources/pipelines/recreate-keys/change-ingestion-definition/out.pipeline_get.direct.json @@ -0,0 +1,31 @@ +{ + "creator_user_name": "[USERNAME]", + "effective_publishing_mode": "DEFAULT_PUBLISHING_MODE", + "last_modified": [UNIX_TIME_MILLIS], + "name": "test-pipeline-[UNIQUE_NAME]", + "pipeline_id": "[MY_ID_2]", + "run_as_user_name": "[USERNAME]", + "spec": { + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/acc-[UNIQUE_NAME]/default/state/metadata.json" + }, + "id": "[MY_ID_2]", + "ingestion_definition": { + "connection_name": "my_new_connection", + "objects": [ + {} + ] + }, + "libraries": [ + { + "file": { + "path": "/Workspace/Users/[USERNAME]/.bundle/acc-[UNIQUE_NAME]/default/files/foo.py" + } + } + ], + "name": "test-pipeline-[UNIQUE_NAME]", + "storage": "dbfs:/pipelines/[MY_ID_2]" + }, + "state": "IDLE" +} diff --git a/acceptance/bundle/resources/pipelines/recreate-keys/change-ingestion-definition/out.pipeline_get.terraform.json b/acceptance/bundle/resources/pipelines/recreate-keys/change-ingestion-definition/out.pipeline_get.terraform.json new file mode 100644 index 00000000000..c1f35c46790 --- /dev/null +++ b/acceptance/bundle/resources/pipelines/recreate-keys/change-ingestion-definition/out.pipeline_get.terraform.json @@ -0,0 +1,33 @@ +{ + "creator_user_name": "[USERNAME]", + "effective_publishing_mode": "DEFAULT_PUBLISHING_MODE", + "last_modified": [UNIX_TIME_MILLIS], + "name": "test-pipeline-[UNIQUE_NAME]", + "pipeline_id": "[MY_ID_2]", + "run_as_user_name": "[USERNAME]", + "spec": { + "channel": "CURRENT", + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/acc-[UNIQUE_NAME]/default/state/metadata.json" + }, + "edition": "ADVANCED", + "id": "[MY_ID_2]", + "ingestion_definition": { + "connection_name": "my_new_connection", + "objects": [ + {} + ] + }, + "libraries": [ + { + "file": { + "path": "/Workspace/Users/[USERNAME]/.bundle/acc-[UNIQUE_NAME]/default/files/foo.py" + } + } + ], + "name": "test-pipeline-[UNIQUE_NAME]", + "storage": "dbfs:/pipelines/[MY_ID_2]" + }, + "state": "IDLE" +} diff --git a/acceptance/bundle/resources/pipelines/recreate-keys/change-ingestion-definition/out.plan_create.direct.json b/acceptance/bundle/resources/pipelines/recreate-keys/change-ingestion-definition/out.plan_create.direct.json index bd540374432..5f4b9774b2c 100644 --- a/acceptance/bundle/resources/pipelines/recreate-keys/change-ingestion-definition/out.plan_create.direct.json +++ b/acceptance/bundle/resources/pipelines/recreate-keys/change-ingestion-definition/out.plan_create.direct.json @@ -6,12 +6,10 @@ "action": "create", "new_state": { "value": { - "channel": "CURRENT", "deployment": { "kind": "BUNDLE", "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/acc-[UNIQUE_NAME]/default/state/metadata.json" }, - "edition": "ADVANCED", "ingestion_definition": { "connection_name": "my_connection", "objects": [ diff --git a/acceptance/bundle/resources/pipelines/recreate-keys/change-ingestion-definition/out.plan_recreate.direct.json b/acceptance/bundle/resources/pipelines/recreate-keys/change-ingestion-definition/out.plan_recreate.direct.json index ee6b7940b47..3cc42d243fe 100644 --- a/acceptance/bundle/resources/pipelines/recreate-keys/change-ingestion-definition/out.plan_recreate.direct.json +++ b/acceptance/bundle/resources/pipelines/recreate-keys/change-ingestion-definition/out.plan_recreate.direct.json @@ -8,12 +8,10 @@ "action": "recreate", "new_state": { "value": { - "channel": "CURRENT", "deployment": { "kind": "BUNDLE", "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/acc-[UNIQUE_NAME]/default/state/metadata.json" }, - "edition": "ADVANCED", "ingestion_definition": { "connection_name": "my_new_connection", "objects": [ @@ -31,13 +29,11 @@ } }, "remote_state": { - "channel": "CURRENT", "creator_user_name": "[USERNAME]", "deployment": { "kind": "BUNDLE", "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/acc-[UNIQUE_NAME]/default/state/metadata.json" }, - "edition": "ADVANCED", "effective_publishing_mode": "DEFAULT_PUBLISHING_MODE", "id": "[MY_ID]", "ingestion_definition": { diff --git a/acceptance/bundle/resources/pipelines/recreate-keys/change-ingestion-definition/out.requests.direct.json b/acceptance/bundle/resources/pipelines/recreate-keys/change-ingestion-definition/out.requests.direct.json new file mode 100644 index 00000000000..ab7669a5739 --- /dev/null +++ b/acceptance/bundle/resources/pipelines/recreate-keys/change-ingestion-definition/out.requests.direct.json @@ -0,0 +1,4 @@ +{ + "method": "DELETE", + "path": "/api/2.0/pipelines/[MY_ID_2]" +} diff --git a/acceptance/bundle/resources/pipelines/recreate-keys/change-ingestion-definition/out.requests.terraform.json b/acceptance/bundle/resources/pipelines/recreate-keys/change-ingestion-definition/out.requests.terraform.json new file mode 100644 index 00000000000..ab7669a5739 --- /dev/null +++ b/acceptance/bundle/resources/pipelines/recreate-keys/change-ingestion-definition/out.requests.terraform.json @@ -0,0 +1,4 @@ +{ + "method": "DELETE", + "path": "/api/2.0/pipelines/[MY_ID_2]" +} diff --git a/acceptance/bundle/resources/pipelines/recreate-keys/change-ingestion-definition/output.txt b/acceptance/bundle/resources/pipelines/recreate-keys/change-ingestion-definition/output.txt index 77bfdc09d32..ddfedf361dc 100644 --- a/acceptance/bundle/resources/pipelines/recreate-keys/change-ingestion-definition/output.txt +++ b/acceptance/bundle/resources/pipelines/recreate-keys/change-ingestion-definition/output.txt @@ -25,34 +25,6 @@ Deploying resources... Updating deployment state... Deployment complete! ->>> print_requests -{ - "body": { - "channel": "CURRENT", - "deployment": { - "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/acc-[UNIQUE_NAME]/default/state/metadata.json" - }, - "edition": "ADVANCED", - "ingestion_definition": { - "connection_name": "my_connection", - "objects": [ - {} - ] - }, - "libraries": [ - { - "file": { - "path": "/Workspace/Users/[USERNAME]/.bundle/acc-[UNIQUE_NAME]/default/files/foo.py" - } - } - ], - "name": "test-pipeline-[UNIQUE_NAME]" - }, - "method": "POST", - "path": "/api/2.0/pipelines" -} - >>> update_file.py databricks.yml my_connection my_new_connection >>> [CLI] bundle plan @@ -72,74 +44,7 @@ Deploying resources... Updating deployment state... Deployment complete! ->>> print_requests -{ - "method": "DELETE", - "path": "/api/2.0/pipelines/[MY_ID]" -} -{ - "body": { - "channel": "CURRENT", - "deployment": { - "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/acc-[UNIQUE_NAME]/default/state/metadata.json" - }, - "edition": "ADVANCED", - "ingestion_definition": { - "connection_name": "my_new_connection", - "objects": [ - {} - ] - }, - "libraries": [ - { - "file": { - "path": "/Workspace/Users/[USERNAME]/.bundle/acc-[UNIQUE_NAME]/default/files/foo.py" - } - } - ], - "name": "test-pipeline-[UNIQUE_NAME]" - }, - "method": "POST", - "path": "/api/2.0/pipelines" -} - === Fetch pipeline ID and verify remote state ->>> [CLI] pipelines get [MY_ID_2] -{ - "creator_user_name": "[USERNAME]", - "effective_publishing_mode": "DEFAULT_PUBLISHING_MODE", - "last_modified": [UNIX_TIME_MILLIS], - "name": "test-pipeline-[UNIQUE_NAME]", - "pipeline_id": "[MY_ID_2]", - "run_as_user_name": "[USERNAME]", - "spec": { - "channel": "CURRENT", - "deployment": { - "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/acc-[UNIQUE_NAME]/default/state/metadata.json" - }, - "edition": "ADVANCED", - "id": "[MY_ID_2]", - "ingestion_definition": { - "connection_name": "my_new_connection", - "objects": [ - {} - ] - }, - "libraries": [ - { - "file": { - "path": "/Workspace/Users/[USERNAME]/.bundle/acc-[UNIQUE_NAME]/default/files/foo.py" - } - } - ], - "name": "test-pipeline-[UNIQUE_NAME]", - "storage": "dbfs:/pipelines/[MY_ID_2]" - }, - "state": "IDLE" -} - === Verify that original pipeline is gone >>> musterr [CLI] pipelines get [MY_ID] Error: The specified pipeline [MY_ID] was not found. @@ -158,11 +63,5 @@ All files and directories at the following location will be deleted: /Workspace/ Deleting files... Destroy complete! ->>> print_requests -{ - "method": "DELETE", - "path": "/api/2.0/pipelines/[MY_ID_2]" -} - >>> musterr [CLI] pipelines get [MY_ID_2] Error: The specified pipeline [MY_ID_2] was not found. diff --git a/acceptance/bundle/resources/pipelines/recreate-keys/change-ingestion-definition/script b/acceptance/bundle/resources/pipelines/recreate-keys/change-ingestion-definition/script index 82de4115550..d2ef70cb663 100644 --- a/acceptance/bundle/resources/pipelines/recreate-keys/change-ingestion-definition/script +++ b/acceptance/bundle/resources/pipelines/recreate-keys/change-ingestion-definition/script @@ -1,2 +1,38 @@ envsubst < $TESTDIR/../databricks.yml.tmpl > databricks.yml -source $TESTDIR/../_script + +trace cat databricks.yml +touch foo.py +touch bar.py +trace $CLI bundle plan # should show 'create' +$CLI bundle plan -o json > out.plan_create.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle deploy + +ppid1=`read_id.py my` + +print_requests() { + jq --sort-keys 'select(.method != "GET" and (.path | contains("/pipelines")))' < out.requests.txt > out.requests.$DATABRICKS_BUNDLE_ENGINE.json + rm -f out.requests.txt +} + +print_requests + +trace update_file.py databricks.yml $CONFIG_UPDATE +trace $CLI bundle plan # should show 'recreate' +$CLI bundle plan -o json > out.plan_recreate.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle deploy --auto-approve +print_requests + +title "Fetch pipeline ID and verify remote state" + +ppid2=`read_id.py my` +$CLI pipelines get $ppid2 > out.pipeline_get.$DATABRICKS_BUNDLE_ENGINE.json + +title "Verify that original pipeline is gone" +trace musterr $CLI pipelines get $ppid1 + +title "Destroy the pipeline and verify that it's removed from the state and from remote" +trace $CLI bundle destroy --auto-approve + +print_requests +trace musterr $CLI pipelines get $ppid2 +rm -f out.requests.txt diff --git a/acceptance/bundle/validate/pipeline_defaults/databricks.yml b/acceptance/bundle/validate/pipeline_defaults/databricks.yml new file mode 100644 index 00000000000..58b04c996e7 --- /dev/null +++ b/acceptance/bundle/validate/pipeline_defaults/databricks.yml @@ -0,0 +1,13 @@ +bundle: + name: test-bundle + +resources: + pipelines: + regular: + name: regular-pipeline + + ingestion: + name: ingestion-pipeline + ingestion_definition: + connection_name: myconn + objects: [{}] diff --git a/acceptance/bundle/validate/pipeline_defaults/out.test.toml b/acceptance/bundle/validate/pipeline_defaults/out.test.toml new file mode 100644 index 00000000000..f784a183258 --- /dev/null +++ b/acceptance/bundle/validate/pipeline_defaults/out.test.toml @@ -0,0 +1,3 @@ +Local = true +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/validate/pipeline_defaults/output.txt b/acceptance/bundle/validate/pipeline_defaults/output.txt new file mode 100644 index 00000000000..cf157159348 --- /dev/null +++ b/acceptance/bundle/validate/pipeline_defaults/output.txt @@ -0,0 +1,10 @@ +{ + "regular": { + "edition": "ADVANCED", + "channel": "CURRENT" + }, + "ingestion": { + "edition": null, + "channel": null + } +} diff --git a/acceptance/bundle/validate/pipeline_defaults/script b/acceptance/bundle/validate/pipeline_defaults/script new file mode 100644 index 00000000000..5806946a066 --- /dev/null +++ b/acceptance/bundle/validate/pipeline_defaults/script @@ -0,0 +1,2 @@ +# edition and channel defaults should be set for regular pipelines but not for ingestion pipelines +$CLI bundle validate -o json | jq '{regular: .resources.pipelines.regular | {edition, channel}, ingestion: .resources.pipelines.ingestion | {edition, channel}}' diff --git a/bundle/config/mutator/resourcemutator/resource_mutator.go b/bundle/config/mutator/resourcemutator/resource_mutator.go index 209bbcb06a0..3212c1c75af 100644 --- a/bundle/config/mutator/resourcemutator/resource_mutator.go +++ b/bundle/config/mutator/resourcemutator/resource_mutator.go @@ -80,11 +80,6 @@ func applyInitializeMutators(ctx context.Context, b *bundle.Bundle) { {"resources.jobs.*.job_clusters[*].new_cluster.workload_type.clients.notebooks", true}, {"resources.jobs.*.job_clusters[*].new_cluster.workload_type.clients.jobs", true}, - // Pipelines (same as terraform) - // https://github.com/databricks/terraform-provider-databricks/blob/v1.75.0/pipelines/resource_pipeline.go#L253 - {"resources.pipelines.*.edition", "ADVANCED"}, - {"resources.pipelines.*.channel", "CURRENT"}, - // SqlWarehouses (same as terraform) // https://github.com/databricks/terraform-provider-databricks/blob/v1.75.0/sql/resource_sql_endpoint.go#L59 {"resources.sql_warehouses.*.auto_stop_mins", 120}, @@ -109,6 +104,29 @@ func applyInitializeMutators(ctx context.Context, b *bundle.Bundle) { } } + // Pipelines: edition and channel defaults are only applied to non-ingestion + // pipelines. Ingestion pipelines (those with ingestion_definition) use + // serverless compute by default and the API rejects these fields in that mode. + // https://github.com/databricks/terraform-provider-databricks/blob/v1.75.0/pipelines/resource_pipeline.go#L253 + if err := b.Config.Mutate(func(root dyn.Value) (dyn.Value, error) { + return dyn.MapByPattern(root, + dyn.NewPattern(dyn.Key("resources"), dyn.Key("pipelines"), dyn.AnyKey()), + func(_ dyn.Path, p dyn.Value) (dyn.Value, error) { + if _, err := dyn.GetByPath(p, dyn.NewPath(dyn.Key("ingestion_definition"))); err == nil { + return p, nil + } + var err error + if p, err = setPipelineDefault(p, "edition", "ADVANCED"); err != nil { + return dyn.InvalidValue, err + } + return setPipelineDefault(p, "channel", "CURRENT") + }, + ) + }); err != nil { + logdiag.LogError(ctx, err) + return + } + bundle.ApplySeqContext(ctx, b, // Reads (typed): b.Config.Resources.Dashboards (checks dashboard configurations) // Updates (typed): b.Config.Resources.Dashboards[].ParentPath (ensures /Workspace prefix is present) @@ -395,3 +413,14 @@ func mergeResources(src, dst dyn.Value) (dyn.Value, error) { return newDst, nil } + +// setPipelineDefault sets key to value on the pipeline dyn.Value only if key is not already set. +func setPipelineDefault(v dyn.Value, key string, value any) (dyn.Value, error) { + path := dyn.NewPath(dyn.Key(key)) + switch _, err := dyn.GetByPath(v, path); { + case dyn.IsNoSuchKeyError(err): + return dyn.SetByPath(v, path, dyn.V(value)) + default: + return v, err + } +} From 480816096fe3991dbc3c8d050646e1be1f48c850 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Thu, 4 Jun 2026 10:07:57 +0200 Subject: [PATCH 2/4] bundle: simplify pipeline defaults; add cloud regression test MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace the dyn.Value pattern walk with a direct loop over b.Config.Resources.Pipelines to conditionally apply edition/channel defaults. Also remove the now-unused setPipelineDefault helper. Add acceptance/bundle/resources/pipelines/ingestion-defaults, a cloud test (direct engine) that deploys an ingestion pipeline and asserts edition and channel are absent from the POST /api/2.0/pipelines request body. The deploy fails with "connection not found" — not "cluster settings" — confirming the fix. Co-authored-by: Shreyas Goenka --- .../ingestion-defaults/databricks.yml.tmpl | 22 +++++++++ .../ingestion-defaults/out.test.toml | 3 ++ .../pipelines/ingestion-defaults/output.txt | 20 ++++++++ .../pipelines/ingestion-defaults/script | 11 +++++ .../pipelines/ingestion-defaults/test.toml | 13 ++++++ .../resourcemutator/resource_mutator.go | 46 ++++++------------- 6 files changed, 84 insertions(+), 31 deletions(-) create mode 100644 acceptance/bundle/resources/pipelines/ingestion-defaults/databricks.yml.tmpl create mode 100644 acceptance/bundle/resources/pipelines/ingestion-defaults/out.test.toml create mode 100644 acceptance/bundle/resources/pipelines/ingestion-defaults/output.txt create mode 100644 acceptance/bundle/resources/pipelines/ingestion-defaults/script create mode 100644 acceptance/bundle/resources/pipelines/ingestion-defaults/test.toml diff --git a/acceptance/bundle/resources/pipelines/ingestion-defaults/databricks.yml.tmpl b/acceptance/bundle/resources/pipelines/ingestion-defaults/databricks.yml.tmpl new file mode 100644 index 00000000000..87eebe41f7b --- /dev/null +++ b/acceptance/bundle/resources/pipelines/ingestion-defaults/databricks.yml.tmpl @@ -0,0 +1,22 @@ +bundle: + name: test-ingestion-defaults-$UNIQUE_NAME + +resources: + pipelines: + ingestion: + name: test-ingestion-$UNIQUE_NAME + catalog: main + target: default + ingestion_definition: + connection_name: does_not_exist_$UNIQUE_NAME + objects: + - table: + source_catalog: does_not_exist + source_schema: does_not_exist + source_table: does_not_exist + destination_catalog: main + destination_schema: default + +targets: + dev: + default: true diff --git a/acceptance/bundle/resources/pipelines/ingestion-defaults/out.test.toml b/acceptance/bundle/resources/pipelines/ingestion-defaults/out.test.toml new file mode 100644 index 00000000000..00d03354554 --- /dev/null +++ b/acceptance/bundle/resources/pipelines/ingestion-defaults/out.test.toml @@ -0,0 +1,3 @@ +Local = false +Cloud = true +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/pipelines/ingestion-defaults/output.txt b/acceptance/bundle/resources/pipelines/ingestion-defaults/output.txt new file mode 100644 index 00000000000..2b1f087ae92 --- /dev/null +++ b/acceptance/bundle/resources/pipelines/ingestion-defaults/output.txt @@ -0,0 +1,20 @@ + +>>> errcode [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-ingestion-defaults-[UNIQUE_NAME]/dev/files... +Deploying resources... +Error: cannot create resources.pipelines.ingestion: Failed to retrieve connection 'does_not_exist_[UNIQUE_NAME]' from Unity Catalog. Please check whether the connection exists and contact support if the error persists. (400 INVALID_PARAMETER_VALUE) + +Endpoint: POST [DATABRICKS_URL]/api/2.0/pipelines +HTTP Status: 400 Bad Request +API error_code: INVALID_PARAMETER_VALUE +API message: Failed to retrieve connection 'does_not_exist_[UNIQUE_NAME]' from Unity Catalog. Please check whether the connection exists and contact support if the error persists. + + +Exit code: 1 + +>>> jq -s .[] | select(.path=="/api/2.0/pipelines" and .method=="POST") | .body | {name, edition, channel} out.requests.txt +{ + "name": "test-ingestion-[UNIQUE_NAME]", + "edition": null, + "channel": null +} diff --git a/acceptance/bundle/resources/pipelines/ingestion-defaults/script b/acceptance/bundle/resources/pipelines/ingestion-defaults/script new file mode 100644 index 00000000000..3bcd91458d8 --- /dev/null +++ b/acceptance/bundle/resources/pipelines/ingestion-defaults/script @@ -0,0 +1,11 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +# Deploy the ingestion pipeline. It fails because the connection name is +# intentionally invalid, but the POST /api/2.0/pipelines request is sent and +# recorded first. The key assertion: edition and channel are absent from the +# request body — DABs must not inject them for ingestion pipelines. +trace errcode $CLI bundle deploy + +# Show what was sent to the API. edition and channel must be null (absent). +trace jq -s '.[] | select(.path=="/api/2.0/pipelines" and .method=="POST") | .body | {name, edition, channel}' out.requests.txt +rm out.requests.txt diff --git a/acceptance/bundle/resources/pipelines/ingestion-defaults/test.toml b/acceptance/bundle/resources/pipelines/ingestion-defaults/test.toml new file mode 100644 index 00000000000..e6c0a2e7186 --- /dev/null +++ b/acceptance/bundle/resources/pipelines/ingestion-defaults/test.toml @@ -0,0 +1,13 @@ +Local = false +Cloud = true +RecordRequests = true + +Ignore = [ + "databricks.yml", +] + +# Only test with direct engine: the Terraform provider independently injects +# edition/channel defaults at the provider level regardless of what DABs sends, +# so only the direct engine cleanly shows the DABs fix. +[EnvMatrix] +DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/bundle/config/mutator/resourcemutator/resource_mutator.go b/bundle/config/mutator/resourcemutator/resource_mutator.go index 3212c1c75af..90887bfa00a 100644 --- a/bundle/config/mutator/resourcemutator/resource_mutator.go +++ b/bundle/config/mutator/resourcemutator/resource_mutator.go @@ -104,27 +104,22 @@ func applyInitializeMutators(ctx context.Context, b *bundle.Bundle) { } } - // Pipelines: edition and channel defaults are only applied to non-ingestion - // pipelines. Ingestion pipelines (those with ingestion_definition) use - // serverless compute by default and the API rejects these fields in that mode. + // Apply edition and channel defaults only to non-ingestion pipelines. + // Ingestion pipelines use serverless compute by default; the API rejects + // these fields as incompatible cluster settings in that mode. // https://github.com/databricks/terraform-provider-databricks/blob/v1.75.0/pipelines/resource_pipeline.go#L253 - if err := b.Config.Mutate(func(root dyn.Value) (dyn.Value, error) { - return dyn.MapByPattern(root, - dyn.NewPattern(dyn.Key("resources"), dyn.Key("pipelines"), dyn.AnyKey()), - func(_ dyn.Path, p dyn.Value) (dyn.Value, error) { - if _, err := dyn.GetByPath(p, dyn.NewPath(dyn.Key("ingestion_definition"))); err == nil { - return p, nil - } - var err error - if p, err = setPipelineDefault(p, "edition", "ADVANCED"); err != nil { - return dyn.InvalidValue, err - } - return setPipelineDefault(p, "channel", "CURRENT") - }, - ) - }); err != nil { - logdiag.LogError(ctx, err) - return + for name, p := range b.Config.Resources.Pipelines { + if p == nil || p.IngestionDefinition != nil { + continue + } + bundle.SetDefault(ctx, b, "resources.pipelines."+name+".edition", "ADVANCED") + if logdiag.HasError(ctx) { + return + } + bundle.SetDefault(ctx, b, "resources.pipelines."+name+".channel", "CURRENT") + if logdiag.HasError(ctx) { + return + } } bundle.ApplySeqContext(ctx, b, @@ -413,14 +408,3 @@ func mergeResources(src, dst dyn.Value) (dyn.Value, error) { return newDst, nil } - -// setPipelineDefault sets key to value on the pipeline dyn.Value only if key is not already set. -func setPipelineDefault(v dyn.Value, key string, value any) (dyn.Value, error) { - path := dyn.NewPath(dyn.Key(key)) - switch _, err := dyn.GetByPath(v, path); { - case dyn.IsNoSuchKeyError(err): - return dyn.SetByPath(v, path, dyn.V(value)) - default: - return v, err - } -} From 9c7c82c585ef8201ba36b3f7bb42a37313da3ac4 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Thu, 4 Jun 2026 14:00:54 +0200 Subject: [PATCH 3/4] bundle: add cloud regression test for ingestion pipeline defaults Extends the ingestion-defaults acceptance test to run both direct and terraform engines, showing the contrast: - Direct engine: edition/channel absent from POST (DABs fix working) - Terraform engine: edition/channel present (provider-level SetDefault, companion fix in terraform-provider-databricks#5783) Badness clause documents the known terraform engine gap. Co-authored-by: Shreyas Goenka --- .../ingestion-defaults/out.deploy.direct.txt | 13 +++++++++++++ .../out.deploy.terraform.txt | 17 +++++++++++++++++ .../out.post_body.direct.json | 5 +++++ .../out.post_body.terraform.json | 5 +++++ .../pipelines/ingestion-defaults/out.test.toml | 2 +- .../pipelines/ingestion-defaults/output.txt | 18 ------------------ .../pipelines/ingestion-defaults/script | 13 ++++++------- .../pipelines/ingestion-defaults/test.toml | 7 +++---- 8 files changed, 50 insertions(+), 30 deletions(-) create mode 100644 acceptance/bundle/resources/pipelines/ingestion-defaults/out.deploy.direct.txt create mode 100644 acceptance/bundle/resources/pipelines/ingestion-defaults/out.deploy.terraform.txt create mode 100644 acceptance/bundle/resources/pipelines/ingestion-defaults/out.post_body.direct.json create mode 100644 acceptance/bundle/resources/pipelines/ingestion-defaults/out.post_body.terraform.json diff --git a/acceptance/bundle/resources/pipelines/ingestion-defaults/out.deploy.direct.txt b/acceptance/bundle/resources/pipelines/ingestion-defaults/out.deploy.direct.txt new file mode 100644 index 00000000000..d17f6007753 --- /dev/null +++ b/acceptance/bundle/resources/pipelines/ingestion-defaults/out.deploy.direct.txt @@ -0,0 +1,13 @@ + +>>> errcode [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-ingestion-defaults-[UNIQUE_NAME]/dev/files... +Deploying resources... +Error: cannot create resources.pipelines.ingestion: Failed to retrieve connection 'does_not_exist_[UNIQUE_NAME]' from Unity Catalog. Please check whether the connection exists and contact support if the error persists. (400 INVALID_PARAMETER_VALUE) + +Endpoint: POST [DATABRICKS_URL]/api/2.0/pipelines +HTTP Status: 400 Bad Request +API error_code: INVALID_PARAMETER_VALUE +API message: Failed to retrieve connection 'does_not_exist_[UNIQUE_NAME]' from Unity Catalog. Please check whether the connection exists and contact support if the error persists. + + +Exit code: 1 diff --git a/acceptance/bundle/resources/pipelines/ingestion-defaults/out.deploy.terraform.txt b/acceptance/bundle/resources/pipelines/ingestion-defaults/out.deploy.terraform.txt new file mode 100644 index 00000000000..2dfa66cb79b --- /dev/null +++ b/acceptance/bundle/resources/pipelines/ingestion-defaults/out.deploy.terraform.txt @@ -0,0 +1,17 @@ + +>>> errcode [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-ingestion-defaults-[UNIQUE_NAME]/dev/files... +Deploying resources... +Error: terraform apply: exit status 1 + +Error: cannot create pipeline: Failed to retrieve connection 'does_not_exist_[UNIQUE_NAME]' from Unity Catalog. Please check whether the connection exists and contact support if the error persists. + + with databricks_pipeline.ingestion, + on bundle.tf.json line 37, in resource.databricks_pipeline.ingestion: + 37: } + + + +Updating deployment state... + +Exit code: 1 diff --git a/acceptance/bundle/resources/pipelines/ingestion-defaults/out.post_body.direct.json b/acceptance/bundle/resources/pipelines/ingestion-defaults/out.post_body.direct.json new file mode 100644 index 00000000000..e96f0e7b8ac --- /dev/null +++ b/acceptance/bundle/resources/pipelines/ingestion-defaults/out.post_body.direct.json @@ -0,0 +1,5 @@ +{ + "name": "test-ingestion-[UNIQUE_NAME]", + "edition": null, + "channel": null +} diff --git a/acceptance/bundle/resources/pipelines/ingestion-defaults/out.post_body.terraform.json b/acceptance/bundle/resources/pipelines/ingestion-defaults/out.post_body.terraform.json new file mode 100644 index 00000000000..3b83f878234 --- /dev/null +++ b/acceptance/bundle/resources/pipelines/ingestion-defaults/out.post_body.terraform.json @@ -0,0 +1,5 @@ +{ + "name": "test-ingestion-[UNIQUE_NAME]", + "edition": "ADVANCED", + "channel": "CURRENT" +} diff --git a/acceptance/bundle/resources/pipelines/ingestion-defaults/out.test.toml b/acceptance/bundle/resources/pipelines/ingestion-defaults/out.test.toml index 00d03354554..2f97f2d67f5 100644 --- a/acceptance/bundle/resources/pipelines/ingestion-defaults/out.test.toml +++ b/acceptance/bundle/resources/pipelines/ingestion-defaults/out.test.toml @@ -1,3 +1,3 @@ Local = false Cloud = true -EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/resources/pipelines/ingestion-defaults/output.txt b/acceptance/bundle/resources/pipelines/ingestion-defaults/output.txt index 2b1f087ae92..2988aaa9c46 100644 --- a/acceptance/bundle/resources/pipelines/ingestion-defaults/output.txt +++ b/acceptance/bundle/resources/pipelines/ingestion-defaults/output.txt @@ -1,20 +1,2 @@ ->>> errcode [CLI] bundle deploy -Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-ingestion-defaults-[UNIQUE_NAME]/dev/files... -Deploying resources... -Error: cannot create resources.pipelines.ingestion: Failed to retrieve connection 'does_not_exist_[UNIQUE_NAME]' from Unity Catalog. Please check whether the connection exists and contact support if the error persists. (400 INVALID_PARAMETER_VALUE) - -Endpoint: POST [DATABRICKS_URL]/api/2.0/pipelines -HTTP Status: 400 Bad Request -API error_code: INVALID_PARAMETER_VALUE -API message: Failed to retrieve connection 'does_not_exist_[UNIQUE_NAME]' from Unity Catalog. Please check whether the connection exists and contact support if the error persists. - - -Exit code: 1 - >>> jq -s .[] | select(.path=="/api/2.0/pipelines" and .method=="POST") | .body | {name, edition, channel} out.requests.txt -{ - "name": "test-ingestion-[UNIQUE_NAME]", - "edition": null, - "channel": null -} diff --git a/acceptance/bundle/resources/pipelines/ingestion-defaults/script b/acceptance/bundle/resources/pipelines/ingestion-defaults/script index 3bcd91458d8..ffb6283b71b 100644 --- a/acceptance/bundle/resources/pipelines/ingestion-defaults/script +++ b/acceptance/bundle/resources/pipelines/ingestion-defaults/script @@ -1,11 +1,10 @@ envsubst < databricks.yml.tmpl > databricks.yml -# Deploy the ingestion pipeline. It fails because the connection name is -# intentionally invalid, but the POST /api/2.0/pipelines request is sent and -# recorded first. The key assertion: edition and channel are absent from the -# request body — DABs must not inject them for ingestion pipelines. -trace errcode $CLI bundle deploy +# Deploy fails (invalid connection) but the POST /api/2.0/pipelines request is +# sent and recorded. Direct engine: edition and channel are absent (the fix). +# Terraform engine: edition and channel are present (provider injects them; being +# fixed in terraform-provider-databricks). +trace errcode $CLI bundle deploy > out.deploy.$DATABRICKS_BUNDLE_ENGINE.txt 2>&1 -# Show what was sent to the API. edition and channel must be null (absent). -trace jq -s '.[] | select(.path=="/api/2.0/pipelines" and .method=="POST") | .body | {name, edition, channel}' out.requests.txt +trace jq -s '.[] | select(.path=="/api/2.0/pipelines" and .method=="POST") | .body | {name, edition, channel}' out.requests.txt > out.post_body.$DATABRICKS_BUNDLE_ENGINE.json rm out.requests.txt diff --git a/acceptance/bundle/resources/pipelines/ingestion-defaults/test.toml b/acceptance/bundle/resources/pipelines/ingestion-defaults/test.toml index e6c0a2e7186..f2cdc3e6cb3 100644 --- a/acceptance/bundle/resources/pipelines/ingestion-defaults/test.toml +++ b/acceptance/bundle/resources/pipelines/ingestion-defaults/test.toml @@ -2,12 +2,11 @@ Local = false Cloud = true RecordRequests = true +Badness = "Terraform engine still injects edition/channel via provider-level SetDefault (resource_pipeline.go:253); companion fix: terraform-provider-databricks#5783." + Ignore = [ "databricks.yml", ] -# Only test with direct engine: the Terraform provider independently injects -# edition/channel defaults at the provider level regardless of what DABs sends, -# so only the direct engine cleanly shows the DABs fix. [EnvMatrix] -DATABRICKS_BUNDLE_ENGINE = ["direct"] +DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] From 2662ed8f7c60f8d4b8a88a441593c2d69c611f64 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Thu, 4 Jun 2026 19:52:36 +0200 Subject: [PATCH 4/4] bundle: add cloud regression test for ingestion pipeline defaults Replaces the error-capture test with a real end-to-end deployment on the azure cloud environment using a MySQL ingestion connection. The test: - Deploys an ingestion pipeline via the direct engine - Asserts deployment succeeds (no "cluster settings" error) - Verifies edition and channel are absent from the POST request body - Destroys the pipeline on exit Also confirms classic compute (serverless: false) works the same way. The Terraform-engine gap (provider-level SetDefault still injects edition/channel) is documented in Badness and fixed in terraform-provider-databricks#5783. Co-authored-by: Shreyas Goenka --- .../ingestion-defaults/databricks.yml.tmpl | 10 ++-------- .../ingestion-defaults/out.deploy.direct.txt | 13 ------------- .../out.deploy.terraform.txt | 17 ----------------- .../out.post_body.direct.json | 5 ----- .../out.post_body.terraform.json | 5 ----- .../ingestion-defaults/out.test.toml | 3 ++- .../pipelines/ingestion-defaults/output.txt | 11 +++++++++++ .../pipelines/ingestion-defaults/script | 19 ++++++++++++------- .../pipelines/ingestion-defaults/test.toml | 16 ++++++++++------ 9 files changed, 37 insertions(+), 62 deletions(-) delete mode 100644 acceptance/bundle/resources/pipelines/ingestion-defaults/out.deploy.direct.txt delete mode 100644 acceptance/bundle/resources/pipelines/ingestion-defaults/out.deploy.terraform.txt delete mode 100644 acceptance/bundle/resources/pipelines/ingestion-defaults/out.post_body.direct.json delete mode 100644 acceptance/bundle/resources/pipelines/ingestion-defaults/out.post_body.terraform.json diff --git a/acceptance/bundle/resources/pipelines/ingestion-defaults/databricks.yml.tmpl b/acceptance/bundle/resources/pipelines/ingestion-defaults/databricks.yml.tmpl index 87eebe41f7b..17ebdffeefd 100644 --- a/acceptance/bundle/resources/pipelines/ingestion-defaults/databricks.yml.tmpl +++ b/acceptance/bundle/resources/pipelines/ingestion-defaults/databricks.yml.tmpl @@ -8,14 +8,8 @@ resources: catalog: main target: default ingestion_definition: - connection_name: does_not_exist_$UNIQUE_NAME - objects: - - table: - source_catalog: does_not_exist - source_schema: does_not_exist - source_table: does_not_exist - destination_catalog: main - destination_schema: default + connection_name: $INGESTION_CONNECTION_NAME + objects: [] targets: dev: diff --git a/acceptance/bundle/resources/pipelines/ingestion-defaults/out.deploy.direct.txt b/acceptance/bundle/resources/pipelines/ingestion-defaults/out.deploy.direct.txt deleted file mode 100644 index d17f6007753..00000000000 --- a/acceptance/bundle/resources/pipelines/ingestion-defaults/out.deploy.direct.txt +++ /dev/null @@ -1,13 +0,0 @@ - ->>> errcode [CLI] bundle deploy -Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-ingestion-defaults-[UNIQUE_NAME]/dev/files... -Deploying resources... -Error: cannot create resources.pipelines.ingestion: Failed to retrieve connection 'does_not_exist_[UNIQUE_NAME]' from Unity Catalog. Please check whether the connection exists and contact support if the error persists. (400 INVALID_PARAMETER_VALUE) - -Endpoint: POST [DATABRICKS_URL]/api/2.0/pipelines -HTTP Status: 400 Bad Request -API error_code: INVALID_PARAMETER_VALUE -API message: Failed to retrieve connection 'does_not_exist_[UNIQUE_NAME]' from Unity Catalog. Please check whether the connection exists and contact support if the error persists. - - -Exit code: 1 diff --git a/acceptance/bundle/resources/pipelines/ingestion-defaults/out.deploy.terraform.txt b/acceptance/bundle/resources/pipelines/ingestion-defaults/out.deploy.terraform.txt deleted file mode 100644 index 2dfa66cb79b..00000000000 --- a/acceptance/bundle/resources/pipelines/ingestion-defaults/out.deploy.terraform.txt +++ /dev/null @@ -1,17 +0,0 @@ - ->>> errcode [CLI] bundle deploy -Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-ingestion-defaults-[UNIQUE_NAME]/dev/files... -Deploying resources... -Error: terraform apply: exit status 1 - -Error: cannot create pipeline: Failed to retrieve connection 'does_not_exist_[UNIQUE_NAME]' from Unity Catalog. Please check whether the connection exists and contact support if the error persists. - - with databricks_pipeline.ingestion, - on bundle.tf.json line 37, in resource.databricks_pipeline.ingestion: - 37: } - - - -Updating deployment state... - -Exit code: 1 diff --git a/acceptance/bundle/resources/pipelines/ingestion-defaults/out.post_body.direct.json b/acceptance/bundle/resources/pipelines/ingestion-defaults/out.post_body.direct.json deleted file mode 100644 index e96f0e7b8ac..00000000000 --- a/acceptance/bundle/resources/pipelines/ingestion-defaults/out.post_body.direct.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "name": "test-ingestion-[UNIQUE_NAME]", - "edition": null, - "channel": null -} diff --git a/acceptance/bundle/resources/pipelines/ingestion-defaults/out.post_body.terraform.json b/acceptance/bundle/resources/pipelines/ingestion-defaults/out.post_body.terraform.json deleted file mode 100644 index 3b83f878234..00000000000 --- a/acceptance/bundle/resources/pipelines/ingestion-defaults/out.post_body.terraform.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "name": "test-ingestion-[UNIQUE_NAME]", - "edition": "ADVANCED", - "channel": "CURRENT" -} diff --git a/acceptance/bundle/resources/pipelines/ingestion-defaults/out.test.toml b/acceptance/bundle/resources/pipelines/ingestion-defaults/out.test.toml index 2f97f2d67f5..b4b1fb39d67 100644 --- a/acceptance/bundle/resources/pipelines/ingestion-defaults/out.test.toml +++ b/acceptance/bundle/resources/pipelines/ingestion-defaults/out.test.toml @@ -1,3 +1,4 @@ Local = false Cloud = true -EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] +CloudEnvs.azure = true +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/pipelines/ingestion-defaults/output.txt b/acceptance/bundle/resources/pipelines/ingestion-defaults/output.txt index 2988aaa9c46..bf202db9226 100644 --- a/acceptance/bundle/resources/pipelines/ingestion-defaults/output.txt +++ b/acceptance/bundle/resources/pipelines/ingestion-defaults/output.txt @@ -1,2 +1,13 @@ +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-ingestion-defaults-[UNIQUE_NAME]/dev/files... +Deploying resources... +Updating deployment state... +Deployment complete! + >>> jq -s .[] | select(.path=="/api/2.0/pipelines" and .method=="POST") | .body | {name, edition, channel} out.requests.txt +{ + "name": "test-ingestion-[UNIQUE_NAME]", + "edition": null, + "channel": null +} diff --git a/acceptance/bundle/resources/pipelines/ingestion-defaults/script b/acceptance/bundle/resources/pipelines/ingestion-defaults/script index ffb6283b71b..3d9a068be18 100644 --- a/acceptance/bundle/resources/pipelines/ingestion-defaults/script +++ b/acceptance/bundle/resources/pipelines/ingestion-defaults/script @@ -1,10 +1,15 @@ envsubst < databricks.yml.tmpl > databricks.yml -# Deploy fails (invalid connection) but the POST /api/2.0/pipelines request is -# sent and recorded. Direct engine: edition and channel are absent (the fix). -# Terraform engine: edition and channel are present (provider injects them; being -# fixed in terraform-provider-databricks). -trace errcode $CLI bundle deploy > out.deploy.$DATABRICKS_BUNDLE_ENGINE.txt 2>&1 +cleanup() { + trace $CLI bundle destroy --auto-approve 2>/dev/null || true + rm -f out.requests.txt +} +trap cleanup EXIT -trace jq -s '.[] | select(.path=="/api/2.0/pipelines" and .method=="POST") | .body | {name, edition, channel}' out.requests.txt > out.post_body.$DATABRICKS_BUNDLE_ENGINE.json -rm out.requests.txt +# Deploy the ingestion pipeline. Verifies that: +# 1. Deployment succeeds without "cluster settings" errors +# 2. edition and channel are absent from the created pipeline's spec +trace $CLI bundle deploy + +trace jq -s '.[] | select(.path=="/api/2.0/pipelines" and .method=="POST") | .body | {name, edition, channel}' out.requests.txt +rm -f out.requests.txt diff --git a/acceptance/bundle/resources/pipelines/ingestion-defaults/test.toml b/acceptance/bundle/resources/pipelines/ingestion-defaults/test.toml index f2cdc3e6cb3..c287a956cf9 100644 --- a/acceptance/bundle/resources/pipelines/ingestion-defaults/test.toml +++ b/acceptance/bundle/resources/pipelines/ingestion-defaults/test.toml @@ -2,11 +2,15 @@ Local = false Cloud = true RecordRequests = true -Badness = "Terraform engine still injects edition/channel via provider-level SetDefault (resource_pipeline.go:253); companion fix: terraform-provider-databricks#5783." +# The Terraform engine still injects edition/channel at provider level (SetDefault in +# resource_pipeline.go:253); companion fix: terraform-provider-databricks#5783. +# Only test with direct engine here so we cleanly validate the DABs-side fix. +[EnvMatrix] +DATABRICKS_BUNDLE_ENGINE = ["direct"] -Ignore = [ - "databricks.yml", -] +[Env] +# MySQL connection present in the azure test workspace. +INGESTION_CONNECTION_NAME = "name-sbggefbaaidkb" -[EnvMatrix] -DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] +[CloudEnvs] +azure = true