bundle: don't set edition/channel defaults for ingestion pipelines

acceptance/bundle/resources/pipelines/ingestion-defaults/databricks.yml.tmpl

@@ -0,0 +1,16 @@
+bundle:
+  name: test-ingestion-defaults-$UNIQUE_NAME
+
+resources:
+  pipelines:
+    ingestion:
+      name: test-ingestion-$UNIQUE_NAME
+      catalog: main
+      target: default
+      ingestion_definition:
+        connection_name: $INGESTION_CONNECTION_NAME
+        objects: []
+
+targets:
+  dev:
+    default: true

acceptance/bundle/resources/pipelines/ingestion-defaults/output.txt

@@ -0,0 +1,13 @@
+
+>>> [CLI] bundle deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-ingestion-defaults-[UNIQUE_NAME]/dev/files...
+Deploying resources...
+Updating deployment state...
+Deployment complete!
+
+>>> jq -s .[] | select(.path=="/api/2.0/pipelines" and .method=="POST") | .body | {name, edition, channel} out.requests.txt
+{
+  "name": "test-ingestion-[UNIQUE_NAME]",
+  "edition": null,
+  "channel": null
+}

acceptance/bundle/resources/pipelines/ingestion-defaults/script

@@ -0,0 +1,15 @@
+envsubst < databricks.yml.tmpl > databricks.yml
+
+cleanup() {
+    trace $CLI bundle destroy --auto-approve 2>/dev/null || true
+    rm -f out.requests.txt
+}
+trap cleanup EXIT
+
+# Deploy the ingestion pipeline. Verifies that:
+#   1. Deployment succeeds without "cluster settings" errors
+#   2. edition and channel are absent from the created pipeline's spec
+trace $CLI bundle deploy
+
+trace jq -s '.[] | select(.path=="/api/2.0/pipelines" and .method=="POST") | .body | {name, edition, channel}' out.requests.txt
+rm -f out.requests.txt

acceptance/bundle/resources/pipelines/ingestion-defaults/test.toml

@@ -0,0 +1,16 @@
+Local = false
+Cloud = true
+RecordRequests = true
+
+# The Terraform engine still injects edition/channel at provider level (SetDefault in
+# resource_pipeline.go:253); companion fix: terraform-provider-databricks#5783.
+# Only test with direct engine here so we cleanly validate the DABs-side fix.
+[EnvMatrix]
+DATABRICKS_BUNDLE_ENGINE = ["direct"]
+
+[Env]
+# MySQL connection present in the azure test workspace.
+INGESTION_CONNECTION_NAME = "name-sbggefbaaidkb"
+
+[CloudEnvs]
+azure = true

acceptance/bundle/resources/pipelines/recreate-keys/change-ingestion-definition/out.pipeline_get.direct.json

@@ -0,0 +1,31 @@
+{
+  "creator_user_name": "[USERNAME]",
+  "effective_publishing_mode": "DEFAULT_PUBLISHING_MODE",
+  "last_modified": [UNIX_TIME_MILLIS],
+  "name": "test-pipeline-[UNIQUE_NAME]",
+  "pipeline_id": "[MY_ID_2]",
+  "run_as_user_name": "[USERNAME]",
+  "spec": {
+    "deployment": {
+      "kind": "BUNDLE",
+      "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/acc-[UNIQUE_NAME]/default/state/metadata.json"
+    },
+    "id": "[MY_ID_2]",
+    "ingestion_definition": {
+      "connection_name": "my_new_connection",
+      "objects": [
+        {}
+      ]
+    },
+    "libraries": [
+      {
+        "file": {
+          "path": "/Workspace/Users/[USERNAME]/.bundle/acc-[UNIQUE_NAME]/default/files/foo.py"
+        }
+      }
+    ],
+    "name": "test-pipeline-[UNIQUE_NAME]",
+    "storage": "dbfs:/pipelines/[MY_ID_2]"
+  },
+  "state": "IDLE"
+}

acceptance/bundle/resources/pipelines/recreate-keys/change-ingestion-definition/out.pipeline_get.terraform.json

@@ -0,0 +1,33 @@
+{
+  "creator_user_name": "[USERNAME]",
+  "effective_publishing_mode": "DEFAULT_PUBLISHING_MODE",
+  "last_modified": [UNIX_TIME_MILLIS],
+  "name": "test-pipeline-[UNIQUE_NAME]",
+  "pipeline_id": "[MY_ID_2]",
+  "run_as_user_name": "[USERNAME]",
+  "spec": {
+    "channel": "CURRENT",
+    "deployment": {
+      "kind": "BUNDLE",
+      "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/acc-[UNIQUE_NAME]/default/state/metadata.json"
+    },
+    "edition": "ADVANCED",
+    "id": "[MY_ID_2]",
+    "ingestion_definition": {
+      "connection_name": "my_new_connection",
+      "objects": [
+        {}
+      ]
+    },
+    "libraries": [
+      {
+        "file": {
+          "path": "/Workspace/Users/[USERNAME]/.bundle/acc-[UNIQUE_NAME]/default/files/foo.py"
+        }
+      }
+    ],
+    "name": "test-pipeline-[UNIQUE_NAME]",
+    "storage": "dbfs:/pipelines/[MY_ID_2]"
+  },
+  "state": "IDLE"
+}

acceptance/bundle/resources/pipelines/recreate-keys/change-ingestion-definition/out.plan_create.direct.json

@@ -6,12 +6,10 @@
       "action": "create",
       "new_state": {
         "value": {
-          "channel": "CURRENT",
           "deployment": {
             "kind": "BUNDLE",
             "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/acc-[UNIQUE_NAME]/default/state/metadata.json"
           },
-          "edition": "ADVANCED",
           "ingestion_definition": {
             "connection_name": "my_connection",
             "objects": [

acceptance/bundle/resources/pipelines/recreate-keys/change-ingestion-definition/out.plan_recreate.direct.json

@@ -8,12 +8,10 @@
       "action": "recreate",
       "new_state": {
         "value": {
-          "channel": "CURRENT",
           "deployment": {
             "kind": "BUNDLE",
             "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/acc-[UNIQUE_NAME]/default/state/metadata.json"
           },
-          "edition": "ADVANCED",
           "ingestion_definition": {
             "connection_name": "my_new_connection",
             "objects": [
@@ -31,13 +29,11 @@
         }
       },
       "remote_state": {
-        "channel": "CURRENT",
         "creator_user_name": "[USERNAME]",
         "deployment": {
           "kind": "BUNDLE",
           "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/acc-[UNIQUE_NAME]/default/state/metadata.json"
         },
-        "edition": "ADVANCED",
         "effective_publishing_mode": "DEFAULT_PUBLISHING_MODE",
         "id": "[MY_ID]",
         "ingestion_definition": {

acceptance/bundle/resources/pipelines/recreate-keys/change-ingestion-definition/out.requests.direct.json

@@ -0,0 +1,4 @@
+{
+  "method": "DELETE",
+  "path": "/api/2.0/pipelines/[MY_ID_2]"
+}

acceptance/bundle/resources/pipelines/recreate-keys/change-ingestion-definition/out.requests.terraform.json

@@ -0,0 +1,4 @@
+{
+  "method": "DELETE",
+  "path": "/api/2.0/pipelines/[MY_ID_2]"
+}

acceptance/bundle/resources/pipelines/recreate-keys/change-ingestion-definition/output.txt

@@ -25,34 +25,6 @@ Deploying resources...
 Updating deployment state...
 Deployment complete!
 
->>> print_requests
-{
-  "body": {
-    "channel": "CURRENT",
-    "deployment": {
-      "kind": "BUNDLE",
-      "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/acc-[UNIQUE_NAME]/default/state/metadata.json"
-    },
-    "edition": "ADVANCED",
-    "ingestion_definition": {
-      "connection_name": "my_connection",
-      "objects": [
-        {}
-      ]
-    },
-    "libraries": [
-      {
-        "file": {
-          "path": "/Workspace/Users/[USERNAME]/.bundle/acc-[UNIQUE_NAME]/default/files/foo.py"
-        }
-      }
-    ],
-    "name": "test-pipeline-[UNIQUE_NAME]"
-  },
-  "method": "POST",
-  "path": "/api/2.0/pipelines"
-}
-
 >>> update_file.py databricks.yml my_connection my_new_connection
 
 >>> [CLI] bundle plan
@@ -72,74 +44,7 @@ Deploying resources...
 Updating deployment state...
 Deployment complete!
 
->>> print_requests
-{
-  "method": "DELETE",
-  "path": "/api/2.0/pipelines/[MY_ID]"
-}
-{
-  "body": {
-    "channel": "CURRENT",
-    "deployment": {
-      "kind": "BUNDLE",
-      "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/acc-[UNIQUE_NAME]/default/state/metadata.json"
-    },
-    "edition": "ADVANCED",
-    "ingestion_definition": {
-      "connection_name": "my_new_connection",
-      "objects": [
-        {}
-      ]
-    },
-    "libraries": [
-      {
-        "file": {
-          "path": "/Workspace/Users/[USERNAME]/.bundle/acc-[UNIQUE_NAME]/default/files/foo.py"
-        }
-      }
-    ],
-    "name": "test-pipeline-[UNIQUE_NAME]"
-  },
-  "method": "POST",
-  "path": "/api/2.0/pipelines"
-}
-
 === Fetch pipeline ID and verify remote state
->>> [CLI] pipelines get [MY_ID_2]
-{
-  "creator_user_name": "[USERNAME]",
-  "effective_publishing_mode": "DEFAULT_PUBLISHING_MODE",
-  "last_modified": [UNIX_TIME_MILLIS],
-  "name": "test-pipeline-[UNIQUE_NAME]",
-  "pipeline_id": "[MY_ID_2]",
-  "run_as_user_name": "[USERNAME]",
-  "spec": {
-    "channel": "CURRENT",
-    "deployment": {
-      "kind": "BUNDLE",
-      "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/acc-[UNIQUE_NAME]/default/state/metadata.json"
-    },
-    "edition": "ADVANCED",
-    "id": "[MY_ID_2]",
-    "ingestion_definition": {
-      "connection_name": "my_new_connection",
-      "objects": [
-        {}
-      ]
-    },
-    "libraries": [
-      {
-        "file": {
-          "path": "/Workspace/Users/[USERNAME]/.bundle/acc-[UNIQUE_NAME]/default/files/foo.py"
-        }
-      }
-    ],
-    "name": "test-pipeline-[UNIQUE_NAME]",
-    "storage": "dbfs:/pipelines/[MY_ID_2]"
-  },
-  "state": "IDLE"
-}
-
 === Verify that original pipeline is gone
 >>> musterr [CLI] pipelines get [MY_ID]
 Error: The specified pipeline [MY_ID] was not found.
@@ -158,11 +63,5 @@ All files and directories at the following location will be deleted: /Workspace/
 Deleting files...
 Destroy complete!
 
->>> print_requests
-{
-  "method": "DELETE",
-  "path": "/api/2.0/pipelines/[MY_ID_2]"
-}
-
 >>> musterr [CLI] pipelines get [MY_ID_2]
 Error: The specified pipeline [MY_ID_2] was not found.

acceptance/bundle/resources/pipelines/recreate-keys/change-ingestion-definition/script

@@ -1,2 +1,38 @@
 envsubst < $TESTDIR/../databricks.yml.tmpl > databricks.yml
-source $TESTDIR/../_script
+
+trace cat databricks.yml
+touch foo.py
+touch bar.py
+trace $CLI bundle plan  # should show 'create'
+$CLI bundle plan -o json > out.plan_create.$DATABRICKS_BUNDLE_ENGINE.json
+trace $CLI bundle deploy
+
+ppid1=`read_id.py my`
+
+print_requests() {
+    jq --sort-keys 'select(.method != "GET" and (.path | contains("/pipelines")))' < out.requests.txt > out.requests.$DATABRICKS_BUNDLE_ENGINE.json
+    rm -f out.requests.txt
+}
+
+print_requests
+
+trace update_file.py databricks.yml $CONFIG_UPDATE
+trace $CLI bundle plan  # should show 'recreate'
+$CLI bundle plan -o json > out.plan_recreate.$DATABRICKS_BUNDLE_ENGINE.json
+trace $CLI bundle deploy --auto-approve
+print_requests
+
+title "Fetch pipeline ID and verify remote state"
+
+ppid2=`read_id.py my`
+$CLI pipelines get $ppid2 > out.pipeline_get.$DATABRICKS_BUNDLE_ENGINE.json
+
+title "Verify that original pipeline is gone"
+trace musterr $CLI pipelines get $ppid1
+
+title "Destroy the pipeline and verify that it's removed from the state and from remote"
+trace $CLI bundle destroy --auto-approve
+
+print_requests
+trace musterr $CLI pipelines get $ppid2
+rm -f out.requests.txt

acceptance/bundle/validate/pipeline_defaults/databricks.yml

@@ -0,0 +1,13 @@
+bundle:
+  name: test-bundle
+
+resources:
+  pipelines:
+    regular:
+      name: regular-pipeline
+
+    ingestion:
+      name: ingestion-pipeline
+      ingestion_definition:
+        connection_name: myconn
+        objects: [{}]

acceptance/bundle/validate/pipeline_defaults/output.txt

@@ -0,0 +1,10 @@
+{
+  "regular": {
+    "edition": "ADVANCED",
+    "channel": "CURRENT"
+  },
+  "ingestion": {
+    "edition": null,
+    "channel": null
+  }
+}

acceptance/bundle/validate/pipeline_defaults/script

@@ -0,0 +1,2 @@
+# edition and channel defaults should be set for regular pipelines but not for ingestion pipelines
+$CLI bundle validate -o json | jq '{regular: .resources.pipelines.regular | {edition, channel}, ingestion: .resources.pipelines.ingestion | {edition, channel}}'