From 70350248470748606bb810c5786d316377f7c2b8 Mon Sep 17 00:00:00 2001 From: Bhukya Date: Wed, 18 Nov 2026 10:07:36 +0000 Subject: [PATCH 1/2] Translate the testing paths of pipelines and upload local test wheels Co-authored-by: Isaac --- .../testing-dynamic-version/databricks.yml | 25 ++++ .../testing-dynamic-version/out.test.toml | 2 + .../testing-dynamic-version/output.txt | 27 ++++ .../pipelines/testing-dynamic-version/script | 9 ++ .../testing-dynamic-version/test.toml | 4 + .../tests/test_orders.py | 2 + .../transformations/orders.py | 1 + .../databricks.yml | 21 +++ .../testing-same-name-libraries/out.test.toml | 2 + .../testing-same-name-libraries/output.txt | 10 ++ .../test_helpers-0.1-py3-none-any.whl | 0 .../testing-same-name-libraries/script | 1 + .../testing-same-name-libraries/test.toml | 6 + .../tests/test_helpers-0.1-py3-none-any.whl | 0 .../tests/test_orders.py | 2 + .../transformations/orders.py | 1 + .../pipelines/testing/databricks.yml | 19 +++ .../dist/test_helpers-0.1-py3-none-any.whl | 0 .../resources/pipelines/testing/out.test.toml | 2 + .../resources/pipelines/testing/output.txt | 132 ++++++++++++++++++ .../pipelines/testing/pyproject.toml | 6 + .../bundle/resources/pipelines/testing/script | 35 +++++ .../resources/pipelines/testing/test.toml | 4 + .../pipelines/testing/tests/requirements.txt | 1 + .../pipelines/testing/tests/test_orders.py | 2 + .../testing/transformations/orders.py | 1 + .../mutator/paths/pipeline_paths_visitor.go | 54 ++++--- .../paths/pipeline_paths_visitor_test.go | 7 + bundle/libraries/expand_glob_references.go | 13 ++ bundle/libraries/remote_path.go | 1 + bundle/libraries/same_name_libraries.go | 1 + bundle/libraries/switch_to_patched_wheels.go | 18 +++ 32 files changed, 379 insertions(+), 30 deletions(-) create mode 100644 acceptance/bundle/resources/pipelines/testing-dynamic-version/databricks.yml create mode 100644 acceptance/bundle/resources/pipelines/testing-dynamic-version/out.test.toml create mode 100644 acceptance/bundle/resources/pipelines/testing-dynamic-version/output.txt create mode 100644 acceptance/bundle/resources/pipelines/testing-dynamic-version/script create mode 100644 acceptance/bundle/resources/pipelines/testing-dynamic-version/test.toml create mode 100644 acceptance/bundle/resources/pipelines/testing-dynamic-version/tests/test_orders.py create mode 100644 acceptance/bundle/resources/pipelines/testing-dynamic-version/transformations/orders.py create mode 100644 acceptance/bundle/resources/pipelines/testing-same-name-libraries/databricks.yml create mode 100644 acceptance/bundle/resources/pipelines/testing-same-name-libraries/out.test.toml create mode 100644 acceptance/bundle/resources/pipelines/testing-same-name-libraries/output.txt create mode 100644 acceptance/bundle/resources/pipelines/testing-same-name-libraries/pipeline/test_helpers-0.1-py3-none-any.whl create mode 100644 acceptance/bundle/resources/pipelines/testing-same-name-libraries/script create mode 100644 acceptance/bundle/resources/pipelines/testing-same-name-libraries/test.toml create mode 100644 acceptance/bundle/resources/pipelines/testing-same-name-libraries/tests/test_helpers-0.1-py3-none-any.whl create mode 100644 acceptance/bundle/resources/pipelines/testing-same-name-libraries/tests/test_orders.py create mode 100644 acceptance/bundle/resources/pipelines/testing-same-name-libraries/transformations/orders.py create mode 100644 acceptance/bundle/resources/pipelines/testing/databricks.yml create mode 100644 acceptance/bundle/resources/pipelines/testing/dist/test_helpers-0.1-py3-none-any.whl create mode 100644 acceptance/bundle/resources/pipelines/testing/out.test.toml create mode 100644 acceptance/bundle/resources/pipelines/testing/output.txt create mode 100644 acceptance/bundle/resources/pipelines/testing/pyproject.toml create mode 100644 acceptance/bundle/resources/pipelines/testing/script create mode 100644 acceptance/bundle/resources/pipelines/testing/test.toml create mode 100644 acceptance/bundle/resources/pipelines/testing/tests/requirements.txt create mode 100644 acceptance/bundle/resources/pipelines/testing/tests/test_orders.py create mode 100644 acceptance/bundle/resources/pipelines/testing/transformations/orders.py diff --git a/acceptance/bundle/resources/pipelines/testing-dynamic-version/databricks.yml b/acceptance/bundle/resources/pipelines/testing-dynamic-version/databricks.yml new file mode 100644 index 00000000000..09b1f8647e0 --- /dev/null +++ b/acceptance/bundle/resources/pipelines/testing-dynamic-version/databricks.yml @@ -0,0 +1,25 @@ +bundle: + name: pipeline-testing-dynamic-version + +artifacts: + other_test_code: + type: whl + files: + - source: dist/other_test_code-0.0.1-py3-none-any.whl + dynamic_version: true + +resources: + pipelines: + orders: + name: orders + libraries: + - glob: + include: ./transformations/** + testing: + sources: + - glob: + include: ./tests/** + environment: + dependencies: + - ./dist/other_test_code-0.0.1-py3-none-any.whl + - ./dist/other_test_code-0.0.1-py3-none-any.whl[test] diff --git a/acceptance/bundle/resources/pipelines/testing-dynamic-version/out.test.toml b/acceptance/bundle/resources/pipelines/testing-dynamic-version/out.test.toml new file mode 100644 index 00000000000..a927a5fbc06 --- /dev/null +++ b/acceptance/bundle/resources/pipelines/testing-dynamic-version/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/resources/pipelines/testing-dynamic-version/output.txt b/acceptance/bundle/resources/pipelines/testing-dynamic-version/output.txt new file mode 100644 index 00000000000..9af556df861 --- /dev/null +++ b/acceptance/bundle/resources/pipelines/testing-dynamic-version/output.txt @@ -0,0 +1,27 @@ + +=== Test dependencies use the patched wheel, with extras kept + +>>> [CLI] bundle deploy +Uploading .databricks/bundle/default/patched_wheels/other_test_code_other_test_code/other_test_code-0.0.1+[UNIX_TIME_NANOS]-py3-none-any.whl... +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/pipeline-testing-dynamic-version/default/files... +Created pipelines.orders +Files: 4 uploaded, 0 deleted +Resources: 1 created, 0 changed, 0 deleted, 0 unchanged + +>>> print_requests.py //api/2.0/pipelines --nostamp +[ + "/Workspace/Users/[USERNAME]/.bundle/pipeline-testing-dynamic-version/default/artifacts/.internal/other_test_code-0.0.1+[UNIX_TIME_NANOS]-py3-none-any.whl", + "/Workspace/Users/[USERNAME]/.bundle/pipeline-testing-dynamic-version/default/artifacts/.internal/other_test_code-0.0.1+[UNIX_TIME_NANOS]-py3-none-any.whl[test]" +] + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.pipelines.orders + +This action will result in the deletion of the following Lakeflow Spark Declarative Pipelines along with the +Streaming Tables (STs) and Materialized Views (MVs) managed by them. Set 'cascade_on_destroy: false' on a pipeline to retain datasets on pipeline deletion: + delete resources.pipelines.orders + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/pipeline-testing-dynamic-version/default + +Destroy: 1 deleted diff --git a/acceptance/bundle/resources/pipelines/testing-dynamic-version/script b/acceptance/bundle/resources/pipelines/testing-dynamic-version/script new file mode 100644 index 00000000000..e71c775893f --- /dev/null +++ b/acceptance/bundle/resources/pipelines/testing-dynamic-version/script @@ -0,0 +1,9 @@ +trap "rm -fr dist" EXIT +mkdir dist +cp $TESTDIR/../../../artifacts/whl_prebuilt_multiple/dist/lib/other_test_code-0.0.1-py3-none-any.whl dist + +title "Test dependencies use the patched wheel, with extras kept" +trace $CLI bundle deploy +trace print_requests.py //api/2.0/pipelines --nostamp | jq '.body.testing.environment.dependencies' +trace $CLI bundle destroy --auto-approve +rm "$OUT_REQUESTS" diff --git a/acceptance/bundle/resources/pipelines/testing-dynamic-version/test.toml b/acceptance/bundle/resources/pipelines/testing-dynamic-version/test.toml new file mode 100644 index 00000000000..25d8c36639f --- /dev/null +++ b/acceptance/bundle/resources/pipelines/testing-dynamic-version/test.toml @@ -0,0 +1,4 @@ +# Runs only against the local test server. +Cloud = false +RecordRequests = true +Ignore = [".databricks"] diff --git a/acceptance/bundle/resources/pipelines/testing-dynamic-version/tests/test_orders.py b/acceptance/bundle/resources/pipelines/testing-dynamic-version/tests/test_orders.py new file mode 100644 index 00000000000..eb4379fd048 --- /dev/null +++ b/acceptance/bundle/resources/pipelines/testing-dynamic-version/tests/test_orders.py @@ -0,0 +1,2 @@ +def test_orders(): + pass diff --git a/acceptance/bundle/resources/pipelines/testing-dynamic-version/transformations/orders.py b/acceptance/bundle/resources/pipelines/testing-dynamic-version/transformations/orders.py new file mode 100644 index 00000000000..3a22621c3bf --- /dev/null +++ b/acceptance/bundle/resources/pipelines/testing-dynamic-version/transformations/orders.py @@ -0,0 +1 @@ +from pyspark import pipelines as dp diff --git a/acceptance/bundle/resources/pipelines/testing-same-name-libraries/databricks.yml b/acceptance/bundle/resources/pipelines/testing-same-name-libraries/databricks.yml new file mode 100644 index 00000000000..381095e1cd3 --- /dev/null +++ b/acceptance/bundle/resources/pipelines/testing-same-name-libraries/databricks.yml @@ -0,0 +1,21 @@ +bundle: + name: pipeline-testing-same-name-libraries + +resources: + pipelines: + orders: + name: orders + libraries: + - file: + path: ./transformations/orders.py + environment: + dependencies: + - ./pipeline/test_helpers-0.1-py3-none-any.whl + testing: + sources: + - glob: + include: ./tests/** + environment: + dependencies: + # A different wheel with the same file name would be uploaded to the same path. + - ./tests/test_helpers-0.1-py3-none-any.whl diff --git a/acceptance/bundle/resources/pipelines/testing-same-name-libraries/out.test.toml b/acceptance/bundle/resources/pipelines/testing-same-name-libraries/out.test.toml new file mode 100644 index 00000000000..a927a5fbc06 --- /dev/null +++ b/acceptance/bundle/resources/pipelines/testing-same-name-libraries/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/resources/pipelines/testing-same-name-libraries/output.txt b/acceptance/bundle/resources/pipelines/testing-same-name-libraries/output.txt new file mode 100644 index 00000000000..68399999172 --- /dev/null +++ b/acceptance/bundle/resources/pipelines/testing-same-name-libraries/output.txt @@ -0,0 +1,10 @@ + +>>> musterr [CLI] bundle deploy +Error: Duplicate local library names: test_helpers-0.1-py3-none-any.whl + at resources.pipelines.orders.environment.dependencies[0] + resources.pipelines.orders.testing.environment.dependencies[0] + in databricks.yml:13:13 + databricks.yml:21:15 + +Local library names must be unique but found libraries with the same name: pipeline/test_helpers-0.1-py3-none-any.whl, tests/test_helpers-0.1-py3-none-any.whl + diff --git a/acceptance/bundle/resources/pipelines/testing-same-name-libraries/pipeline/test_helpers-0.1-py3-none-any.whl b/acceptance/bundle/resources/pipelines/testing-same-name-libraries/pipeline/test_helpers-0.1-py3-none-any.whl new file mode 100644 index 00000000000..e69de29bb2d diff --git a/acceptance/bundle/resources/pipelines/testing-same-name-libraries/script b/acceptance/bundle/resources/pipelines/testing-same-name-libraries/script new file mode 100644 index 00000000000..32f10d10630 --- /dev/null +++ b/acceptance/bundle/resources/pipelines/testing-same-name-libraries/script @@ -0,0 +1 @@ +trace musterr $CLI bundle deploy diff --git a/acceptance/bundle/resources/pipelines/testing-same-name-libraries/test.toml b/acceptance/bundle/resources/pipelines/testing-same-name-libraries/test.toml new file mode 100644 index 00000000000..d77028ab40a --- /dev/null +++ b/acceptance/bundle/resources/pipelines/testing-same-name-libraries/test.toml @@ -0,0 +1,6 @@ +# Runs only against the local test server. +Cloud = false + +[[Repls]] +Old = '\\' +New = '/' diff --git a/acceptance/bundle/resources/pipelines/testing-same-name-libraries/tests/test_helpers-0.1-py3-none-any.whl b/acceptance/bundle/resources/pipelines/testing-same-name-libraries/tests/test_helpers-0.1-py3-none-any.whl new file mode 100644 index 00000000000..e69de29bb2d diff --git a/acceptance/bundle/resources/pipelines/testing-same-name-libraries/tests/test_orders.py b/acceptance/bundle/resources/pipelines/testing-same-name-libraries/tests/test_orders.py new file mode 100644 index 00000000000..eb4379fd048 --- /dev/null +++ b/acceptance/bundle/resources/pipelines/testing-same-name-libraries/tests/test_orders.py @@ -0,0 +1,2 @@ +def test_orders(): + pass diff --git a/acceptance/bundle/resources/pipelines/testing-same-name-libraries/transformations/orders.py b/acceptance/bundle/resources/pipelines/testing-same-name-libraries/transformations/orders.py new file mode 100644 index 00000000000..7d4290a117a --- /dev/null +++ b/acceptance/bundle/resources/pipelines/testing-same-name-libraries/transformations/orders.py @@ -0,0 +1 @@ +x = 1 diff --git a/acceptance/bundle/resources/pipelines/testing/databricks.yml b/acceptance/bundle/resources/pipelines/testing/databricks.yml new file mode 100644 index 00000000000..346cc9d7963 --- /dev/null +++ b/acceptance/bundle/resources/pipelines/testing/databricks.yml @@ -0,0 +1,19 @@ +bundle: + name: pipeline-testing + +resources: + pipelines: + orders: + name: orders + libraries: + - glob: + include: ./transformations/** + testing: + sources: + - glob: + include: ./tests/** + environment: + dependencies: + - -e .[test] + - -r ./tests/requirements.txt + - ./dist/*.whl diff --git a/acceptance/bundle/resources/pipelines/testing/dist/test_helpers-0.1-py3-none-any.whl b/acceptance/bundle/resources/pipelines/testing/dist/test_helpers-0.1-py3-none-any.whl new file mode 100644 index 00000000000..e69de29bb2d diff --git a/acceptance/bundle/resources/pipelines/testing/out.test.toml b/acceptance/bundle/resources/pipelines/testing/out.test.toml new file mode 100644 index 00000000000..a927a5fbc06 --- /dev/null +++ b/acceptance/bundle/resources/pipelines/testing/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/resources/pipelines/testing/output.txt b/acceptance/bundle/resources/pipelines/testing/output.txt new file mode 100644 index 00000000000..3cccee04481 --- /dev/null +++ b/acceptance/bundle/resources/pipelines/testing/output.txt @@ -0,0 +1,132 @@ + +=== Relative test paths resolve to the deployed workspace paths + +>>> [CLI] bundle validate -o json +{ + "environment": { + "dependencies": [ + "-e /Workspace/Users/[USERNAME]/.bundle/pipeline-testing/default/files[test]", + "-r /Workspace/Users/[USERNAME]/.bundle/pipeline-testing/default/files/tests/requirements.txt", + "./dist/*.whl" + ] + }, + "sources": [ + { + "glob": { + "include": "/Workspace/Users/[USERNAME]/.bundle/pipeline-testing/default/files/tests/**" + } + } + ] +} + +=== Deploy sends testing, with the local wheel uploaded + +>>> [CLI] bundle deploy +Uploading dist/test_helpers-0.1-py3-none-any.whl... +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/pipeline-testing/default/files... +Created pipelines.orders +Files: 6 uploaded, 0 deleted +Resources: 1 created, 0 changed, 0 deleted, 0 unchanged + +>>> print_requests.py //api/2.0/pipelines --nostamp +{ + "method": "POST", + "path": "/api/2.0/pipelines", + "testing": { + "environment": { + "dependencies": [ + "-e /Workspace/Users/[USERNAME]/.bundle/pipeline-testing/default/files[test]", + "-r /Workspace/Users/[USERNAME]/.bundle/pipeline-testing/default/files/tests/requirements.txt", + "/Workspace/Users/[USERNAME]/.bundle/pipeline-testing/default/artifacts/.internal/test_helpers-0.1-py3-none-any.whl" + ] + }, + "sources": [ + { + "glob": { + "include": "/Workspace/Users/[USERNAME]/.bundle/pipeline-testing/default/files/tests/**" + } + } + ] + } +} + +=== No changes after the deploy + +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged + +=== A remote edit that drops testing shows in the plan + +>>> [CLI] bundle plan -o json +"update" + +>>> [CLI] bundle deploy +Uploading dist/test_helpers-0.1-py3-none-any.whl... +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/pipeline-testing/default/files... +Updated pipelines.orders +Files: 1 uploaded, 0 deleted +Resources: 0 created, 1 changed, 0 deleted, 0 unchanged + +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged + +=== Changing testing updates the pipeline + +>>> update_file.py databricks.yml ./tests/** ./tests/unit/** + +>>> [CLI] bundle plan +update pipelines.orders + +Plan: 0 to add, 1 to change, 0 to delete, 0 unchanged + +>>> [CLI] bundle deploy +Uploading dist/test_helpers-0.1-py3-none-any.whl... +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/pipeline-testing/default/files... +Updated pipelines.orders +Files: 1 uploaded, 0 deleted +Resources: 0 created, 1 changed, 0 deleted, 0 unchanged + +>>> print_requests.py //api/2.0/pipelines --nostamp +{ + "method": "PUT", + "path": "/api/2.0/pipelines/[ORDERS_ID]", + "sources": [ + { + "glob": { + "include": "/Workspace/Users/[USERNAME]/.bundle/pipeline-testing/default/files/tests/unit/**" + } + } + ] +} + +=== Removing testing sends an update without it + +>>> [CLI] bundle plan +update pipelines.orders + +Plan: 0 to add, 1 to change, 0 to delete, 0 unchanged + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/pipeline-testing/default/files... +Updated pipelines.orders +Files: 1 uploaded, 0 deleted +Resources: 0 created, 1 changed, 0 deleted, 0 unchanged + +>>> print_requests.py //api/2.0/pipelines --nostamp +{ + "method": "PUT", + "path": "/api/2.0/pipelines/[ORDERS_ID]", + "has_testing": false +} + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.pipelines.orders + +This action will result in the deletion of the following Lakeflow Spark Declarative Pipelines along with the +Streaming Tables (STs) and Materialized Views (MVs) managed by them. Set 'cascade_on_destroy: false' on a pipeline to retain datasets on pipeline deletion: + delete resources.pipelines.orders + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/pipeline-testing/default + +Destroy: 1 deleted diff --git a/acceptance/bundle/resources/pipelines/testing/pyproject.toml b/acceptance/bundle/resources/pipelines/testing/pyproject.toml new file mode 100644 index 00000000000..1c198acc7cb --- /dev/null +++ b/acceptance/bundle/resources/pipelines/testing/pyproject.toml @@ -0,0 +1,6 @@ +[project] +name = "orders" +version = "0.1" + +[project.optional-dependencies] +test = ["faker"] diff --git a/acceptance/bundle/resources/pipelines/testing/script b/acceptance/bundle/resources/pipelines/testing/script new file mode 100644 index 00000000000..2c12917d00f --- /dev/null +++ b/acceptance/bundle/resources/pipelines/testing/script @@ -0,0 +1,35 @@ +title "Relative test paths resolve to the deployed workspace paths" +trace $CLI bundle validate -o json | jq .resources.pipelines.orders.testing + +title "Deploy sends testing, with the local wheel uploaded" +trace $CLI bundle deploy +trace print_requests.py //api/2.0/pipelines --nostamp | jq '{method, path, testing: .body.testing}' + +title "No changes after the deploy" +trace $CLI bundle plan +rm "$OUT_REQUESTS" + +title "A remote edit that drops testing shows in the plan" +PIPELINE_ID=$(read_id.py orders) +MSYS_NO_PATHCONV=1 $CLI api get "/api/2.0/pipelines/$PIPELINE_ID" | jq '.spec | del(.testing)' > edit.json +MSYS_NO_PATHCONV=1 $CLI api put "/api/2.0/pipelines/$PIPELINE_ID" --json @edit.json &> LOG.edit +rm edit.json +trace $CLI bundle plan -o json | jq '.plan["resources.pipelines.orders"].changes.testing.action' +trace $CLI bundle deploy +trace $CLI bundle plan +rm "$OUT_REQUESTS" + +title "Changing testing updates the pipeline" +trace update_file.py databricks.yml './tests/**' './tests/unit/**' +trace $CLI bundle plan +trace $CLI bundle deploy +trace print_requests.py //api/2.0/pipelines --nostamp | jq '{method, path, sources: .body.testing.sources}' + +title "Removing testing sends an update without it" +update_file.py databricks.yml "$(sed -n '/^ testing:/,$p' databricks.yml)" "" +trace $CLI bundle plan +trace $CLI bundle deploy +trace print_requests.py //api/2.0/pipelines --nostamp | jq '{method, path, has_testing: (.body | has("testing"))}' + +trace $CLI bundle destroy --auto-approve +rm "$OUT_REQUESTS" diff --git a/acceptance/bundle/resources/pipelines/testing/test.toml b/acceptance/bundle/resources/pipelines/testing/test.toml new file mode 100644 index 00000000000..25d8c36639f --- /dev/null +++ b/acceptance/bundle/resources/pipelines/testing/test.toml @@ -0,0 +1,4 @@ +# Runs only against the local test server. +Cloud = false +RecordRequests = true +Ignore = [".databricks"] diff --git a/acceptance/bundle/resources/pipelines/testing/tests/requirements.txt b/acceptance/bundle/resources/pipelines/testing/tests/requirements.txt new file mode 100644 index 00000000000..9d7f6e827a1 --- /dev/null +++ b/acceptance/bundle/resources/pipelines/testing/tests/requirements.txt @@ -0,0 +1 @@ +faker diff --git a/acceptance/bundle/resources/pipelines/testing/tests/test_orders.py b/acceptance/bundle/resources/pipelines/testing/tests/test_orders.py new file mode 100644 index 00000000000..eb4379fd048 --- /dev/null +++ b/acceptance/bundle/resources/pipelines/testing/tests/test_orders.py @@ -0,0 +1,2 @@ +def test_orders(): + pass diff --git a/acceptance/bundle/resources/pipelines/testing/transformations/orders.py b/acceptance/bundle/resources/pipelines/testing/transformations/orders.py new file mode 100644 index 00000000000..3a22621c3bf --- /dev/null +++ b/acceptance/bundle/resources/pipelines/testing/transformations/orders.py @@ -0,0 +1 @@ +from pyspark import pipelines as dp diff --git a/bundle/config/mutator/paths/pipeline_paths_visitor.go b/bundle/config/mutator/paths/pipeline_paths_visitor.go index b5af086dd34..0824dbbd696 100644 --- a/bundle/config/mutator/paths/pipeline_paths_visitor.go +++ b/bundle/config/mutator/paths/pipeline_paths_visitor.go @@ -44,48 +44,42 @@ func pipelineRewritePatterns() []pipelineRewritePattern { mode: TranslateModeDirectory, skipRewrite: noSkipRewrite, }, + { + pattern: base.Append(dyn.Key("testing"), dyn.Key("sources"), dyn.AnyIndex(), dyn.Key("glob"), dyn.Key("include")), + mode: TranslateModeGlob, + skipRewrite: noSkipRewrite, + }, } return allPatterns } func pipelineLibrariesRewritePatterns() []pipelineRewritePattern { - pipelineEnvironmentsPatterns := []pipelineRewritePattern{ - { - pattern: dyn.NewPattern( - dyn.Key("resources"), - dyn.Key("pipelines"), - dyn.AnyKey(), - dyn.Key("environment"), - dyn.Key("dependencies"), - dyn.AnyIndex(), - ), - mode: TranslateModeLocalRelativeWithPrefix, - skipRewrite: func(s string) bool { - return !libraries.IsLibraryLocal(s) - }, - }, + // Test dependencies take the same forms as the pipeline's environment dependencies. + dependencies := []dyn.Pattern{ + base.Append(dyn.Key("environment"), dyn.Key("dependencies"), dyn.AnyIndex()), + base.Append(dyn.Key("testing"), dyn.Key("environment"), dyn.Key("dependencies"), dyn.AnyIndex()), } - pipelineEnvironmentsPatternsWithPipFlags := []pipelineRewritePattern{ - { - dyn.NewPattern( - dyn.Key("resources"), - dyn.Key("pipelines"), - dyn.AnyKey(), - dyn.Key("environment"), - dyn.Key("dependencies"), - dyn.AnyIndex(), - ), - TranslateModeEnvironmentPipFlag, - func(s string) bool { + var patterns []pipelineRewritePattern + for _, pattern := range dependencies { + patterns = append(patterns, pipelineRewritePattern{ + pattern: pattern, + mode: TranslateModeLocalRelativeWithPrefix, + skipRewrite: func(s string) bool { return !libraries.IsLibraryLocal(s) }, + }) + } + for _, pattern := range dependencies { + patterns = append(patterns, pipelineRewritePattern{ + pattern: pattern, + mode: TranslateModeEnvironmentPipFlag, + skipRewrite: func(s string) bool { _, _, ok := libraries.IsLocalPathInPipFlag(s) return !ok }, - }, + }) } - - return append(pipelineEnvironmentsPatterns, pipelineEnvironmentsPatternsWithPipFlags...) + return patterns } func VisitPipelinePaths(value dyn.Value, fn VisitFunc) (dyn.Value, error) { diff --git a/bundle/config/mutator/paths/pipeline_paths_visitor_test.go b/bundle/config/mutator/paths/pipeline_paths_visitor_test.go index eb9748bba37..6b527cf7942 100644 --- a/bundle/config/mutator/paths/pipeline_paths_visitor_test.go +++ b/bundle/config/mutator/paths/pipeline_paths_visitor_test.go @@ -39,6 +39,13 @@ func TestVisitPipelinePaths(t *testing.T) { "src/foo.whl", }, }, + // The acceptance test bundle/resources/pipelines/testing covers the testing paths + // that are rewritten. A package name is not a path. + Testing: &pipelines.PipelineTesting{ + Environment: &pipelines.PipelineTestEnvironment{ + Dependencies: []string{"faker"}, + }, + }, }, }, }, diff --git a/bundle/libraries/expand_glob_references.go b/bundle/libraries/expand_glob_references.go index df089f880a7..91f5ab261b1 100644 --- a/bundle/libraries/expand_glob_references.go +++ b/bundle/libraries/expand_glob_references.go @@ -198,6 +198,15 @@ var pipelineEnvDepsPattern = dyn.NewPattern( dyn.Key("dependencies"), ) +var pipelineTestingEnvDepsPattern = dyn.NewPattern( + dyn.Key("resources"), + dyn.Key("pipelines"), + dyn.AnyKey(), + dyn.Key("testing"), + dyn.Key("environment"), + dyn.Key("dependencies"), +) + var clusterLibrariesPattern = dyn.NewPattern( dyn.Key("resources"), dyn.Key("clusters"), @@ -223,6 +232,10 @@ func (e *expand) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { pattern: pipelineEnvDepsPattern, fn: expandEnvironmentDeps, }, + { + pattern: pipelineTestingEnvDepsPattern, + fn: expandEnvironmentDeps, + }, { pattern: clusterLibrariesPattern, fn: expandLibraries, diff --git a/bundle/libraries/remote_path.go b/bundle/libraries/remote_path.go index ff6be05c3b2..020fd5bbdaa 100644 --- a/bundle/libraries/remote_path.go +++ b/bundle/libraries/remote_path.go @@ -144,6 +144,7 @@ func collectLocalLibraries(b *bundle.Bundle) (map[string][]LocationToUpdate, err clusterLibrariesPattern.Append(dyn.AnyIndex(), dyn.Key("jar")), envDepsPattern.Append(dyn.AnyIndex()), pipelineEnvDepsPattern.Append(dyn.AnyIndex()), + pipelineTestingEnvDepsPattern.Append(dyn.AnyIndex()), // The AI Runtime task's code_source_path is a local archive (typically an // artifact-built .tar.gz) that must be uploaded and referenced by its remote // path, exactly like a wheel or jar library. diff --git a/bundle/libraries/same_name_libraries.go b/bundle/libraries/same_name_libraries.go index 722d42c7f64..e4e1c62c614 100644 --- a/bundle/libraries/same_name_libraries.go +++ b/bundle/libraries/same_name_libraries.go @@ -22,6 +22,7 @@ var patterns = []dyn.Pattern{ clusterLibrariesPattern.Append(dyn.AnyIndex(), dyn.Key("jar")), envDepsPattern.Append(dyn.AnyIndex()), pipelineEnvDepsPattern.Append(dyn.AnyIndex()), + pipelineTestingEnvDepsPattern.Append(dyn.AnyIndex()), } type libData struct { diff --git a/bundle/libraries/switch_to_patched_wheels.go b/bundle/libraries/switch_to_patched_wheels.go index 3c5ff184a97..0169f1adb3a 100644 --- a/bundle/libraries/switch_to_patched_wheels.go +++ b/bundle/libraries/switch_to_patched_wheels.go @@ -95,6 +95,24 @@ func (c switchToPatchedWheels) Apply(ctx context.Context, b *bundle.Bundle) diag } } + // Update resources.pipelines.*.testing.environment.dependencies[*] + for pipelineName, pipelineRef := range b.Config.Resources.Pipelines { + if pipelineRef == nil || pipelineRef.Testing == nil || pipelineRef.Testing.Environment == nil { + continue + } + deps := pipelineRef.Testing.Environment.Dependencies + for depInd, dep := range deps { + base, extras := patchwheel.SplitWheelExtras(dep) + repl := replacements[base] + if repl != "" { + log.Debugf(ctx, "Updating resources.pipelines.%s.testing.environment.dependencies[%d] from %s to %s", pipelineName, depInd, dep, repl+extras) + deps[depInd] = repl + extras + } else { + log.Debugf(ctx, "Not updating resources.pipelines.%s.testing.environment.dependencies[%d] from %s. Available replacements: %v", pipelineName, depInd, dep, slices.Sorted(maps.Keys(replacements))) + } + } + } + return nil } From 393738d19bf13d7839950736f7d1ce66138d834e Mon Sep 17 00:00:00 2001 From: Bhukya Date: Wed, 18 Nov 2026 10:08:50 +0000 Subject: [PATCH 2/2] Keep the testing acc tests to the deploy request, with the wheel checked in Co-authored-by: Isaac --- .../other_test_code-0.0.1-py3-none-any.whl | Bin 0 -> 1893 bytes .../pipelines/testing-dynamic-version/script | 3 - .../resources/pipelines/testing/output.txt | 69 ------------------ .../bundle/resources/pipelines/testing/script | 26 ------- 4 files changed, 98 deletions(-) create mode 100644 acceptance/bundle/resources/pipelines/testing-dynamic-version/dist/other_test_code-0.0.1-py3-none-any.whl diff --git a/acceptance/bundle/resources/pipelines/testing-dynamic-version/dist/other_test_code-0.0.1-py3-none-any.whl b/acceptance/bundle/resources/pipelines/testing-dynamic-version/dist/other_test_code-0.0.1-py3-none-any.whl new file mode 100644 index 0000000000000000000000000000000000000000..c2cde02cc53b63e1ae67fa5b66fa34ca3a3de3a2 GIT binary patch literal 1893 zcmWIWW@Zs#U|`^2U~S2ZI$`~MsveMM0L1)2TwIi_A0MBYmst`YuUAmn-LLPht997mP0$hdk=)_t()qc|yO5W0%S_6*bXRwRCaqLPmxF6r&C6u76$xH1ZRQ z(YcA4d0?YM`z-km83?#m&wq5<#Pcwt($cm`Or4jyK1f7=QajH1>FrDvR)eSapLuE? zy(^?~?b3wF-J&NKY%0mVx~!vJ^^;SP-;N2vD`(u+X}ORhpeDY%PT+y{!@Rpa4tFk9 zSihfm+H8C7(@9~=J(IQ8tA5yA`Tk9#U);&xEWtqz83x`pr~X~vzvP!LU%lMsIX9!K z)r#1>j%rQ_l5qD}YG$|e31h{9E`Ee}G*U~WGUomASOfIRFCf+c;{1|~)S~#3)Z&u( zJwv^e%;FN=%)GRGeP7oQM;FHs$Jn!ud`$)dtq*JcFLH}qNDZI0!60z; z#TBo)7OxFrvv%g$F1{$~(YN#F`PU9?+mdrtP^ogw);Vgzi8&0q6J=eFI;`Q+aqLm? z-X!<-wm{CCEj)K;pK3g{oO#6&Cgv5#;(kjWzcaz_^24*1zaIH{Haeb(E|b`L=J+2^ zuBNY3r(( zNgcgTefE`s7Yr^Kn_V})@PzBsdHsuDb*}Mh=z5(!>9Z+FLsQH5DVL|u_8^TFCfClM zKJR_i_sr@ikC>3smpU*)-p)<}Fj(uWQ}?=Y7wd>F(do6c`hOW}DXDeGA!uwy*%PE)lkso=qGF&F+8OiWWXXNB%?DDTqJJ)0U_LTUKfAtT3ax0dGb309`o7kqsEdBTD5%c-W zwj}MfKlos-%>7MkFFfaSVE%mK>#NK=x7tmoebjMGHvJQx7kT$@P{7Oj^UEebed%E> z-f-77cGG2tzFX;GCOdgb&b!1fSrO#SB(P@23JvupzNRN{yi!90r*17;%dU7m_Fx>J zUr&z6;&%IG-!C8KzIV9Mh2!^8*_~3CoerN)J^!Bhdp?KohMk{|mZ@wAod2uywZ(6C zzvnk4GtXWV+Yw+s@6p$7#_Q&*ujg5Q)Jtcf+Vh0`trz<|cRzo>MYu-McD1#k#-nH( zBlFjb&!*p75g~F2iC1ukMpa zF+JP+x4B%D&F9!~%7o!V!Yc8WwLzjqmHA(cp0RV<`6e9a+NkTfx<0_0kx7IZcL52E zDKJ>l2%^x6OLWcXMIb~g1H+QWXt-v0frzXbTY-bntOZOqgi<@K_(3-nJyRo0l>sJ1 zLZ*VVII@x0G9tpr-9#9Po-NT$N6&Z&(>0k1I~XM!A{&h@3&M=v!dOLw(U1%o;LQpu Q78uxp&;aONc~%e)06LtWdjJ3c literal 0 HcmV?d00001 diff --git a/acceptance/bundle/resources/pipelines/testing-dynamic-version/script b/acceptance/bundle/resources/pipelines/testing-dynamic-version/script index e71c775893f..864b5b03220 100644 --- a/acceptance/bundle/resources/pipelines/testing-dynamic-version/script +++ b/acceptance/bundle/resources/pipelines/testing-dynamic-version/script @@ -1,6 +1,3 @@ -trap "rm -fr dist" EXIT -mkdir dist -cp $TESTDIR/../../../artifacts/whl_prebuilt_multiple/dist/lib/other_test_code-0.0.1-py3-none-any.whl dist title "Test dependencies use the patched wheel, with extras kept" trace $CLI bundle deploy diff --git a/acceptance/bundle/resources/pipelines/testing/output.txt b/acceptance/bundle/resources/pipelines/testing/output.txt index 3cccee04481..32d18e1679f 100644 --- a/acceptance/bundle/resources/pipelines/testing/output.txt +++ b/acceptance/bundle/resources/pipelines/testing/output.txt @@ -50,75 +50,6 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged } } -=== No changes after the deploy - ->>> [CLI] bundle plan -Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged - -=== A remote edit that drops testing shows in the plan - ->>> [CLI] bundle plan -o json -"update" - ->>> [CLI] bundle deploy -Uploading dist/test_helpers-0.1-py3-none-any.whl... -Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/pipeline-testing/default/files... -Updated pipelines.orders -Files: 1 uploaded, 0 deleted -Resources: 0 created, 1 changed, 0 deleted, 0 unchanged - ->>> [CLI] bundle plan -Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged - -=== Changing testing updates the pipeline - ->>> update_file.py databricks.yml ./tests/** ./tests/unit/** - ->>> [CLI] bundle plan -update pipelines.orders - -Plan: 0 to add, 1 to change, 0 to delete, 0 unchanged - ->>> [CLI] bundle deploy -Uploading dist/test_helpers-0.1-py3-none-any.whl... -Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/pipeline-testing/default/files... -Updated pipelines.orders -Files: 1 uploaded, 0 deleted -Resources: 0 created, 1 changed, 0 deleted, 0 unchanged - ->>> print_requests.py //api/2.0/pipelines --nostamp -{ - "method": "PUT", - "path": "/api/2.0/pipelines/[ORDERS_ID]", - "sources": [ - { - "glob": { - "include": "/Workspace/Users/[USERNAME]/.bundle/pipeline-testing/default/files/tests/unit/**" - } - } - ] -} - -=== Removing testing sends an update without it - ->>> [CLI] bundle plan -update pipelines.orders - -Plan: 0 to add, 1 to change, 0 to delete, 0 unchanged - ->>> [CLI] bundle deploy -Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/pipeline-testing/default/files... -Updated pipelines.orders -Files: 1 uploaded, 0 deleted -Resources: 0 created, 1 changed, 0 deleted, 0 unchanged - ->>> print_requests.py //api/2.0/pipelines --nostamp -{ - "method": "PUT", - "path": "/api/2.0/pipelines/[ORDERS_ID]", - "has_testing": false -} - >>> [CLI] bundle destroy --auto-approve The following resources will be deleted: delete resources.pipelines.orders diff --git a/acceptance/bundle/resources/pipelines/testing/script b/acceptance/bundle/resources/pipelines/testing/script index 2c12917d00f..ea297538e87 100644 --- a/acceptance/bundle/resources/pipelines/testing/script +++ b/acceptance/bundle/resources/pipelines/testing/script @@ -5,31 +5,5 @@ title "Deploy sends testing, with the local wheel uploaded" trace $CLI bundle deploy trace print_requests.py //api/2.0/pipelines --nostamp | jq '{method, path, testing: .body.testing}' -title "No changes after the deploy" -trace $CLI bundle plan -rm "$OUT_REQUESTS" - -title "A remote edit that drops testing shows in the plan" -PIPELINE_ID=$(read_id.py orders) -MSYS_NO_PATHCONV=1 $CLI api get "/api/2.0/pipelines/$PIPELINE_ID" | jq '.spec | del(.testing)' > edit.json -MSYS_NO_PATHCONV=1 $CLI api put "/api/2.0/pipelines/$PIPELINE_ID" --json @edit.json &> LOG.edit -rm edit.json -trace $CLI bundle plan -o json | jq '.plan["resources.pipelines.orders"].changes.testing.action' -trace $CLI bundle deploy -trace $CLI bundle plan -rm "$OUT_REQUESTS" - -title "Changing testing updates the pipeline" -trace update_file.py databricks.yml './tests/**' './tests/unit/**' -trace $CLI bundle plan -trace $CLI bundle deploy -trace print_requests.py //api/2.0/pipelines --nostamp | jq '{method, path, sources: .body.testing.sources}' - -title "Removing testing sends an update without it" -update_file.py databricks.yml "$(sed -n '/^ testing:/,$p' databricks.yml)" "" -trace $CLI bundle plan -trace $CLI bundle deploy -trace print_requests.py //api/2.0/pipelines --nostamp | jq '{method, path, has_testing: (.body | has("testing"))}' - trace $CLI bundle destroy --auto-approve rm "$OUT_REQUESTS"