From 9d435efafc6a1105887a42962fa58df00a36b119 Mon Sep 17 00:00:00 2001 From: Bhukya Date: Mon, 16 Nov 2026 20:50:45 +0000 Subject: [PATCH 01/10] Add the testing property of pipelines to bundles Add resources.pipelines..testing, with the test sources and test-only dependencies that databricks pipelines test runs with. The field comes from the Go SDK's CreatePipeline. The CLI adds its own descriptions for it. The property is experimental. It needs DATABRICKS_BUNDLE_ENABLE_EXPERIMENTAL_PIPELINE_TESTING=true. While the variable is off, bundle validate rejects testing and names the variable. Bundles without testing are not changed. Later PRs translate its paths. Co-authored-by: Isaac --- .../pipelines/testing/databricks.yml | 19 +++++++ .../resources/pipelines/testing/out.test.toml | 2 + .../resources/pipelines/testing/output.txt | 33 +++++++++++++ .../pipelines/testing/pyproject.toml | 6 +++ .../bundle/resources/pipelines/testing/script | 4 ++ .../resources/pipelines/testing/test.toml | 5 ++ .../pipelines/testing/tests/requirements.txt | 1 + .../pipelines/testing/tests/test_orders.py | 2 + .../testing/transformations/orders.py | 1 + .../validate/pipeline_testing_enabled.go | 49 +++++++++++++++++++ .../validate/pipeline_testing_enabled_test.go | 42 ++++++++++++++++ bundle/env/experimental_pipeline_testing.go | 17 +++++++ bundle/internal/schema/annotations.yml | 14 ++++++ bundle/phases/initialize.go | 4 ++ bundle/schema/jsonschema.json | 6 +-- .../bundles/pipelines/_models/pipeline.py | 4 +- .../_models/pipeline_test_environment.py | 10 +--- .../pipelines/_models/pipeline_test_source.py | 4 +- 18 files changed, 208 insertions(+), 15 deletions(-) create mode 100644 acceptance/bundle/resources/pipelines/testing/databricks.yml create mode 100644 acceptance/bundle/resources/pipelines/testing/out.test.toml create mode 100644 acceptance/bundle/resources/pipelines/testing/output.txt create mode 100644 acceptance/bundle/resources/pipelines/testing/pyproject.toml create mode 100644 acceptance/bundle/resources/pipelines/testing/script create mode 100644 acceptance/bundle/resources/pipelines/testing/test.toml create mode 100644 acceptance/bundle/resources/pipelines/testing/tests/requirements.txt create mode 100644 acceptance/bundle/resources/pipelines/testing/tests/test_orders.py create mode 100644 acceptance/bundle/resources/pipelines/testing/transformations/orders.py create mode 100644 bundle/config/validate/pipeline_testing_enabled.go create mode 100644 bundle/config/validate/pipeline_testing_enabled_test.go create mode 100644 bundle/env/experimental_pipeline_testing.go diff --git a/acceptance/bundle/resources/pipelines/testing/databricks.yml b/acceptance/bundle/resources/pipelines/testing/databricks.yml new file mode 100644 index 00000000000..346cc9d7963 --- /dev/null +++ b/acceptance/bundle/resources/pipelines/testing/databricks.yml @@ -0,0 +1,19 @@ +bundle: + name: pipeline-testing + +resources: + pipelines: + orders: + name: orders + libraries: + - glob: + include: ./transformations/** + testing: + sources: + - glob: + include: ./tests/** + environment: + dependencies: + - -e .[test] + - -r ./tests/requirements.txt + - ./dist/*.whl diff --git a/acceptance/bundle/resources/pipelines/testing/out.test.toml b/acceptance/bundle/resources/pipelines/testing/out.test.toml new file mode 100644 index 00000000000..a927a5fbc06 --- /dev/null +++ b/acceptance/bundle/resources/pipelines/testing/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/resources/pipelines/testing/output.txt b/acceptance/bundle/resources/pipelines/testing/output.txt new file mode 100644 index 00000000000..6133e93eb75 --- /dev/null +++ b/acceptance/bundle/resources/pipelines/testing/output.txt @@ -0,0 +1,33 @@ + +>>> [CLI] bundle validate -o json +{ + "environment": { + "dependencies": [ + "-e .[test]", + "-r ./tests/requirements.txt", + "./dist/*.whl" + ] + }, + "sources": [ + { + "glob": { + "include": "./tests/**" + } + } + ] +} + +=== Without pipeline testing, the property is an error + +>>> musterr env -u DATABRICKS_BUNDLE_ENABLE_EXPERIMENTAL_PIPELINE_TESTING [CLI] bundle validate +Error: the testing property of pipelines is experimental. Set DATABRICKS_BUNDLE_ENABLE_EXPERIMENTAL_PIPELINE_TESTING=true to enable it + at resources.pipelines.orders.testing + in databricks.yml:12:9 + +Name: pipeline-testing +Target: default +Workspace: + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/pipeline-testing/default + +Found 1 error diff --git a/acceptance/bundle/resources/pipelines/testing/pyproject.toml b/acceptance/bundle/resources/pipelines/testing/pyproject.toml new file mode 100644 index 00000000000..1c198acc7cb --- /dev/null +++ b/acceptance/bundle/resources/pipelines/testing/pyproject.toml @@ -0,0 +1,6 @@ +[project] +name = "orders" +version = "0.1" + +[project.optional-dependencies] +test = ["faker"] diff --git a/acceptance/bundle/resources/pipelines/testing/script b/acceptance/bundle/resources/pipelines/testing/script new file mode 100644 index 00000000000..1dc320485b7 --- /dev/null +++ b/acceptance/bundle/resources/pipelines/testing/script @@ -0,0 +1,4 @@ +trace $CLI bundle validate -o json | jq .resources.pipelines.orders.testing + +title "Without pipeline testing, the property is an error" +trace musterr env -u DATABRICKS_BUNDLE_ENABLE_EXPERIMENTAL_PIPELINE_TESTING $CLI bundle validate diff --git a/acceptance/bundle/resources/pipelines/testing/test.toml b/acceptance/bundle/resources/pipelines/testing/test.toml new file mode 100644 index 00000000000..965c596a302 --- /dev/null +++ b/acceptance/bundle/resources/pipelines/testing/test.toml @@ -0,0 +1,5 @@ +# Runs only against the local test server. +Cloud = false +RecordRequests = true +Ignore = [".databricks"] +Env.DATABRICKS_BUNDLE_ENABLE_EXPERIMENTAL_PIPELINE_TESTING = "true" diff --git a/acceptance/bundle/resources/pipelines/testing/tests/requirements.txt b/acceptance/bundle/resources/pipelines/testing/tests/requirements.txt new file mode 100644 index 00000000000..9d7f6e827a1 --- /dev/null +++ b/acceptance/bundle/resources/pipelines/testing/tests/requirements.txt @@ -0,0 +1 @@ +faker diff --git a/acceptance/bundle/resources/pipelines/testing/tests/test_orders.py b/acceptance/bundle/resources/pipelines/testing/tests/test_orders.py new file mode 100644 index 00000000000..eb4379fd048 --- /dev/null +++ b/acceptance/bundle/resources/pipelines/testing/tests/test_orders.py @@ -0,0 +1,2 @@ +def test_orders(): + pass diff --git a/acceptance/bundle/resources/pipelines/testing/transformations/orders.py b/acceptance/bundle/resources/pipelines/testing/transformations/orders.py new file mode 100644 index 00000000000..3a22621c3bf --- /dev/null +++ b/acceptance/bundle/resources/pipelines/testing/transformations/orders.py @@ -0,0 +1 @@ +from pyspark import pipelines as dp diff --git a/bundle/config/validate/pipeline_testing_enabled.go b/bundle/config/validate/pipeline_testing_enabled.go new file mode 100644 index 00000000000..85fdb85ed4b --- /dev/null +++ b/bundle/config/validate/pipeline_testing_enabled.go @@ -0,0 +1,49 @@ +package validate + +import ( + "context" + "fmt" + "maps" + "slices" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/env" + "github.com/databricks/cli/libs/diag" + "github.com/databricks/cli/libs/dyn" +) + +type pipelineTestingEnabled struct{ bundle.RO } + +// PipelineTestingEnabled rejects the testing property of pipelines while the experimental feature is off. +func PipelineTestingEnabled() bundle.ReadOnlyMutator { + return &pipelineTestingEnabled{} +} + +func (v *pipelineTestingEnabled) Name() string { + return "validate:pipeline_testing_enabled" +} + +func (v *pipelineTestingEnabled) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { + if env.ExperimentalPipelineTesting(ctx) { + return nil + } + var diags diag.Diagnostics + pipelines := b.Config.Resources.Pipelines + for _, key := range slices.Sorted(maps.Keys(pipelines)) { + if pipelines[key] == nil || pipelines[key].Testing == nil { + continue + } + path := dyn.NewPath(dyn.Key("resources"), dyn.Key("pipelines"), dyn.Key(key), dyn.Key("testing")) + value, err := dyn.GetByPath(b.Config.Value(), path) + if err != nil { + return diag.FromErr(err) + } + diags = append(diags, diag.Diagnostic{ + Severity: diag.Error, + Summary: fmt.Sprintf("the testing property of pipelines is experimental. Set %s=true to enable it", env.ExperimentalPipelineTestingVariable), + Locations: value.Locations(), + Paths: dyn.ToStructPaths(path), + }) + } + return diags +} diff --git a/bundle/config/validate/pipeline_testing_enabled_test.go b/bundle/config/validate/pipeline_testing_enabled_test.go new file mode 100644 index 00000000000..126013c10fe --- /dev/null +++ b/bundle/config/validate/pipeline_testing_enabled_test.go @@ -0,0 +1,42 @@ +package validate + +import ( + "testing" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config" + "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/cli/bundle/env" + "github.com/databricks/cli/bundle/internal/bundletest" + "github.com/databricks/cli/libs/diag" + libsenv "github.com/databricks/cli/libs/env" + "github.com/databricks/databricks-sdk-go/service/pipelines" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestPipelineTestingEnabled(t *testing.T) { + b := &bundle.Bundle{ + Config: config.Root{ + Resources: config.Resources{ + Pipelines: map[string]*resources.Pipeline{ + "orders": {CreatePipeline: pipelines.CreatePipeline{Testing: &pipelines.PipelineTesting{}}}, + "reports": {}, + }, + }, + }, + } + loc := diag.Location{File: "databricks.yml", Line: 9, Column: 9} + bundletest.SetLocation(b, "resources.pipelines.orders.testing", []diag.Location{loc}) + + diags := PipelineTestingEnabled().Apply(t.Context(), b) + require.Len(t, diags, 1) + assert.Equal(t, diag.Error, diags[0].Severity) + assert.Contains(t, diags[0].Summary, env.ExperimentalPipelineTestingVariable) + assert.Equal(t, []diag.Location{loc}, diags[0].Locations) + require.Len(t, diags[0].Paths, 1) + assert.Equal(t, "resources.pipelines.orders.testing", diags[0].Paths[0].String()) + + ctx := libsenv.Set(t.Context(), env.ExperimentalPipelineTestingVariable, "true") + assert.Empty(t, PipelineTestingEnabled().Apply(ctx, b)) +} diff --git a/bundle/env/experimental_pipeline_testing.go b/bundle/env/experimental_pipeline_testing.go new file mode 100644 index 00000000000..b00a94176e9 --- /dev/null +++ b/bundle/env/experimental_pipeline_testing.go @@ -0,0 +1,17 @@ +package env + +import ( + "context" + + envlib "github.com/databricks/cli/libs/env" +) + +// ExperimentalPipelineTestingVariable enables the experimental testing property of pipelines. +const ExperimentalPipelineTestingVariable = "DATABRICKS_BUNDLE_ENABLE_EXPERIMENTAL_PIPELINE_TESTING" + +// ExperimentalPipelineTesting reports whether the experimental testing property of pipelines is +// enabled. While it is off, bundles deploy pipelines exactly as they did before the property existed. +func ExperimentalPipelineTesting(ctx context.Context) bool { + enabled, _ := envlib.GetBool(ctx, ExperimentalPipelineTestingVariable) + return enabled +} diff --git a/bundle/internal/schema/annotations.yml b/bundle/internal/schema/annotations.yml index 58abf721658..1a0018f2598 100644 --- a/bundle/internal/schema/annotations.yml +++ b/bundle/internal/schema/annotations.yml @@ -1465,6 +1465,20 @@ resources: "user_name": "description": |- The name of the user granted the permission level. + "testing": + "description": |- + The test configuration of the pipeline. Only test-only updates, such as those started by `databricks pipelines test`, use it. It is experimental and needs `DATABRICKS_BUNDLE_ENABLE_EXPERIMENTAL_PIPELINE_TESTING=true`. + "$fields": + "environment": + "$fields": + "dependencies": + "description": |- + A list of files that declare the packages the tests need: a project folder, a requirements file, or a wheel, in the workspace or a UC volume. A project folder can take `-e` and an extra, for example `-e ./project[test]`. A relative path, such as `-e ./project[test]` or `-r ./tests/requirements.txt`, is relative to the YAML file that defines it. Package names and URLs are rejected. + "sources": + "$fields": + "glob": + "description": |- + A `.py` test file, or a folder followed by `/**`. Other wildcards are rejected, and the sources must not overlap the pipeline's `libraries`. A relative path is relative to the YAML file that defines it. "trigger": "deprecation_message": |- Use continuous instead diff --git a/bundle/phases/initialize.go b/bundle/phases/initialize.go index 8c5a68d3130..b4a08596bb1 100644 --- a/bundle/phases/initialize.go +++ b/bundle/phases/initialize.go @@ -166,6 +166,10 @@ func Initialize(ctx context.Context, b *bundle.Bundle) { // Must run after PythonMutator to see development values set by Python mutators. validate.PipelineDevelopmentDeprecated(), + // Reject the experimental testing property of pipelines while it is off. Must run after + // PythonMutator, which can set testing, and before TranslatePaths and the deploy use it. + validate.PipelineTestingEnabled(), + // Compute resources.volumes.*.volume_path and resolve references to it. Must run after // PythonMutator: volume_path is computed and read-only, not declared by the PyDABs Volume // model, so exposing it to Python would fail resource loading (like "deployment" below). diff --git a/bundle/schema/jsonschema.json b/bundle/schema/jsonschema.json index 2d5a35b5a5e..d3119d19d47 100644 --- a/bundle/schema/jsonschema.json +++ b/bundle/schema/jsonschema.json @@ -2454,7 +2454,7 @@ "deprecated": true }, "testing": { - "description": "[Private Preview] Test configuration for this pipeline. Used only by test-only updates.", + "description": "[Private Preview] The test configuration of the pipeline. Only test-only updates, such as those started by `databricks pipelines test`, use it. It is experimental and needs `DATABRICKS_BUNDLE_ENABLE_EXPERIMENTAL_PIPELINE_TESTING=true`.", "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/pipelines.PipelineTesting", "x-databricks-launch-stage": "PRIVATE_PREVIEW", "doNotSuggest": true @@ -18261,7 +18261,7 @@ "description": "The packages that a pipeline's tests need in addition to the pipeline's environment.", "properties": { "dependencies": { - "description": "[Private Preview] A list of files that declare the packages the tests need: a project folder, a requirements\nfile, or a wheel, in the workspace or a UC volume. A project folder can take `--editable` and\nan extra, for example `--editable /Workspace/Users/someone@example.com/orders[test]`. Package\nnames, such as `faker==33.0`, and URLs are rejected.", + "description": "[Private Preview] A list of files that declare the packages the tests need: a project folder, a requirements file, or a wheel, in the workspace or a UC volume. A project folder can take `-e` and an extra, for example `-e ./project[test]`. A relative path, such as `-e ./project[test]` or `-r ./tests/requirements.txt`, is relative to the YAML file that defines it. Package names and URLs are rejected.", "$ref": "#/$defs/slice/string", "x-databricks-launch-stage": "PRIVATE_PREVIEW", "doNotSuggest": true @@ -18282,7 +18282,7 @@ "description": "A workspace location that holds pipeline tests.", "properties": { "glob": { - "description": "[Private Preview] An absolute workspace path to a test file, or to a folder followed by `/**`.", + "description": "[Private Preview] A `.py` test file, or a folder followed by `/**`. Other wildcards are rejected, and the sources must not overlap the pipeline's `libraries`. A relative path is relative to the YAML file that defines it.", "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/pipelines.PathPattern", "x-databricks-launch-stage": "PRIVATE_PREVIEW", "doNotSuggest": true diff --git a/python/databricks/bundles/pipelines/_models/pipeline.py b/python/databricks/bundles/pipelines/_models/pipeline.py index c8efe551d4f..6a4551c3407 100644 --- a/python/databricks/bundles/pipelines/_models/pipeline.py +++ b/python/databricks/bundles/pipelines/_models/pipeline.py @@ -266,7 +266,7 @@ class Pipeline(Resource): """ :meta private: [EXPERIMENTAL] - [Private Preview] Test configuration for this pipeline. Used only by test-only updates. + [Private Preview] The test configuration of the pipeline. Only test-only updates, such as those started by `databricks pipelines test`, use it. It is experimental and needs `DATABRICKS_BUNDLE_ENABLE_EXPERIMENTAL_PIPELINE_TESTING=true`. """ usage_policy_id: VariableOrOptional[str] = None @@ -490,7 +490,7 @@ class PipelineDict(TypedDict, total=False): """ :meta private: [EXPERIMENTAL] - [Private Preview] Test configuration for this pipeline. Used only by test-only updates. + [Private Preview] The test configuration of the pipeline. Only test-only updates, such as those started by `databricks pipelines test`, use it. It is experimental and needs `DATABRICKS_BUNDLE_ENABLE_EXPERIMENTAL_PIPELINE_TESTING=true`. """ usage_policy_id: VariableOrOptional[str] diff --git a/python/databricks/bundles/pipelines/_models/pipeline_test_environment.py b/python/databricks/bundles/pipelines/_models/pipeline_test_environment.py index 827e30e91a4..897a1e94785 100644 --- a/python/databricks/bundles/pipelines/_models/pipeline_test_environment.py +++ b/python/databricks/bundles/pipelines/_models/pipeline_test_environment.py @@ -23,10 +23,7 @@ class PipelineTestEnvironment: """ :meta private: [EXPERIMENTAL] - [Private Preview] A list of files that declare the packages the tests need: a project folder, a requirements - file, or a wheel, in the workspace or a UC volume. A project folder can take `--editable` and - an extra, for example `--editable /Workspace/Users/someone@example.com/orders[test]`. Package - names, such as `faker==33.0`, and URLs are rejected. + [Private Preview] A list of files that declare the packages the tests need: a project folder, a requirements file, or a wheel, in the workspace or a UC volume. A project folder can take `-e` and an extra, for example `-e ./project[test]`. A relative path, such as `-e ./project[test]` or `-r ./tests/requirements.txt`, is relative to the YAML file that defines it. Package names and URLs are rejected. """ @classmethod @@ -44,10 +41,7 @@ class PipelineTestEnvironmentDict(TypedDict, total=False): """ :meta private: [EXPERIMENTAL] - [Private Preview] A list of files that declare the packages the tests need: a project folder, a requirements - file, or a wheel, in the workspace or a UC volume. A project folder can take `--editable` and - an extra, for example `--editable /Workspace/Users/someone@example.com/orders[test]`. Package - names, such as `faker==33.0`, and URLs are rejected. + [Private Preview] A list of files that declare the packages the tests need: a project folder, a requirements file, or a wheel, in the workspace or a UC volume. A project folder can take `-e` and an extra, for example `-e ./project[test]`. A relative path, such as `-e ./project[test]` or `-r ./tests/requirements.txt`, is relative to the YAML file that defines it. Package names and URLs are rejected. """ diff --git a/python/databricks/bundles/pipelines/_models/pipeline_test_source.py b/python/databricks/bundles/pipelines/_models/pipeline_test_source.py index 8abc02e9b7d..fcf4dcfc671 100644 --- a/python/databricks/bundles/pipelines/_models/pipeline_test_source.py +++ b/python/databricks/bundles/pipelines/_models/pipeline_test_source.py @@ -27,7 +27,7 @@ class PipelineTestSource: """ :meta private: [EXPERIMENTAL] - [Private Preview] An absolute workspace path to a test file, or to a folder followed by `/**`. + [Private Preview] A `.py` test file, or a folder followed by `/**`. Other wildcards are rejected, and the sources must not overlap the pipeline's `libraries`. A relative path is relative to the YAML file that defines it. """ @classmethod @@ -45,7 +45,7 @@ class PipelineTestSourceDict(TypedDict, total=False): """ :meta private: [EXPERIMENTAL] - [Private Preview] An absolute workspace path to a test file, or to a folder followed by `/**`. + [Private Preview] A `.py` test file, or a folder followed by `/**`. Other wildcards are rejected, and the sources must not overlap the pipeline's `libraries`. A relative path is relative to the YAML file that defines it. """ From 13872f8745d074d49a0e3901d7e34026b322f5d4 Mon Sep 17 00:00:00 2001 From: Bhukya Date: Mon, 16 Nov 2026 23:53:48 +0000 Subject: [PATCH 02/10] Find the testing property of pipelines with a dyn pattern The check walks resources.pipelines.*.testing with dyn.MapByPattern, like the check of the development property, which gives the path and the locations directly. The acceptance test covers the error and the enabled case, so the unit test is removed. Co-authored-by: Isaac --- .../validate/pipeline_testing_enabled.go | 24 +++++------ .../validate/pipeline_testing_enabled_test.go | 42 ------------------- 2 files changed, 11 insertions(+), 55 deletions(-) delete mode 100644 bundle/config/validate/pipeline_testing_enabled_test.go diff --git a/bundle/config/validate/pipeline_testing_enabled.go b/bundle/config/validate/pipeline_testing_enabled.go index 85fdb85ed4b..8b907a7090a 100644 --- a/bundle/config/validate/pipeline_testing_enabled.go +++ b/bundle/config/validate/pipeline_testing_enabled.go @@ -3,8 +3,6 @@ package validate import ( "context" "fmt" - "maps" - "slices" "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/env" @@ -28,22 +26,22 @@ func (v *pipelineTestingEnabled) Apply(ctx context.Context, b *bundle.Bundle) di return nil } var diags diag.Diagnostics - pipelines := b.Config.Resources.Pipelines - for _, key := range slices.Sorted(maps.Keys(pipelines)) { - if pipelines[key] == nil || pipelines[key].Testing == nil { - continue - } - path := dyn.NewPath(dyn.Key("resources"), dyn.Key("pipelines"), dyn.Key(key), dyn.Key("testing")) - value, err := dyn.GetByPath(b.Config.Value(), path) - if err != nil { - return diag.FromErr(err) + pattern := dyn.NewPattern(dyn.Key("resources"), dyn.Key("pipelines"), dyn.AnyKey(), dyn.Key("testing")) + _, err := dyn.MapByPattern(b.Config.Value(), pattern, func(p dyn.Path, v dyn.Value) (dyn.Value, error) { + // "testing:" without a value is not set, as in the typed configuration. + if v.Kind() == dyn.KindNil { + return v, nil } diags = append(diags, diag.Diagnostic{ Severity: diag.Error, Summary: fmt.Sprintf("the testing property of pipelines is experimental. Set %s=true to enable it", env.ExperimentalPipelineTestingVariable), - Locations: value.Locations(), - Paths: dyn.ToStructPaths(path), + Locations: v.Locations(), + Paths: dyn.ToStructPaths(p), }) + return v, nil + }) + if err != nil { + return diag.FromErr(err) } return diags } diff --git a/bundle/config/validate/pipeline_testing_enabled_test.go b/bundle/config/validate/pipeline_testing_enabled_test.go deleted file mode 100644 index 126013c10fe..00000000000 --- a/bundle/config/validate/pipeline_testing_enabled_test.go +++ /dev/null @@ -1,42 +0,0 @@ -package validate - -import ( - "testing" - - "github.com/databricks/cli/bundle" - "github.com/databricks/cli/bundle/config" - "github.com/databricks/cli/bundle/config/resources" - "github.com/databricks/cli/bundle/env" - "github.com/databricks/cli/bundle/internal/bundletest" - "github.com/databricks/cli/libs/diag" - libsenv "github.com/databricks/cli/libs/env" - "github.com/databricks/databricks-sdk-go/service/pipelines" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestPipelineTestingEnabled(t *testing.T) { - b := &bundle.Bundle{ - Config: config.Root{ - Resources: config.Resources{ - Pipelines: map[string]*resources.Pipeline{ - "orders": {CreatePipeline: pipelines.CreatePipeline{Testing: &pipelines.PipelineTesting{}}}, - "reports": {}, - }, - }, - }, - } - loc := diag.Location{File: "databricks.yml", Line: 9, Column: 9} - bundletest.SetLocation(b, "resources.pipelines.orders.testing", []diag.Location{loc}) - - diags := PipelineTestingEnabled().Apply(t.Context(), b) - require.Len(t, diags, 1) - assert.Equal(t, diag.Error, diags[0].Severity) - assert.Contains(t, diags[0].Summary, env.ExperimentalPipelineTestingVariable) - assert.Equal(t, []diag.Location{loc}, diags[0].Locations) - require.Len(t, diags[0].Paths, 1) - assert.Equal(t, "resources.pipelines.orders.testing", diags[0].Paths[0].String()) - - ctx := libsenv.Set(t.Context(), env.ExperimentalPipelineTestingVariable, "true") - assert.Empty(t, PipelineTestingEnabled().Apply(ctx, b)) -} From c14224ed60f3b405c3a0fae46d299a7919318fbe Mon Sep 17 00:00:00 2001 From: Bhukya Date: Tue, 17 Nov 2026 15:50:16 +0000 Subject: [PATCH 03/10] Accept the testing property of pipelines without an environment variable The Go SDK now has the testing field, so the property needs no separate opt-in. Co-authored-by: Isaac --- .../resources/pipelines/testing/output.txt | 15 ------ .../bundle/resources/pipelines/testing/script | 3 -- .../resources/pipelines/testing/test.toml | 1 - .../validate/pipeline_testing_enabled.go | 47 ------------------- bundle/env/experimental_pipeline_testing.go | 17 ------- bundle/internal/schema/annotations.yml | 2 +- bundle/phases/initialize.go | 4 -- bundle/schema/jsonschema.json | 2 +- .../bundles/pipelines/_models/pipeline.py | 4 +- 9 files changed, 4 insertions(+), 91 deletions(-) delete mode 100644 bundle/config/validate/pipeline_testing_enabled.go delete mode 100644 bundle/env/experimental_pipeline_testing.go diff --git a/acceptance/bundle/resources/pipelines/testing/output.txt b/acceptance/bundle/resources/pipelines/testing/output.txt index 6133e93eb75..edb0eb1c49a 100644 --- a/acceptance/bundle/resources/pipelines/testing/output.txt +++ b/acceptance/bundle/resources/pipelines/testing/output.txt @@ -16,18 +16,3 @@ } ] } - -=== Without pipeline testing, the property is an error - ->>> musterr env -u DATABRICKS_BUNDLE_ENABLE_EXPERIMENTAL_PIPELINE_TESTING [CLI] bundle validate -Error: the testing property of pipelines is experimental. Set DATABRICKS_BUNDLE_ENABLE_EXPERIMENTAL_PIPELINE_TESTING=true to enable it - at resources.pipelines.orders.testing - in databricks.yml:12:9 - -Name: pipeline-testing -Target: default -Workspace: - User: [USERNAME] - Path: /Workspace/Users/[USERNAME]/.bundle/pipeline-testing/default - -Found 1 error diff --git a/acceptance/bundle/resources/pipelines/testing/script b/acceptance/bundle/resources/pipelines/testing/script index 1dc320485b7..b7f915f687c 100644 --- a/acceptance/bundle/resources/pipelines/testing/script +++ b/acceptance/bundle/resources/pipelines/testing/script @@ -1,4 +1 @@ trace $CLI bundle validate -o json | jq .resources.pipelines.orders.testing - -title "Without pipeline testing, the property is an error" -trace musterr env -u DATABRICKS_BUNDLE_ENABLE_EXPERIMENTAL_PIPELINE_TESTING $CLI bundle validate diff --git a/acceptance/bundle/resources/pipelines/testing/test.toml b/acceptance/bundle/resources/pipelines/testing/test.toml index 965c596a302..25d8c36639f 100644 --- a/acceptance/bundle/resources/pipelines/testing/test.toml +++ b/acceptance/bundle/resources/pipelines/testing/test.toml @@ -2,4 +2,3 @@ Cloud = false RecordRequests = true Ignore = [".databricks"] -Env.DATABRICKS_BUNDLE_ENABLE_EXPERIMENTAL_PIPELINE_TESTING = "true" diff --git a/bundle/config/validate/pipeline_testing_enabled.go b/bundle/config/validate/pipeline_testing_enabled.go deleted file mode 100644 index 8b907a7090a..00000000000 --- a/bundle/config/validate/pipeline_testing_enabled.go +++ /dev/null @@ -1,47 +0,0 @@ -package validate - -import ( - "context" - "fmt" - - "github.com/databricks/cli/bundle" - "github.com/databricks/cli/bundle/env" - "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" -) - -type pipelineTestingEnabled struct{ bundle.RO } - -// PipelineTestingEnabled rejects the testing property of pipelines while the experimental feature is off. -func PipelineTestingEnabled() bundle.ReadOnlyMutator { - return &pipelineTestingEnabled{} -} - -func (v *pipelineTestingEnabled) Name() string { - return "validate:pipeline_testing_enabled" -} - -func (v *pipelineTestingEnabled) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { - if env.ExperimentalPipelineTesting(ctx) { - return nil - } - var diags diag.Diagnostics - pattern := dyn.NewPattern(dyn.Key("resources"), dyn.Key("pipelines"), dyn.AnyKey(), dyn.Key("testing")) - _, err := dyn.MapByPattern(b.Config.Value(), pattern, func(p dyn.Path, v dyn.Value) (dyn.Value, error) { - // "testing:" without a value is not set, as in the typed configuration. - if v.Kind() == dyn.KindNil { - return v, nil - } - diags = append(diags, diag.Diagnostic{ - Severity: diag.Error, - Summary: fmt.Sprintf("the testing property of pipelines is experimental. Set %s=true to enable it", env.ExperimentalPipelineTestingVariable), - Locations: v.Locations(), - Paths: dyn.ToStructPaths(p), - }) - return v, nil - }) - if err != nil { - return diag.FromErr(err) - } - return diags -} diff --git a/bundle/env/experimental_pipeline_testing.go b/bundle/env/experimental_pipeline_testing.go deleted file mode 100644 index b00a94176e9..00000000000 --- a/bundle/env/experimental_pipeline_testing.go +++ /dev/null @@ -1,17 +0,0 @@ -package env - -import ( - "context" - - envlib "github.com/databricks/cli/libs/env" -) - -// ExperimentalPipelineTestingVariable enables the experimental testing property of pipelines. -const ExperimentalPipelineTestingVariable = "DATABRICKS_BUNDLE_ENABLE_EXPERIMENTAL_PIPELINE_TESTING" - -// ExperimentalPipelineTesting reports whether the experimental testing property of pipelines is -// enabled. While it is off, bundles deploy pipelines exactly as they did before the property existed. -func ExperimentalPipelineTesting(ctx context.Context) bool { - enabled, _ := envlib.GetBool(ctx, ExperimentalPipelineTestingVariable) - return enabled -} diff --git a/bundle/internal/schema/annotations.yml b/bundle/internal/schema/annotations.yml index 1a0018f2598..41a3b0084d0 100644 --- a/bundle/internal/schema/annotations.yml +++ b/bundle/internal/schema/annotations.yml @@ -1467,7 +1467,7 @@ resources: The name of the user granted the permission level. "testing": "description": |- - The test configuration of the pipeline. Only test-only updates, such as those started by `databricks pipelines test`, use it. It is experimental and needs `DATABRICKS_BUNDLE_ENABLE_EXPERIMENTAL_PIPELINE_TESTING=true`. + The test configuration of the pipeline. Only test-only updates, such as those started by `databricks pipelines test`, use it. "$fields": "environment": "$fields": diff --git a/bundle/phases/initialize.go b/bundle/phases/initialize.go index b4a08596bb1..8c5a68d3130 100644 --- a/bundle/phases/initialize.go +++ b/bundle/phases/initialize.go @@ -166,10 +166,6 @@ func Initialize(ctx context.Context, b *bundle.Bundle) { // Must run after PythonMutator to see development values set by Python mutators. validate.PipelineDevelopmentDeprecated(), - // Reject the experimental testing property of pipelines while it is off. Must run after - // PythonMutator, which can set testing, and before TranslatePaths and the deploy use it. - validate.PipelineTestingEnabled(), - // Compute resources.volumes.*.volume_path and resolve references to it. Must run after // PythonMutator: volume_path is computed and read-only, not declared by the PyDABs Volume // model, so exposing it to Python would fail resource loading (like "deployment" below). diff --git a/bundle/schema/jsonschema.json b/bundle/schema/jsonschema.json index d3119d19d47..b45eb01c7a4 100644 --- a/bundle/schema/jsonschema.json +++ b/bundle/schema/jsonschema.json @@ -2454,7 +2454,7 @@ "deprecated": true }, "testing": { - "description": "[Private Preview] The test configuration of the pipeline. Only test-only updates, such as those started by `databricks pipelines test`, use it. It is experimental and needs `DATABRICKS_BUNDLE_ENABLE_EXPERIMENTAL_PIPELINE_TESTING=true`.", + "description": "[Private Preview] The test configuration of the pipeline. Only test-only updates, such as those started by `databricks pipelines test`, use it.", "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/pipelines.PipelineTesting", "x-databricks-launch-stage": "PRIVATE_PREVIEW", "doNotSuggest": true diff --git a/python/databricks/bundles/pipelines/_models/pipeline.py b/python/databricks/bundles/pipelines/_models/pipeline.py index 6a4551c3407..c33ade8cd12 100644 --- a/python/databricks/bundles/pipelines/_models/pipeline.py +++ b/python/databricks/bundles/pipelines/_models/pipeline.py @@ -266,7 +266,7 @@ class Pipeline(Resource): """ :meta private: [EXPERIMENTAL] - [Private Preview] The test configuration of the pipeline. Only test-only updates, such as those started by `databricks pipelines test`, use it. It is experimental and needs `DATABRICKS_BUNDLE_ENABLE_EXPERIMENTAL_PIPELINE_TESTING=true`. + [Private Preview] The test configuration of the pipeline. Only test-only updates, such as those started by `databricks pipelines test`, use it. """ usage_policy_id: VariableOrOptional[str] = None @@ -490,7 +490,7 @@ class PipelineDict(TypedDict, total=False): """ :meta private: [EXPERIMENTAL] - [Private Preview] The test configuration of the pipeline. Only test-only updates, such as those started by `databricks pipelines test`, use it. It is experimental and needs `DATABRICKS_BUNDLE_ENABLE_EXPERIMENTAL_PIPELINE_TESTING=true`. + [Private Preview] The test configuration of the pipeline. Only test-only updates, such as those started by `databricks pipelines test`, use it. """ usage_policy_id: VariableOrOptional[str] From 9c9bff6f803b15492ea28b5f4baf2a33db42035e Mon Sep 17 00:00:00 2001 From: Bhukya Date: Wed, 11 Nov 2026 21:57:50 +0000 Subject: [PATCH 04/10] Translate the testing paths of pipelines and upload local test wheels Relative paths in testing.sources and testing.environment.dependencies resolve to their deployed workspace paths, like libraries and environment dependencies. Local wheels in test dependencies are uploaded like other libraries, and the same-name check covers them. The testing property is only accepted with DATABRICKS_BUNDLE_ENABLE_EXPERIMENTAL_PIPELINE_TESTING=true, so bundles without it are not changed. Co-authored-by: Isaac --- .../databricks.yml | 21 +++++++++++++++++++ .../testing-same-name-libraries/out.test.toml | 2 ++ .../testing-same-name-libraries/output.txt | 10 +++++++++ .../test_helpers-0.1-py3-none-any.whl | 0 .../testing-same-name-libraries/script | 1 + .../testing-same-name-libraries/test.toml | 7 +++++++ .../tests/test_helpers-0.1-py3-none-any.whl | 0 .../tests/test_orders.py | 2 ++ .../transformations/orders.py | 1 + .../dist/test_helpers-0.1-py3-none-any.whl | 0 .../resources/pipelines/testing/output.txt | 8 ++++--- .../bundle/resources/pipelines/testing/script | 1 + .../mutator/paths/pipeline_paths_visitor.go | 19 +++++++++++++++++ .../paths/pipeline_paths_visitor_test.go | 7 +++++++ bundle/libraries/expand_glob_references.go | 13 ++++++++++++ bundle/libraries/remote_path.go | 1 + bundle/libraries/same_name_libraries.go | 1 + 17 files changed, 91 insertions(+), 3 deletions(-) create mode 100644 acceptance/bundle/resources/pipelines/testing-same-name-libraries/databricks.yml create mode 100644 acceptance/bundle/resources/pipelines/testing-same-name-libraries/out.test.toml create mode 100644 acceptance/bundle/resources/pipelines/testing-same-name-libraries/output.txt create mode 100644 acceptance/bundle/resources/pipelines/testing-same-name-libraries/pipeline/test_helpers-0.1-py3-none-any.whl create mode 100644 acceptance/bundle/resources/pipelines/testing-same-name-libraries/script create mode 100644 acceptance/bundle/resources/pipelines/testing-same-name-libraries/test.toml create mode 100644 acceptance/bundle/resources/pipelines/testing-same-name-libraries/tests/test_helpers-0.1-py3-none-any.whl create mode 100644 acceptance/bundle/resources/pipelines/testing-same-name-libraries/tests/test_orders.py create mode 100644 acceptance/bundle/resources/pipelines/testing-same-name-libraries/transformations/orders.py create mode 100644 acceptance/bundle/resources/pipelines/testing/dist/test_helpers-0.1-py3-none-any.whl diff --git a/acceptance/bundle/resources/pipelines/testing-same-name-libraries/databricks.yml b/acceptance/bundle/resources/pipelines/testing-same-name-libraries/databricks.yml new file mode 100644 index 00000000000..381095e1cd3 --- /dev/null +++ b/acceptance/bundle/resources/pipelines/testing-same-name-libraries/databricks.yml @@ -0,0 +1,21 @@ +bundle: + name: pipeline-testing-same-name-libraries + +resources: + pipelines: + orders: + name: orders + libraries: + - file: + path: ./transformations/orders.py + environment: + dependencies: + - ./pipeline/test_helpers-0.1-py3-none-any.whl + testing: + sources: + - glob: + include: ./tests/** + environment: + dependencies: + # A different wheel with the same file name would be uploaded to the same path. + - ./tests/test_helpers-0.1-py3-none-any.whl diff --git a/acceptance/bundle/resources/pipelines/testing-same-name-libraries/out.test.toml b/acceptance/bundle/resources/pipelines/testing-same-name-libraries/out.test.toml new file mode 100644 index 00000000000..a927a5fbc06 --- /dev/null +++ b/acceptance/bundle/resources/pipelines/testing-same-name-libraries/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/resources/pipelines/testing-same-name-libraries/output.txt b/acceptance/bundle/resources/pipelines/testing-same-name-libraries/output.txt new file mode 100644 index 00000000000..68399999172 --- /dev/null +++ b/acceptance/bundle/resources/pipelines/testing-same-name-libraries/output.txt @@ -0,0 +1,10 @@ + +>>> musterr [CLI] bundle deploy +Error: Duplicate local library names: test_helpers-0.1-py3-none-any.whl + at resources.pipelines.orders.environment.dependencies[0] + resources.pipelines.orders.testing.environment.dependencies[0] + in databricks.yml:13:13 + databricks.yml:21:15 + +Local library names must be unique but found libraries with the same name: pipeline/test_helpers-0.1-py3-none-any.whl, tests/test_helpers-0.1-py3-none-any.whl + diff --git a/acceptance/bundle/resources/pipelines/testing-same-name-libraries/pipeline/test_helpers-0.1-py3-none-any.whl b/acceptance/bundle/resources/pipelines/testing-same-name-libraries/pipeline/test_helpers-0.1-py3-none-any.whl new file mode 100644 index 00000000000..e69de29bb2d diff --git a/acceptance/bundle/resources/pipelines/testing-same-name-libraries/script b/acceptance/bundle/resources/pipelines/testing-same-name-libraries/script new file mode 100644 index 00000000000..32f10d10630 --- /dev/null +++ b/acceptance/bundle/resources/pipelines/testing-same-name-libraries/script @@ -0,0 +1 @@ +trace musterr $CLI bundle deploy diff --git a/acceptance/bundle/resources/pipelines/testing-same-name-libraries/test.toml b/acceptance/bundle/resources/pipelines/testing-same-name-libraries/test.toml new file mode 100644 index 00000000000..d018f5851ee --- /dev/null +++ b/acceptance/bundle/resources/pipelines/testing-same-name-libraries/test.toml @@ -0,0 +1,7 @@ +# Runs only against the local test server. +Cloud = false +Env.DATABRICKS_BUNDLE_ENABLE_EXPERIMENTAL_PIPELINE_TESTING = "true" + +[[Repls]] +Old = '\\' +New = '/' diff --git a/acceptance/bundle/resources/pipelines/testing-same-name-libraries/tests/test_helpers-0.1-py3-none-any.whl b/acceptance/bundle/resources/pipelines/testing-same-name-libraries/tests/test_helpers-0.1-py3-none-any.whl new file mode 100644 index 00000000000..e69de29bb2d diff --git a/acceptance/bundle/resources/pipelines/testing-same-name-libraries/tests/test_orders.py b/acceptance/bundle/resources/pipelines/testing-same-name-libraries/tests/test_orders.py new file mode 100644 index 00000000000..eb4379fd048 --- /dev/null +++ b/acceptance/bundle/resources/pipelines/testing-same-name-libraries/tests/test_orders.py @@ -0,0 +1,2 @@ +def test_orders(): + pass diff --git a/acceptance/bundle/resources/pipelines/testing-same-name-libraries/transformations/orders.py b/acceptance/bundle/resources/pipelines/testing-same-name-libraries/transformations/orders.py new file mode 100644 index 00000000000..7d4290a117a --- /dev/null +++ b/acceptance/bundle/resources/pipelines/testing-same-name-libraries/transformations/orders.py @@ -0,0 +1 @@ +x = 1 diff --git a/acceptance/bundle/resources/pipelines/testing/dist/test_helpers-0.1-py3-none-any.whl b/acceptance/bundle/resources/pipelines/testing/dist/test_helpers-0.1-py3-none-any.whl new file mode 100644 index 00000000000..e69de29bb2d diff --git a/acceptance/bundle/resources/pipelines/testing/output.txt b/acceptance/bundle/resources/pipelines/testing/output.txt index edb0eb1c49a..a1c2243b694 100644 --- a/acceptance/bundle/resources/pipelines/testing/output.txt +++ b/acceptance/bundle/resources/pipelines/testing/output.txt @@ -1,17 +1,19 @@ +=== Relative test paths resolve to the deployed workspace paths + >>> [CLI] bundle validate -o json { "environment": { "dependencies": [ - "-e .[test]", - "-r ./tests/requirements.txt", + "-e /Workspace/Users/[USERNAME]/.bundle/pipeline-testing/default/files[test]", + "-r /Workspace/Users/[USERNAME]/.bundle/pipeline-testing/default/files/tests/requirements.txt", "./dist/*.whl" ] }, "sources": [ { "glob": { - "include": "./tests/**" + "include": "/Workspace/Users/[USERNAME]/.bundle/pipeline-testing/default/files/tests/**" } } ] diff --git a/acceptance/bundle/resources/pipelines/testing/script b/acceptance/bundle/resources/pipelines/testing/script index b7f915f687c..e32724e5611 100644 --- a/acceptance/bundle/resources/pipelines/testing/script +++ b/acceptance/bundle/resources/pipelines/testing/script @@ -1 +1,2 @@ +title "Relative test paths resolve to the deployed workspace paths" trace $CLI bundle validate -o json | jq .resources.pipelines.orders.testing diff --git a/bundle/config/mutator/paths/pipeline_paths_visitor.go b/bundle/config/mutator/paths/pipeline_paths_visitor.go index b5af086dd34..83be40afd7d 100644 --- a/bundle/config/mutator/paths/pipeline_paths_visitor.go +++ b/bundle/config/mutator/paths/pipeline_paths_visitor.go @@ -44,6 +44,11 @@ func pipelineRewritePatterns() []pipelineRewritePattern { mode: TranslateModeDirectory, skipRewrite: noSkipRewrite, }, + { + pattern: base.Append(dyn.Key("testing"), dyn.Key("sources"), dyn.AnyIndex(), dyn.Key("glob"), dyn.Key("include")), + mode: TranslateModeGlob, + skipRewrite: noSkipRewrite, + }, } return allPatterns @@ -65,6 +70,12 @@ func pipelineLibrariesRewritePatterns() []pipelineRewritePattern { return !libraries.IsLibraryLocal(s) }, }, + // Test dependencies take the same forms as the pipeline's environment dependencies. + { + pattern: base.Append(dyn.Key("testing"), dyn.Key("environment"), dyn.Key("dependencies"), dyn.AnyIndex()), + mode: TranslateModeLocalRelativeWithPrefix, + skipRewrite: func(s string) bool { return !libraries.IsLibraryLocal(s) }, + }, } pipelineEnvironmentsPatternsWithPipFlags := []pipelineRewritePattern{ @@ -83,6 +94,14 @@ func pipelineLibrariesRewritePatterns() []pipelineRewritePattern { return !ok }, }, + { + base.Append(dyn.Key("testing"), dyn.Key("environment"), dyn.Key("dependencies"), dyn.AnyIndex()), + TranslateModeEnvironmentPipFlag, + func(s string) bool { + _, _, ok := libraries.IsLocalPathInPipFlag(s) + return !ok + }, + }, } return append(pipelineEnvironmentsPatterns, pipelineEnvironmentsPatternsWithPipFlags...) diff --git a/bundle/config/mutator/paths/pipeline_paths_visitor_test.go b/bundle/config/mutator/paths/pipeline_paths_visitor_test.go index eb9748bba37..6b527cf7942 100644 --- a/bundle/config/mutator/paths/pipeline_paths_visitor_test.go +++ b/bundle/config/mutator/paths/pipeline_paths_visitor_test.go @@ -39,6 +39,13 @@ func TestVisitPipelinePaths(t *testing.T) { "src/foo.whl", }, }, + // The acceptance test bundle/resources/pipelines/testing covers the testing paths + // that are rewritten. A package name is not a path. + Testing: &pipelines.PipelineTesting{ + Environment: &pipelines.PipelineTestEnvironment{ + Dependencies: []string{"faker"}, + }, + }, }, }, }, diff --git a/bundle/libraries/expand_glob_references.go b/bundle/libraries/expand_glob_references.go index df089f880a7..91f5ab261b1 100644 --- a/bundle/libraries/expand_glob_references.go +++ b/bundle/libraries/expand_glob_references.go @@ -198,6 +198,15 @@ var pipelineEnvDepsPattern = dyn.NewPattern( dyn.Key("dependencies"), ) +var pipelineTestingEnvDepsPattern = dyn.NewPattern( + dyn.Key("resources"), + dyn.Key("pipelines"), + dyn.AnyKey(), + dyn.Key("testing"), + dyn.Key("environment"), + dyn.Key("dependencies"), +) + var clusterLibrariesPattern = dyn.NewPattern( dyn.Key("resources"), dyn.Key("clusters"), @@ -223,6 +232,10 @@ func (e *expand) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { pattern: pipelineEnvDepsPattern, fn: expandEnvironmentDeps, }, + { + pattern: pipelineTestingEnvDepsPattern, + fn: expandEnvironmentDeps, + }, { pattern: clusterLibrariesPattern, fn: expandLibraries, diff --git a/bundle/libraries/remote_path.go b/bundle/libraries/remote_path.go index ff6be05c3b2..020fd5bbdaa 100644 --- a/bundle/libraries/remote_path.go +++ b/bundle/libraries/remote_path.go @@ -144,6 +144,7 @@ func collectLocalLibraries(b *bundle.Bundle) (map[string][]LocationToUpdate, err clusterLibrariesPattern.Append(dyn.AnyIndex(), dyn.Key("jar")), envDepsPattern.Append(dyn.AnyIndex()), pipelineEnvDepsPattern.Append(dyn.AnyIndex()), + pipelineTestingEnvDepsPattern.Append(dyn.AnyIndex()), // The AI Runtime task's code_source_path is a local archive (typically an // artifact-built .tar.gz) that must be uploaded and referenced by its remote // path, exactly like a wheel or jar library. diff --git a/bundle/libraries/same_name_libraries.go b/bundle/libraries/same_name_libraries.go index 722d42c7f64..e4e1c62c614 100644 --- a/bundle/libraries/same_name_libraries.go +++ b/bundle/libraries/same_name_libraries.go @@ -22,6 +22,7 @@ var patterns = []dyn.Pattern{ clusterLibrariesPattern.Append(dyn.AnyIndex(), dyn.Key("jar")), envDepsPattern.Append(dyn.AnyIndex()), pipelineEnvDepsPattern.Append(dyn.AnyIndex()), + pipelineTestingEnvDepsPattern.Append(dyn.AnyIndex()), } type libData struct { From 86c487866d7485cd240afd4748916f548f6cbf6d Mon Sep 17 00:00:00 2001 From: Bhukya Date: Mon, 16 Nov 2026 23:55:08 +0000 Subject: [PATCH 05/10] Define the dependency path patterns of pipelines once for both locations environment.dependencies and testing.environment.dependencies share the translate modes and skip functions, so the patterns are built in one loop over the two locations. The order of the patterns does not change. Co-authored-by: Isaac --- .../mutator/paths/pipeline_paths_visitor.go | 61 ++++++------------- 1 file changed, 18 insertions(+), 43 deletions(-) diff --git a/bundle/config/mutator/paths/pipeline_paths_visitor.go b/bundle/config/mutator/paths/pipeline_paths_visitor.go index 83be40afd7d..0824dbbd696 100644 --- a/bundle/config/mutator/paths/pipeline_paths_visitor.go +++ b/bundle/config/mutator/paths/pipeline_paths_visitor.go @@ -55,56 +55,31 @@ func pipelineRewritePatterns() []pipelineRewritePattern { } func pipelineLibrariesRewritePatterns() []pipelineRewritePattern { - pipelineEnvironmentsPatterns := []pipelineRewritePattern{ - { - pattern: dyn.NewPattern( - dyn.Key("resources"), - dyn.Key("pipelines"), - dyn.AnyKey(), - dyn.Key("environment"), - dyn.Key("dependencies"), - dyn.AnyIndex(), - ), - mode: TranslateModeLocalRelativeWithPrefix, - skipRewrite: func(s string) bool { - return !libraries.IsLibraryLocal(s) - }, - }, - // Test dependencies take the same forms as the pipeline's environment dependencies. - { - pattern: base.Append(dyn.Key("testing"), dyn.Key("environment"), dyn.Key("dependencies"), dyn.AnyIndex()), + // Test dependencies take the same forms as the pipeline's environment dependencies. + dependencies := []dyn.Pattern{ + base.Append(dyn.Key("environment"), dyn.Key("dependencies"), dyn.AnyIndex()), + base.Append(dyn.Key("testing"), dyn.Key("environment"), dyn.Key("dependencies"), dyn.AnyIndex()), + } + + var patterns []pipelineRewritePattern + for _, pattern := range dependencies { + patterns = append(patterns, pipelineRewritePattern{ + pattern: pattern, mode: TranslateModeLocalRelativeWithPrefix, skipRewrite: func(s string) bool { return !libraries.IsLibraryLocal(s) }, - }, + }) } - - pipelineEnvironmentsPatternsWithPipFlags := []pipelineRewritePattern{ - { - dyn.NewPattern( - dyn.Key("resources"), - dyn.Key("pipelines"), - dyn.AnyKey(), - dyn.Key("environment"), - dyn.Key("dependencies"), - dyn.AnyIndex(), - ), - TranslateModeEnvironmentPipFlag, - func(s string) bool { - _, _, ok := libraries.IsLocalPathInPipFlag(s) - return !ok - }, - }, - { - base.Append(dyn.Key("testing"), dyn.Key("environment"), dyn.Key("dependencies"), dyn.AnyIndex()), - TranslateModeEnvironmentPipFlag, - func(s string) bool { + for _, pattern := range dependencies { + patterns = append(patterns, pipelineRewritePattern{ + pattern: pattern, + mode: TranslateModeEnvironmentPipFlag, + skipRewrite: func(s string) bool { _, _, ok := libraries.IsLocalPathInPipFlag(s) return !ok }, - }, + }) } - - return append(pipelineEnvironmentsPatterns, pipelineEnvironmentsPatternsWithPipFlags...) + return patterns } func VisitPipelinePaths(value dyn.Value, fn VisitFunc) (dyn.Value, error) { From 7cb505dea9cfd17779534f088283f5c8726d4e83 Mon Sep 17 00:00:00 2001 From: Bhukya Date: Tue, 17 Nov 2026 15:54:05 +0000 Subject: [PATCH 06/10] Use the patched wheel in the test dependencies of pipelines With dynamic_version, the deploy uploaded the patched wheel, but testing.environment.dependencies kept the original file name, so tests could run cached code. Co-authored-by: Isaac --- .../testing-dynamic-version/databricks.yml | 25 +++++++++++++++++ .../testing-dynamic-version/out.test.toml | 2 ++ .../testing-dynamic-version/output.txt | 27 +++++++++++++++++++ .../pipelines/testing-dynamic-version/script | 5 ++++ .../testing-dynamic-version/test.toml | 4 +++ .../tests/test_orders.py | 2 ++ .../transformations/orders.py | 1 + bundle/libraries/switch_to_patched_wheels.go | 18 +++++++++++++ 8 files changed, 84 insertions(+) create mode 100644 acceptance/bundle/resources/pipelines/testing-dynamic-version/databricks.yml create mode 100644 acceptance/bundle/resources/pipelines/testing-dynamic-version/out.test.toml create mode 100644 acceptance/bundle/resources/pipelines/testing-dynamic-version/output.txt create mode 100644 acceptance/bundle/resources/pipelines/testing-dynamic-version/script create mode 100644 acceptance/bundle/resources/pipelines/testing-dynamic-version/test.toml create mode 100644 acceptance/bundle/resources/pipelines/testing-dynamic-version/tests/test_orders.py create mode 100644 acceptance/bundle/resources/pipelines/testing-dynamic-version/transformations/orders.py diff --git a/acceptance/bundle/resources/pipelines/testing-dynamic-version/databricks.yml b/acceptance/bundle/resources/pipelines/testing-dynamic-version/databricks.yml new file mode 100644 index 00000000000..09b1f8647e0 --- /dev/null +++ b/acceptance/bundle/resources/pipelines/testing-dynamic-version/databricks.yml @@ -0,0 +1,25 @@ +bundle: + name: pipeline-testing-dynamic-version + +artifacts: + other_test_code: + type: whl + files: + - source: dist/other_test_code-0.0.1-py3-none-any.whl + dynamic_version: true + +resources: + pipelines: + orders: + name: orders + libraries: + - glob: + include: ./transformations/** + testing: + sources: + - glob: + include: ./tests/** + environment: + dependencies: + - ./dist/other_test_code-0.0.1-py3-none-any.whl + - ./dist/other_test_code-0.0.1-py3-none-any.whl[test] diff --git a/acceptance/bundle/resources/pipelines/testing-dynamic-version/out.test.toml b/acceptance/bundle/resources/pipelines/testing-dynamic-version/out.test.toml new file mode 100644 index 00000000000..a927a5fbc06 --- /dev/null +++ b/acceptance/bundle/resources/pipelines/testing-dynamic-version/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/resources/pipelines/testing-dynamic-version/output.txt b/acceptance/bundle/resources/pipelines/testing-dynamic-version/output.txt new file mode 100644 index 00000000000..9af556df861 --- /dev/null +++ b/acceptance/bundle/resources/pipelines/testing-dynamic-version/output.txt @@ -0,0 +1,27 @@ + +=== Test dependencies use the patched wheel, with extras kept + +>>> [CLI] bundle deploy +Uploading .databricks/bundle/default/patched_wheels/other_test_code_other_test_code/other_test_code-0.0.1+[UNIX_TIME_NANOS]-py3-none-any.whl... +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/pipeline-testing-dynamic-version/default/files... +Created pipelines.orders +Files: 4 uploaded, 0 deleted +Resources: 1 created, 0 changed, 0 deleted, 0 unchanged + +>>> print_requests.py //api/2.0/pipelines --nostamp +[ + "/Workspace/Users/[USERNAME]/.bundle/pipeline-testing-dynamic-version/default/artifacts/.internal/other_test_code-0.0.1+[UNIX_TIME_NANOS]-py3-none-any.whl", + "/Workspace/Users/[USERNAME]/.bundle/pipeline-testing-dynamic-version/default/artifacts/.internal/other_test_code-0.0.1+[UNIX_TIME_NANOS]-py3-none-any.whl[test]" +] + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.pipelines.orders + +This action will result in the deletion of the following Lakeflow Spark Declarative Pipelines along with the +Streaming Tables (STs) and Materialized Views (MVs) managed by them. Set 'cascade_on_destroy: false' on a pipeline to retain datasets on pipeline deletion: + delete resources.pipelines.orders + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/pipeline-testing-dynamic-version/default + +Destroy: 1 deleted diff --git a/acceptance/bundle/resources/pipelines/testing-dynamic-version/script b/acceptance/bundle/resources/pipelines/testing-dynamic-version/script new file mode 100644 index 00000000000..0b3ca1ea393 --- /dev/null +++ b/acceptance/bundle/resources/pipelines/testing-dynamic-version/script @@ -0,0 +1,5 @@ +title "Test dependencies use the patched wheel, with extras kept" +trace $CLI bundle deploy +trace print_requests.py //api/2.0/pipelines --nostamp | jq '.body.testing.environment.dependencies' +trace $CLI bundle destroy --auto-approve +rm "$OUT_REQUESTS" diff --git a/acceptance/bundle/resources/pipelines/testing-dynamic-version/test.toml b/acceptance/bundle/resources/pipelines/testing-dynamic-version/test.toml new file mode 100644 index 00000000000..25d8c36639f --- /dev/null +++ b/acceptance/bundle/resources/pipelines/testing-dynamic-version/test.toml @@ -0,0 +1,4 @@ +# Runs only against the local test server. +Cloud = false +RecordRequests = true +Ignore = [".databricks"] diff --git a/acceptance/bundle/resources/pipelines/testing-dynamic-version/tests/test_orders.py b/acceptance/bundle/resources/pipelines/testing-dynamic-version/tests/test_orders.py new file mode 100644 index 00000000000..eb4379fd048 --- /dev/null +++ b/acceptance/bundle/resources/pipelines/testing-dynamic-version/tests/test_orders.py @@ -0,0 +1,2 @@ +def test_orders(): + pass diff --git a/acceptance/bundle/resources/pipelines/testing-dynamic-version/transformations/orders.py b/acceptance/bundle/resources/pipelines/testing-dynamic-version/transformations/orders.py new file mode 100644 index 00000000000..3a22621c3bf --- /dev/null +++ b/acceptance/bundle/resources/pipelines/testing-dynamic-version/transformations/orders.py @@ -0,0 +1 @@ +from pyspark import pipelines as dp diff --git a/bundle/libraries/switch_to_patched_wheels.go b/bundle/libraries/switch_to_patched_wheels.go index 3c5ff184a97..0169f1adb3a 100644 --- a/bundle/libraries/switch_to_patched_wheels.go +++ b/bundle/libraries/switch_to_patched_wheels.go @@ -95,6 +95,24 @@ func (c switchToPatchedWheels) Apply(ctx context.Context, b *bundle.Bundle) diag } } + // Update resources.pipelines.*.testing.environment.dependencies[*] + for pipelineName, pipelineRef := range b.Config.Resources.Pipelines { + if pipelineRef == nil || pipelineRef.Testing == nil || pipelineRef.Testing.Environment == nil { + continue + } + deps := pipelineRef.Testing.Environment.Dependencies + for depInd, dep := range deps { + base, extras := patchwheel.SplitWheelExtras(dep) + repl := replacements[base] + if repl != "" { + log.Debugf(ctx, "Updating resources.pipelines.%s.testing.environment.dependencies[%d] from %s to %s", pipelineName, depInd, dep, repl+extras) + deps[depInd] = repl + extras + } else { + log.Debugf(ctx, "Not updating resources.pipelines.%s.testing.environment.dependencies[%d] from %s. Available replacements: %v", pipelineName, depInd, dep, slices.Sorted(maps.Keys(replacements))) + } + } + } + return nil } From 41faaddfd3db8d902279ff891b5698b1dfe9fe8c Mon Sep 17 00:00:00 2001 From: Bhukya Date: Tue, 17 Nov 2026 15:56:36 +0000 Subject: [PATCH 07/10] Run the same-name test of pipeline testing without the variable Co-authored-by: Isaac --- .../resources/pipelines/testing-same-name-libraries/test.toml | 1 - 1 file changed, 1 deletion(-) diff --git a/acceptance/bundle/resources/pipelines/testing-same-name-libraries/test.toml b/acceptance/bundle/resources/pipelines/testing-same-name-libraries/test.toml index d018f5851ee..d77028ab40a 100644 --- a/acceptance/bundle/resources/pipelines/testing-same-name-libraries/test.toml +++ b/acceptance/bundle/resources/pipelines/testing-same-name-libraries/test.toml @@ -1,6 +1,5 @@ # Runs only against the local test server. Cloud = false -Env.DATABRICKS_BUNDLE_ENABLE_EXPERIMENTAL_PIPELINE_TESTING = "true" [[Repls]] Old = '\\' From 669ee9bdaac4783c1d27186e35514c04a1cea1dc Mon Sep 17 00:00:00 2001 From: Bhukya Date: Tue, 17 Nov 2026 16:16:53 +0000 Subject: [PATCH 08/10] Copy the prebuilt wheel in the dynamic_version test of pipeline testing dist/ is ignored, so the test now copies the wheel from the artifacts tests. Co-authored-by: Isaac --- .../bundle/resources/pipelines/testing-dynamic-version/script | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/acceptance/bundle/resources/pipelines/testing-dynamic-version/script b/acceptance/bundle/resources/pipelines/testing-dynamic-version/script index 0b3ca1ea393..e71c775893f 100644 --- a/acceptance/bundle/resources/pipelines/testing-dynamic-version/script +++ b/acceptance/bundle/resources/pipelines/testing-dynamic-version/script @@ -1,3 +1,7 @@ +trap "rm -fr dist" EXIT +mkdir dist +cp $TESTDIR/../../../artifacts/whl_prebuilt_multiple/dist/lib/other_test_code-0.0.1-py3-none-any.whl dist + title "Test dependencies use the patched wheel, with extras kept" trace $CLI bundle deploy trace print_requests.py //api/2.0/pipelines --nostamp | jq '.body.testing.environment.dependencies' From 3173c73153d4f0d7ec5944cf707515fd200d6bc8 Mon Sep 17 00:00:00 2001 From: Bhukya Date: Mon, 16 Nov 2026 20:54:32 +0000 Subject: [PATCH 09/10] Test that bundles deploy the testing property of pipelines and show remote changes in bundle plan bundle deploy sends a pipeline's testing on Create and Edit, and bundle plan reads it back, so a change made outside the bundle shows in the plan and the next deploy undoes it. The Go SDK has the field, so the direct engine and the test server already handle it. This change adds the acceptance test for it. Co-authored-by: Isaac --- .../resources/pipelines/testing/output.txt | 112 ++++++++++++++++++ .../bundle/resources/pipelines/testing/script | 33 ++++++ 2 files changed, 145 insertions(+) diff --git a/acceptance/bundle/resources/pipelines/testing/output.txt b/acceptance/bundle/resources/pipelines/testing/output.txt index a1c2243b694..2fd4f720c15 100644 --- a/acceptance/bundle/resources/pipelines/testing/output.txt +++ b/acceptance/bundle/resources/pipelines/testing/output.txt @@ -18,3 +18,115 @@ } ] } + +=== Deploy sends testing, with the local wheel uploaded + +>>> [CLI] bundle deploy +Uploading dist/test_helpers-0.1-py3-none-any.whl... +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/pipeline-testing/default/files... +Created pipelines.orders +Files: 6 uploaded, 0 deleted +Resources: 1 created, 0 changed, 0 deleted, 0 unchanged + +>>> print_requests.py //api/2.0/pipelines --nostamp +{ + "method": "POST", + "path": "/api/2.0/pipelines", + "testing": { + "environment": { + "dependencies": [ + "-e /Workspace/Users/[USERNAME]/.bundle/pipeline-testing/default/files[test]", + "-r /Workspace/Users/[USERNAME]/.bundle/pipeline-testing/default/files/tests/requirements.txt", + "/Workspace/Users/[USERNAME]/.bundle/pipeline-testing/default/artifacts/.internal/test_helpers-0.1-py3-none-any.whl" + ] + }, + "sources": [ + { + "glob": { + "include": "/Workspace/Users/[USERNAME]/.bundle/pipeline-testing/default/files/tests/**" + } + } + ] + } +} + +=== No changes after the deploy + +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged + +=== A remote edit that drops testing shows in the plan + +>>> [CLI] bundle plan -o json +"update" + +>>> [CLI] bundle deploy +Uploading dist/test_helpers-0.1-py3-none-any.whl... +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/pipeline-testing/default/files... +Updated pipelines.orders +Files: 1 uploaded, 0 deleted +Resources: 0 created, 1 changed, 0 deleted, 0 unchanged + +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged + +=== Changing testing updates the pipeline + +>>> update_file.py databricks.yml ./tests/** ./tests/unit/** + +>>> [CLI] bundle plan +update pipelines.orders + +Plan: 0 to add, 1 to change, 0 to delete, 0 unchanged + +>>> [CLI] bundle deploy +Uploading dist/test_helpers-0.1-py3-none-any.whl... +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/pipeline-testing/default/files... +Updated pipelines.orders +Files: 1 uploaded, 0 deleted +Resources: 0 created, 1 changed, 0 deleted, 0 unchanged + +>>> print_requests.py //api/2.0/pipelines --nostamp +{ + "method": "PUT", + "path": "/api/2.0/pipelines/[ORDERS_ID]", + "sources": [ + { + "glob": { + "include": "/Workspace/Users/[USERNAME]/.bundle/pipeline-testing/default/files/tests/unit/**" + } + } + ] +} + +=== Removing testing sends an update without it + +>>> [CLI] bundle plan +update pipelines.orders + +Plan: 0 to add, 1 to change, 0 to delete, 0 unchanged + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/pipeline-testing/default/files... +Updated pipelines.orders +Files: 1 uploaded, 0 deleted +Resources: 0 created, 1 changed, 0 deleted, 0 unchanged + +>>> print_requests.py //api/2.0/pipelines --nostamp +{ + "method": "PUT", + "path": "/api/2.0/pipelines/[ORDERS_ID]", + "testing": null +} + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.pipelines.orders + +This action will result in the deletion of the following Lakeflow Spark Declarative Pipelines along with the +Streaming Tables (STs) and Materialized Views (MVs) managed by them. Set 'cascade_on_destroy: false' on a pipeline to retain datasets on pipeline deletion: + delete resources.pipelines.orders + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/pipeline-testing/default + +Destroy: 1 deleted diff --git a/acceptance/bundle/resources/pipelines/testing/script b/acceptance/bundle/resources/pipelines/testing/script index e32724e5611..f3914bdfd48 100644 --- a/acceptance/bundle/resources/pipelines/testing/script +++ b/acceptance/bundle/resources/pipelines/testing/script @@ -1,2 +1,35 @@ title "Relative test paths resolve to the deployed workspace paths" trace $CLI bundle validate -o json | jq .resources.pipelines.orders.testing + +title "Deploy sends testing, with the local wheel uploaded" +trace $CLI bundle deploy +trace print_requests.py //api/2.0/pipelines --nostamp | jq '{method, path, testing: .body.testing}' + +title "No changes after the deploy" +trace $CLI bundle plan +rm "$OUT_REQUESTS" + +title "A remote edit that drops testing shows in the plan" +PIPELINE_ID=$(read_id.py orders) +MSYS_NO_PATHCONV=1 $CLI api get "/api/2.0/pipelines/$PIPELINE_ID" | jq '.spec | del(.testing)' > edit.json +MSYS_NO_PATHCONV=1 $CLI api put "/api/2.0/pipelines/$PIPELINE_ID" --json @edit.json &> LOG.edit +rm edit.json +trace $CLI bundle plan -o json | jq '.plan["resources.pipelines.orders"].changes.testing.action' +trace $CLI bundle deploy +trace $CLI bundle plan +rm "$OUT_REQUESTS" + +title "Changing testing updates the pipeline" +trace update_file.py databricks.yml './tests/**' './tests/unit/**' +trace $CLI bundle plan +trace $CLI bundle deploy +trace print_requests.py //api/2.0/pipelines --nostamp | jq '{method, path, sources: .body.testing.sources}' + +title "Removing testing sends an update without it" +update_file.py databricks.yml "$(sed -n '/^ testing:/,$p' databricks.yml)" "" +trace $CLI bundle plan +trace $CLI bundle deploy +trace print_requests.py //api/2.0/pipelines --nostamp | jq '{method, path, testing: .body.testing}' + +trace $CLI bundle destroy --auto-approve +rm "$OUT_REQUESTS" From 96529443660be20600940d202f836177158f590f Mon Sep 17 00:00:00 2001 From: Bhukya Date: Mon, 16 Nov 2026 23:55:53 +0000 Subject: [PATCH 10/10] Check that the update without testing has no testing key The last step of the acceptance test now asserts that the request body has no testing key, instead of printing testing as null, which can also mean an explicit null. Co-authored-by: Isaac --- acceptance/bundle/resources/pipelines/testing/output.txt | 2 +- acceptance/bundle/resources/pipelines/testing/script | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/acceptance/bundle/resources/pipelines/testing/output.txt b/acceptance/bundle/resources/pipelines/testing/output.txt index 2fd4f720c15..3cccee04481 100644 --- a/acceptance/bundle/resources/pipelines/testing/output.txt +++ b/acceptance/bundle/resources/pipelines/testing/output.txt @@ -116,7 +116,7 @@ Resources: 0 created, 1 changed, 0 deleted, 0 unchanged { "method": "PUT", "path": "/api/2.0/pipelines/[ORDERS_ID]", - "testing": null + "has_testing": false } >>> [CLI] bundle destroy --auto-approve diff --git a/acceptance/bundle/resources/pipelines/testing/script b/acceptance/bundle/resources/pipelines/testing/script index f3914bdfd48..2c12917d00f 100644 --- a/acceptance/bundle/resources/pipelines/testing/script +++ b/acceptance/bundle/resources/pipelines/testing/script @@ -29,7 +29,7 @@ title "Removing testing sends an update without it" update_file.py databricks.yml "$(sed -n '/^ testing:/,$p' databricks.yml)" "" trace $CLI bundle plan trace $CLI bundle deploy -trace print_requests.py //api/2.0/pipelines --nostamp | jq '{method, path, testing: .body.testing}' +trace print_requests.py //api/2.0/pipelines --nostamp | jq '{method, path, has_testing: (.body | has("testing"))}' trace $CLI bundle destroy --auto-approve rm "$OUT_REQUESTS"