diff --git a/acceptance/bundle/resources/pipelines/testing/databricks.yml b/acceptance/bundle/resources/pipelines/testing/databricks.yml new file mode 100644 index 00000000000..346cc9d7963 --- /dev/null +++ b/acceptance/bundle/resources/pipelines/testing/databricks.yml @@ -0,0 +1,19 @@ +bundle: + name: pipeline-testing + +resources: + pipelines: + orders: + name: orders + libraries: + - glob: + include: ./transformations/** + testing: + sources: + - glob: + include: ./tests/** + environment: + dependencies: + - -e .[test] + - -r ./tests/requirements.txt + - ./dist/*.whl diff --git a/acceptance/bundle/resources/pipelines/testing/out.test.toml b/acceptance/bundle/resources/pipelines/testing/out.test.toml new file mode 100644 index 00000000000..a927a5fbc06 --- /dev/null +++ b/acceptance/bundle/resources/pipelines/testing/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/resources/pipelines/testing/output.txt b/acceptance/bundle/resources/pipelines/testing/output.txt new file mode 100644 index 00000000000..edb0eb1c49a --- /dev/null +++ b/acceptance/bundle/resources/pipelines/testing/output.txt @@ -0,0 +1,18 @@ + +>>> [CLI] bundle validate -o json +{ + "environment": { + "dependencies": [ + "-e .[test]", + "-r ./tests/requirements.txt", + "./dist/*.whl" + ] + }, + "sources": [ + { + "glob": { + "include": "./tests/**" + } + } + ] +} diff --git a/acceptance/bundle/resources/pipelines/testing/pyproject.toml b/acceptance/bundle/resources/pipelines/testing/pyproject.toml new file mode 100644 index 00000000000..1c198acc7cb --- /dev/null +++ b/acceptance/bundle/resources/pipelines/testing/pyproject.toml @@ -0,0 +1,6 @@ +[project] +name = "orders" +version = "0.1" + +[project.optional-dependencies] +test = ["faker"] diff --git a/acceptance/bundle/resources/pipelines/testing/script b/acceptance/bundle/resources/pipelines/testing/script new file mode 100644 index 00000000000..b7f915f687c --- /dev/null +++ b/acceptance/bundle/resources/pipelines/testing/script @@ -0,0 +1 @@ +trace $CLI bundle validate -o json | jq .resources.pipelines.orders.testing diff --git a/acceptance/bundle/resources/pipelines/testing/test.toml b/acceptance/bundle/resources/pipelines/testing/test.toml new file mode 100644 index 00000000000..25d8c36639f --- /dev/null +++ b/acceptance/bundle/resources/pipelines/testing/test.toml @@ -0,0 +1,4 @@ +# Runs only against the local test server. +Cloud = false +RecordRequests = true +Ignore = [".databricks"] diff --git a/acceptance/bundle/resources/pipelines/testing/tests/requirements.txt b/acceptance/bundle/resources/pipelines/testing/tests/requirements.txt new file mode 100644 index 00000000000..9d7f6e827a1 --- /dev/null +++ b/acceptance/bundle/resources/pipelines/testing/tests/requirements.txt @@ -0,0 +1 @@ +faker diff --git a/acceptance/bundle/resources/pipelines/testing/tests/test_orders.py b/acceptance/bundle/resources/pipelines/testing/tests/test_orders.py new file mode 100644 index 00000000000..eb4379fd048 --- /dev/null +++ b/acceptance/bundle/resources/pipelines/testing/tests/test_orders.py @@ -0,0 +1,2 @@ +def test_orders(): + pass diff --git a/acceptance/bundle/resources/pipelines/testing/transformations/orders.py b/acceptance/bundle/resources/pipelines/testing/transformations/orders.py new file mode 100644 index 00000000000..3a22621c3bf --- /dev/null +++ b/acceptance/bundle/resources/pipelines/testing/transformations/orders.py @@ -0,0 +1 @@ +from pyspark import pipelines as dp diff --git a/bundle/internal/schema/annotations.yml b/bundle/internal/schema/annotations.yml index 58abf721658..41a3b0084d0 100644 --- a/bundle/internal/schema/annotations.yml +++ b/bundle/internal/schema/annotations.yml @@ -1465,6 +1465,20 @@ resources: "user_name": "description": |- The name of the user granted the permission level. + "testing": + "description": |- + The test configuration of the pipeline. Only test-only updates, such as those started by `databricks pipelines test`, use it. + "$fields": + "environment": + "$fields": + "dependencies": + "description": |- + A list of files that declare the packages the tests need: a project folder, a requirements file, or a wheel, in the workspace or a UC volume. A project folder can take `-e` and an extra, for example `-e ./project[test]`. A relative path, such as `-e ./project[test]` or `-r ./tests/requirements.txt`, is relative to the YAML file that defines it. Package names and URLs are rejected. + "sources": + "$fields": + "glob": + "description": |- + A `.py` test file, or a folder followed by `/**`. Other wildcards are rejected, and the sources must not overlap the pipeline's `libraries`. A relative path is relative to the YAML file that defines it. "trigger": "deprecation_message": |- Use continuous instead diff --git a/bundle/schema/jsonschema.json b/bundle/schema/jsonschema.json index 2d5a35b5a5e..b45eb01c7a4 100644 --- a/bundle/schema/jsonschema.json +++ b/bundle/schema/jsonschema.json @@ -2454,7 +2454,7 @@ "deprecated": true }, "testing": { - "description": "[Private Preview] Test configuration for this pipeline. Used only by test-only updates.", + "description": "[Private Preview] The test configuration of the pipeline. Only test-only updates, such as those started by `databricks pipelines test`, use it.", "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/pipelines.PipelineTesting", "x-databricks-launch-stage": "PRIVATE_PREVIEW", "doNotSuggest": true @@ -18261,7 +18261,7 @@ "description": "The packages that a pipeline's tests need in addition to the pipeline's environment.", "properties": { "dependencies": { - "description": "[Private Preview] A list of files that declare the packages the tests need: a project folder, a requirements\nfile, or a wheel, in the workspace or a UC volume. A project folder can take `--editable` and\nan extra, for example `--editable /Workspace/Users/someone@example.com/orders[test]`. Package\nnames, such as `faker==33.0`, and URLs are rejected.", + "description": "[Private Preview] A list of files that declare the packages the tests need: a project folder, a requirements file, or a wheel, in the workspace or a UC volume. A project folder can take `-e` and an extra, for example `-e ./project[test]`. A relative path, such as `-e ./project[test]` or `-r ./tests/requirements.txt`, is relative to the YAML file that defines it. Package names and URLs are rejected.", "$ref": "#/$defs/slice/string", "x-databricks-launch-stage": "PRIVATE_PREVIEW", "doNotSuggest": true @@ -18282,7 +18282,7 @@ "description": "A workspace location that holds pipeline tests.", "properties": { "glob": { - "description": "[Private Preview] An absolute workspace path to a test file, or to a folder followed by `/**`.", + "description": "[Private Preview] A `.py` test file, or a folder followed by `/**`. Other wildcards are rejected, and the sources must not overlap the pipeline's `libraries`. A relative path is relative to the YAML file that defines it.", "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/pipelines.PathPattern", "x-databricks-launch-stage": "PRIVATE_PREVIEW", "doNotSuggest": true diff --git a/python/databricks/bundles/pipelines/_models/pipeline.py b/python/databricks/bundles/pipelines/_models/pipeline.py index c8efe551d4f..c33ade8cd12 100644 --- a/python/databricks/bundles/pipelines/_models/pipeline.py +++ b/python/databricks/bundles/pipelines/_models/pipeline.py @@ -266,7 +266,7 @@ class Pipeline(Resource): """ :meta private: [EXPERIMENTAL] - [Private Preview] Test configuration for this pipeline. Used only by test-only updates. + [Private Preview] The test configuration of the pipeline. Only test-only updates, such as those started by `databricks pipelines test`, use it. """ usage_policy_id: VariableOrOptional[str] = None @@ -490,7 +490,7 @@ class PipelineDict(TypedDict, total=False): """ :meta private: [EXPERIMENTAL] - [Private Preview] Test configuration for this pipeline. Used only by test-only updates. + [Private Preview] The test configuration of the pipeline. Only test-only updates, such as those started by `databricks pipelines test`, use it. """ usage_policy_id: VariableOrOptional[str] diff --git a/python/databricks/bundles/pipelines/_models/pipeline_test_environment.py b/python/databricks/bundles/pipelines/_models/pipeline_test_environment.py index 827e30e91a4..897a1e94785 100644 --- a/python/databricks/bundles/pipelines/_models/pipeline_test_environment.py +++ b/python/databricks/bundles/pipelines/_models/pipeline_test_environment.py @@ -23,10 +23,7 @@ class PipelineTestEnvironment: """ :meta private: [EXPERIMENTAL] - [Private Preview] A list of files that declare the packages the tests need: a project folder, a requirements - file, or a wheel, in the workspace or a UC volume. A project folder can take `--editable` and - an extra, for example `--editable /Workspace/Users/someone@example.com/orders[test]`. Package - names, such as `faker==33.0`, and URLs are rejected. + [Private Preview] A list of files that declare the packages the tests need: a project folder, a requirements file, or a wheel, in the workspace or a UC volume. A project folder can take `-e` and an extra, for example `-e ./project[test]`. A relative path, such as `-e ./project[test]` or `-r ./tests/requirements.txt`, is relative to the YAML file that defines it. Package names and URLs are rejected. """ @classmethod @@ -44,10 +41,7 @@ class PipelineTestEnvironmentDict(TypedDict, total=False): """ :meta private: [EXPERIMENTAL] - [Private Preview] A list of files that declare the packages the tests need: a project folder, a requirements - file, or a wheel, in the workspace or a UC volume. A project folder can take `--editable` and - an extra, for example `--editable /Workspace/Users/someone@example.com/orders[test]`. Package - names, such as `faker==33.0`, and URLs are rejected. + [Private Preview] A list of files that declare the packages the tests need: a project folder, a requirements file, or a wheel, in the workspace or a UC volume. A project folder can take `-e` and an extra, for example `-e ./project[test]`. A relative path, such as `-e ./project[test]` or `-r ./tests/requirements.txt`, is relative to the YAML file that defines it. Package names and URLs are rejected. """ diff --git a/python/databricks/bundles/pipelines/_models/pipeline_test_source.py b/python/databricks/bundles/pipelines/_models/pipeline_test_source.py index 8abc02e9b7d..fcf4dcfc671 100644 --- a/python/databricks/bundles/pipelines/_models/pipeline_test_source.py +++ b/python/databricks/bundles/pipelines/_models/pipeline_test_source.py @@ -27,7 +27,7 @@ class PipelineTestSource: """ :meta private: [EXPERIMENTAL] - [Private Preview] An absolute workspace path to a test file, or to a folder followed by `/**`. + [Private Preview] A `.py` test file, or a folder followed by `/**`. Other wildcards are rejected, and the sources must not overlap the pipeline's `libraries`. A relative path is relative to the YAML file that defines it. """ @classmethod @@ -45,7 +45,7 @@ class PipelineTestSourceDict(TypedDict, total=False): """ :meta private: [EXPERIMENTAL] - [Private Preview] An absolute workspace path to a test file, or to a folder followed by `/**`. + [Private Preview] A `.py` test file, or a folder followed by `/**`. Other wildcards are rejected, and the sources must not overlap the pipeline's `libraries`. A relative path is relative to the YAML file that defines it. """