From 0a8febb95e716cca38412f47395880e0c52fa404 Mon Sep 17 00:00:00 2001 From: Bhukya Date: Tue, 17 Nov 2026 10:03:21 +0000 Subject: [PATCH 1/3] Remove the experimental gate from databricks pipelines test The command no longer needs DATABRICKS_CLI_PIPELINES_TEST=true. Whether a workspace can run pipeline tests is decided by the service, so a run of all tests now always sends test_options, empty when nothing is selected. Where the testing API is off, the service rejects such a request with a clear error, instead of running its older test-only update. Co-authored-by: Isaac --- .../pipelines/test/arguments/out.gate-off.xml | 8 -------- .../pipelines/test/arguments/output.txt | 12 ----------- acceptance/pipelines/test/arguments/script | 7 ------- .../pipelines/test/immutable/output.txt | 3 ++- .../test/not-enabled/out.not-enabled.xml | 8 ++++++++ .../pipelines/test/not-enabled/out.test.toml | 1 + .../pipelines/test/not-enabled/output.txt | 15 ++++++++++++++ acceptance/pipelines/test/not-enabled/script | 6 ++++++ .../pipelines/test/not-enabled/test.toml | 11 ++++++++++ .../pipelines/test/selection/output.txt | 3 ++- .../pipelines/test/submit-error/output.txt | 3 ++- acceptance/pipelines/test/test.toml | 2 -- cmd/pipelines/test.go | 20 ++++++------------- cmd/pipelines/test_test.go | 8 +++++++- 14 files changed, 60 insertions(+), 47 deletions(-) delete mode 100644 acceptance/pipelines/test/arguments/out.gate-off.xml create mode 100644 acceptance/pipelines/test/not-enabled/out.not-enabled.xml create mode 100644 acceptance/pipelines/test/not-enabled/out.test.toml create mode 100644 acceptance/pipelines/test/not-enabled/output.txt create mode 100644 acceptance/pipelines/test/not-enabled/script create mode 100644 acceptance/pipelines/test/not-enabled/test.toml diff --git a/acceptance/pipelines/test/arguments/out.gate-off.xml b/acceptance/pipelines/test/arguments/out.gate-off.xml deleted file mode 100644 index efe909a6fa0..00000000000 --- a/acceptance/pipelines/test/arguments/out.gate-off.xml +++ /dev/null @@ -1,8 +0,0 @@ - - - - - pipelines test is experimental. Set DATABRICKS_CLI_PIPELINES_TEST=true to enable it - - - diff --git a/acceptance/pipelines/test/arguments/output.txt b/acceptance/pipelines/test/arguments/output.txt index eac823b2a59..6a5c7b13133 100644 --- a/acceptance/pipelines/test/arguments/output.txt +++ b/acceptance/pipelines/test/arguments/output.txt @@ -2,8 +2,6 @@ >>> [CLI] pipelines test --help Run the tests of a pipeline in a test-only update. -This command is experimental. Set DATABRICKS_CLI_PIPELINES_TEST=true to enable it. - KEY is the unique name of the pipeline, as defined in its YAML file. It is required when test selections are given. If there is only one pipeline in the project and no arguments are given, it is selected automatically. @@ -58,16 +56,6 @@ Global Flags: -t, --target string bundle target to use (if applicable) --var strings set values for variables defined in project config. Example: --var="foo=bar" -=== The command is experimental and off by default - ->>> musterr [CLI] pipelines test --pipeline-id 1234 -Error: pipelines test is experimental. Set DATABRICKS_CLI_PIPELINES_TEST=true to enable it - -=== With the command off, an older report is still replaced - ->>> musterr [CLI] pipelines test --pipeline-id 1234 --junit-xml out.gate-off.xml -Error: pipelines test is experimental. Set DATABRICKS_CLI_PIPELINES_TEST=true to enable it - === pytest arguments must be passed with --pytest-arg >>> musterr [CLI] pipelines test --pipeline-id 1234 -k orders diff --git a/acceptance/pipelines/test/arguments/script b/acceptance/pipelines/test/arguments/script index ceb54ec432e..a18c49b0d2d 100644 --- a/acceptance/pipelines/test/arguments/script +++ b/acceptance/pipelines/test/arguments/script @@ -1,12 +1,5 @@ trace $CLI pipelines test --help -title "The command is experimental and off by default" -(unset DATABRICKS_CLI_PIPELINES_TEST; trace musterr $CLI pipelines test --pipeline-id 1234) - -title "With the command off, an older report is still replaced" -echo '' > out.gate-off.xml -(unset DATABRICKS_CLI_PIPELINES_TEST; trace musterr $CLI pipelines test --pipeline-id 1234 --junit-xml out.gate-off.xml) - title "pytest arguments must be passed with --pytest-arg" trace musterr $CLI pipelines test --pipeline-id 1234 -k orders trace musterr $CLI pipelines test --pipeline-id 1234 -- -k orders diff --git a/acceptance/pipelines/test/immutable/output.txt b/acceptance/pipelines/test/immutable/output.txt index 27c6cd2712e..5996cdd714c 100644 --- a/acceptance/pipelines/test/immutable/output.txt +++ b/acceptance/pipelines/test/immutable/output.txt @@ -50,6 +50,7 @@ Update: [UUID][2] "method": "POST", "path": "/api/2.0/pipelines/[UUID][0]/updates", "body": { - "test_only": true + "test_only": true, + "test_options": {} } } diff --git a/acceptance/pipelines/test/not-enabled/out.not-enabled.xml b/acceptance/pipelines/test/not-enabled/out.not-enabled.xml new file mode 100644 index 00000000000..2e3314a1f21 --- /dev/null +++ b/acceptance/pipelines/test/not-enabled/out.not-enabled.xml @@ -0,0 +1,8 @@ + + + + + Pipeline testing is not enabled for this pipeline. Contact your Databricks account team to join the preview. + + + diff --git a/acceptance/pipelines/test/not-enabled/out.test.toml b/acceptance/pipelines/test/not-enabled/out.test.toml new file mode 100644 index 00000000000..18b1a88417e --- /dev/null +++ b/acceptance/pipelines/test/not-enabled/out.test.toml @@ -0,0 +1 @@ +Cloud = false diff --git a/acceptance/pipelines/test/not-enabled/output.txt b/acceptance/pipelines/test/not-enabled/output.txt new file mode 100644 index 00000000000..da81ba5cd34 --- /dev/null +++ b/acceptance/pipelines/test/not-enabled/output.txt @@ -0,0 +1,15 @@ + +=== Where the testing API is off, a run of all tests is rejected, not run as an older test-only update + +>>> musterr [CLI] pipelines test --pipeline-id [UUID] --junit-xml out.not-enabled.xml +Error: Pipeline testing is not enabled for this pipeline. Contact your Databricks account team to join the preview. + +>>> print_requests.py //updates +{ + "method": "POST", + "path": "/api/2.0/pipelines/[UUID]/updates", + "body": { + "test_only": true, + "test_options": {} + } +} diff --git a/acceptance/pipelines/test/not-enabled/script b/acceptance/pipelines/test/not-enabled/script new file mode 100644 index 00000000000..053924a20d4 --- /dev/null +++ b/acceptance/pipelines/test/not-enabled/script @@ -0,0 +1,6 @@ +pipeline_id=$($CLI pipelines create --json '{"name": "orders", "root_path": "/Workspace/Users/someone@example.com/orders"}' | jq -r .pipeline_id) +rm "$OUT_REQUESTS" + +title "Where the testing API is off, a run of all tests is rejected, not run as an older test-only update" +trace musterr $CLI pipelines test --pipeline-id $pipeline_id --junit-xml out.not-enabled.xml +trace print_requests.py //updates diff --git a/acceptance/pipelines/test/not-enabled/test.toml b/acceptance/pipelines/test/not-enabled/test.toml new file mode 100644 index 00000000000..ce53b5dd9fb --- /dev/null +++ b/acceptance/pipelines/test/not-enabled/test.toml @@ -0,0 +1,11 @@ +# The service rejects a request with test_options where the testing API is off. The test server +# does not model that per-workspace setting, so the response is stubbed. +[[Server]] +Pattern = "POST /api/2.0/pipelines/{pipeline_id}/updates" +Response.Body = ''' +{ + "error_code": "INVALID_PARAMETER_VALUE", + "message": "Pipeline testing is not enabled for this pipeline. Contact your Databricks account team to join the preview." +} +''' +Response.StatusCode = 400 diff --git a/acceptance/pipelines/test/selection/output.txt b/acceptance/pipelines/test/selection/output.txt index a4fab81a602..d989fede538 100644 --- a/acceptance/pipelines/test/selection/output.txt +++ b/acceptance/pipelines/test/selection/output.txt @@ -21,7 +21,8 @@ Update: [UUID][1] "method": "POST", "path": "/api/2.0/pipelines/[UUID][0]/updates", "body": { - "test_only": true + "test_only": true, + "test_options": {} } } diff --git a/acceptance/pipelines/test/submit-error/output.txt b/acceptance/pipelines/test/submit-error/output.txt index 2bc638a6fcc..2951c9b22c2 100644 --- a/acceptance/pipelines/test/submit-error/output.txt +++ b/acceptance/pipelines/test/submit-error/output.txt @@ -10,6 +10,7 @@ Wait for it to finish, or pass its update ID to --resume to read its results "method": "POST", "path": "/api/2.0/pipelines/[UUID][0]/updates", "body": { - "test_only": true + "test_only": true, + "test_options": {} } } diff --git a/acceptance/pipelines/test/test.toml b/acceptance/pipelines/test/test.toml index 66f23523071..4a1a3bb8f55 100644 --- a/acceptance/pipelines/test/test.toml +++ b/acceptance/pipelines/test/test.toml @@ -1,4 +1,2 @@ RecordRequests = true Ignore = ['.databricks'] -# The command is experimental; the gate case in arguments turns it off. -Env.DATABRICKS_CLI_PIPELINES_TEST = "true" diff --git a/cmd/pipelines/test.go b/cmd/pipelines/test.go index 9e3e9aa1426..b48998722f1 100644 --- a/cmd/pipelines/test.go +++ b/cmd/pipelines/test.go @@ -25,7 +25,6 @@ import ( "github.com/databricks/cli/cmd/root" "github.com/databricks/cli/libs/cmdctx" "github.com/databricks/cli/libs/cmdgroup" - "github.com/databricks/cli/libs/env" "github.com/databricks/cli/libs/flags" "github.com/databricks/cli/libs/logdiag" "github.com/databricks/cli/libs/shellquote" @@ -36,18 +35,16 @@ import ( "github.com/spf13/pflag" ) -// newStartTestUpdateRequest returns the Start Update request of a test run. +// newStartTestUpdateRequest returns the Start Update request of a test run. It always sends +// test_options, empty to run every test: only a request with test_options is rejected where the +// testing API is off; without them, the service runs its older test-only update instead. func newStartTestUpdateRequest(selection, pytestArgs []string) pipelines.StartUpdate { - req := pipelines.StartUpdate{TestOnly: true} - if len(selection) > 0 || len(pytestArgs) > 0 { - req.TestOptions = &pipelines.PipelineTestOptions{TestSelection: selection, PytestArgs: pytestArgs} + return pipelines.StartUpdate{ + TestOnly: true, + TestOptions: &pipelines.PipelineTestOptions{TestSelection: selection, PytestArgs: pytestArgs}, } - return req } -// pipelinesTestEnvVar enables the experimental pipelines test command. -const pipelinesTestEnvVar = "DATABRICKS_CLI_PIPELINES_TEST" - const pytestArgHint = "pass pytest arguments with --pytest-arg, for example --pytest-arg=-k --pytest-arg='orders and not slow'" // validatePytestArgs accepts only -k and -m, each followed by its value in the next argument, as the API @@ -542,9 +539,6 @@ type testFlags struct { // checkTestFlags rejects the flag combinations that pipelines test does not support, before any API call. func checkTestFlags(cmd *cobra.Command, args []string, f testFlags) error { - if enabled, _ := env.GetBool(cmd.Context(), pipelinesTestEnvVar); !enabled { - return fmt.Errorf("pipelines test is experimental. Set %s=true to enable it", pipelinesTestEnvVar) - } if cmd.ArgsLenAtDash() >= 0 { return errors.New("the -- separator is not supported: " + pytestArgHint) } @@ -617,8 +611,6 @@ func testCommand() *cobra.Command { Short: "*Experimental* Run the tests of a pipeline", Long: `Run the tests of a pipeline in a test-only update. -This command is experimental. Set DATABRICKS_CLI_PIPELINES_TEST=true to enable it. - KEY is the unique name of the pipeline, as defined in its YAML file. It is required when test selections are given. If there is only one pipeline in the project and no arguments are given, it is selected automatically. diff --git a/cmd/pipelines/test_test.go b/cmd/pipelines/test_test.go index 51553b161bd..c3be41bb9f9 100644 --- a/cmd/pipelines/test_test.go +++ b/cmd/pipelines/test_test.go @@ -3,6 +3,7 @@ package pipelines import ( "bytes" "context" + "encoding/json" "errors" "net/http" "net/http/httptest" @@ -81,7 +82,12 @@ func TestResolveTestSelections(t *testing.T) { } func TestNewStartTestUpdateRequest(t *testing.T) { - assert.Equal(t, pipelines.StartUpdate{TestOnly: true}, newStartTestUpdateRequest(nil, nil)) + all := newStartTestUpdateRequest(nil, nil) + assert.Equal(t, pipelines.StartUpdate{TestOnly: true, TestOptions: &pipelines.PipelineTestOptions{}}, all) + // An empty test_options must reach the service, where it is rejected if the testing API is off. + body, err := json.Marshal(all) + require.NoError(t, err) + assert.JSONEq(t, `{"test_only": true, "test_options": {}}`, string(body)) assert.Equal(t, pipelines.StartUpdate{ TestOnly: true, TestOptions: &pipelines.PipelineTestOptions{TestSelection: []string{"/Workspace/tests"}, PytestArgs: []string{"-k", "orders"}}, From ef068afb1d2859bd8866571fa74419f90ecf853d Mon Sep 17 00:00:00 2001 From: Bhukya Date: Tue, 17 Nov 2026 15:43:49 +0000 Subject: [PATCH 2/3] Drop the Experimental label from the pipelines test help The command no longer needs an environment variable to run. Co-authored-by: Isaac --- acceptance/pipelines/databricks-cli-help/output.txt | 2 +- cmd/pipelines/test.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/acceptance/pipelines/databricks-cli-help/output.txt b/acceptance/pipelines/databricks-cli-help/output.txt index a4ddbc1288b..ee5a979b8aa 100644 --- a/acceptance/pipelines/databricks-cli-help/output.txt +++ b/acceptance/pipelines/databricks-cli-help/output.txt @@ -30,7 +30,7 @@ Main Commands run Run a pipeline show Preview a table's columns and sample rows stop Stop a pipeline - test *Experimental* Run the tests of a pipeline + test Run the tests of a pipeline Management Commands apply-environment *Public Preview* Apply the latest environment to the pipeline. diff --git a/cmd/pipelines/test.go b/cmd/pipelines/test.go index b48998722f1..e24233e165c 100644 --- a/cmd/pipelines/test.go +++ b/cmd/pipelines/test.go @@ -608,7 +608,7 @@ func notPipelineKeyError(cmd *cobra.Command, b *bundle.Bundle, key string, args func testCommand() *cobra.Command { cmd := &cobra.Command{ Use: "test [flags] [KEY] [TEST_SELECTION...]", - Short: "*Experimental* Run the tests of a pipeline", + Short: "Run the tests of a pipeline", Long: `Run the tests of a pipeline in a test-only update. KEY is the unique name of the pipeline, as defined in its YAML file. From 58184453dbe333684f0eb8940f95fe83631b2ad9 Mon Sep 17 00:00:00 2001 From: Bhukya Date: Wed, 18 Nov 2026 07:40:35 +0000 Subject: [PATCH 3/3] Mark pipelines test as Beta in its help Co-authored-by: Isaac --- acceptance/pipelines/databricks-cli-help/output.txt | 2 +- cmd/pipelines/test.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/acceptance/pipelines/databricks-cli-help/output.txt b/acceptance/pipelines/databricks-cli-help/output.txt index ee5a979b8aa..f9d056eec5b 100644 --- a/acceptance/pipelines/databricks-cli-help/output.txt +++ b/acceptance/pipelines/databricks-cli-help/output.txt @@ -30,7 +30,7 @@ Main Commands run Run a pipeline show Preview a table's columns and sample rows stop Stop a pipeline - test Run the tests of a pipeline + test *Beta* Run the tests of a pipeline Management Commands apply-environment *Public Preview* Apply the latest environment to the pipeline. diff --git a/cmd/pipelines/test.go b/cmd/pipelines/test.go index e24233e165c..685a89f27b0 100644 --- a/cmd/pipelines/test.go +++ b/cmd/pipelines/test.go @@ -608,7 +608,7 @@ func notPipelineKeyError(cmd *cobra.Command, b *bundle.Bundle, key string, args func testCommand() *cobra.Command { cmd := &cobra.Command{ Use: "test [flags] [KEY] [TEST_SELECTION...]", - Short: "Run the tests of a pipeline", + Short: "*Beta* Run the tests of a pipeline", Long: `Run the tests of a pipeline in a test-only update. KEY is the unique name of the pipeline, as defined in its YAML file.