diff --git a/acceptance/pipelines/databricks-cli-help/output.txt b/acceptance/pipelines/databricks-cli-help/output.txt
index a4ddbc1288b..f9d056eec5b 100644
--- a/acceptance/pipelines/databricks-cli-help/output.txt
+++ b/acceptance/pipelines/databricks-cli-help/output.txt
@@ -30,7 +30,7 @@ Main Commands
run Run a pipeline
show Preview a table's columns and sample rows
stop Stop a pipeline
- test *Experimental* Run the tests of a pipeline
+ test *Beta* Run the tests of a pipeline
Management Commands
apply-environment *Public Preview* Apply the latest environment to the pipeline.
diff --git a/acceptance/pipelines/test/arguments/out.gate-off.xml b/acceptance/pipelines/test/arguments/out.gate-off.xml
deleted file mode 100644
index efe909a6fa0..00000000000
--- a/acceptance/pipelines/test/arguments/out.gate-off.xml
+++ /dev/null
@@ -1,8 +0,0 @@
-
-
-
-
- pipelines test is experimental. Set DATABRICKS_CLI_PIPELINES_TEST=true to enable it
-
-
-
diff --git a/acceptance/pipelines/test/arguments/output.txt b/acceptance/pipelines/test/arguments/output.txt
index eac823b2a59..6a5c7b13133 100644
--- a/acceptance/pipelines/test/arguments/output.txt
+++ b/acceptance/pipelines/test/arguments/output.txt
@@ -2,8 +2,6 @@
>>> [CLI] pipelines test --help
Run the tests of a pipeline in a test-only update.
-This command is experimental. Set DATABRICKS_CLI_PIPELINES_TEST=true to enable it.
-
KEY is the unique name of the pipeline, as defined in its YAML file.
It is required when test selections are given. If there is only one pipeline
in the project and no arguments are given, it is selected automatically.
@@ -58,16 +56,6 @@ Global Flags:
-t, --target string bundle target to use (if applicable)
--var strings set values for variables defined in project config. Example: --var="foo=bar"
-=== The command is experimental and off by default
-
->>> musterr [CLI] pipelines test --pipeline-id 1234
-Error: pipelines test is experimental. Set DATABRICKS_CLI_PIPELINES_TEST=true to enable it
-
-=== With the command off, an older report is still replaced
-
->>> musterr [CLI] pipelines test --pipeline-id 1234 --junit-xml out.gate-off.xml
-Error: pipelines test is experimental. Set DATABRICKS_CLI_PIPELINES_TEST=true to enable it
-
=== pytest arguments must be passed with --pytest-arg
>>> musterr [CLI] pipelines test --pipeline-id 1234 -k orders
diff --git a/acceptance/pipelines/test/arguments/script b/acceptance/pipelines/test/arguments/script
index ceb54ec432e..a18c49b0d2d 100644
--- a/acceptance/pipelines/test/arguments/script
+++ b/acceptance/pipelines/test/arguments/script
@@ -1,12 +1,5 @@
trace $CLI pipelines test --help
-title "The command is experimental and off by default"
-(unset DATABRICKS_CLI_PIPELINES_TEST; trace musterr $CLI pipelines test --pipeline-id 1234)
-
-title "With the command off, an older report is still replaced"
-echo '' > out.gate-off.xml
-(unset DATABRICKS_CLI_PIPELINES_TEST; trace musterr $CLI pipelines test --pipeline-id 1234 --junit-xml out.gate-off.xml)
-
title "pytest arguments must be passed with --pytest-arg"
trace musterr $CLI pipelines test --pipeline-id 1234 -k orders
trace musterr $CLI pipelines test --pipeline-id 1234 -- -k orders
diff --git a/acceptance/pipelines/test/immutable/output.txt b/acceptance/pipelines/test/immutable/output.txt
index 27c6cd2712e..5996cdd714c 100644
--- a/acceptance/pipelines/test/immutable/output.txt
+++ b/acceptance/pipelines/test/immutable/output.txt
@@ -50,6 +50,7 @@ Update: [UUID][2]
"method": "POST",
"path": "/api/2.0/pipelines/[UUID][0]/updates",
"body": {
- "test_only": true
+ "test_only": true,
+ "test_options": {}
}
}
diff --git a/acceptance/pipelines/test/not-enabled/out.not-enabled.xml b/acceptance/pipelines/test/not-enabled/out.not-enabled.xml
new file mode 100644
index 00000000000..2e3314a1f21
--- /dev/null
+++ b/acceptance/pipelines/test/not-enabled/out.not-enabled.xml
@@ -0,0 +1,8 @@
+
+
+
+
+ Pipeline testing is not enabled for this pipeline. Contact your Databricks account team to join the preview.
+
+
+
diff --git a/acceptance/pipelines/test/not-enabled/out.test.toml b/acceptance/pipelines/test/not-enabled/out.test.toml
new file mode 100644
index 00000000000..18b1a88417e
--- /dev/null
+++ b/acceptance/pipelines/test/not-enabled/out.test.toml
@@ -0,0 +1 @@
+Cloud = false
diff --git a/acceptance/pipelines/test/not-enabled/output.txt b/acceptance/pipelines/test/not-enabled/output.txt
new file mode 100644
index 00000000000..da81ba5cd34
--- /dev/null
+++ b/acceptance/pipelines/test/not-enabled/output.txt
@@ -0,0 +1,15 @@
+
+=== Where the testing API is off, a run of all tests is rejected, not run as an older test-only update
+
+>>> musterr [CLI] pipelines test --pipeline-id [UUID] --junit-xml out.not-enabled.xml
+Error: Pipeline testing is not enabled for this pipeline. Contact your Databricks account team to join the preview.
+
+>>> print_requests.py //updates
+{
+ "method": "POST",
+ "path": "/api/2.0/pipelines/[UUID]/updates",
+ "body": {
+ "test_only": true,
+ "test_options": {}
+ }
+}
diff --git a/acceptance/pipelines/test/not-enabled/script b/acceptance/pipelines/test/not-enabled/script
new file mode 100644
index 00000000000..053924a20d4
--- /dev/null
+++ b/acceptance/pipelines/test/not-enabled/script
@@ -0,0 +1,6 @@
+pipeline_id=$($CLI pipelines create --json '{"name": "orders", "root_path": "/Workspace/Users/someone@example.com/orders"}' | jq -r .pipeline_id)
+rm "$OUT_REQUESTS"
+
+title "Where the testing API is off, a run of all tests is rejected, not run as an older test-only update"
+trace musterr $CLI pipelines test --pipeline-id $pipeline_id --junit-xml out.not-enabled.xml
+trace print_requests.py //updates
diff --git a/acceptance/pipelines/test/not-enabled/test.toml b/acceptance/pipelines/test/not-enabled/test.toml
new file mode 100644
index 00000000000..ce53b5dd9fb
--- /dev/null
+++ b/acceptance/pipelines/test/not-enabled/test.toml
@@ -0,0 +1,11 @@
+# The service rejects a request with test_options where the testing API is off. The test server
+# does not model that per-workspace setting, so the response is stubbed.
+[[Server]]
+Pattern = "POST /api/2.0/pipelines/{pipeline_id}/updates"
+Response.Body = '''
+{
+ "error_code": "INVALID_PARAMETER_VALUE",
+ "message": "Pipeline testing is not enabled for this pipeline. Contact your Databricks account team to join the preview."
+}
+'''
+Response.StatusCode = 400
diff --git a/acceptance/pipelines/test/selection/output.txt b/acceptance/pipelines/test/selection/output.txt
index a4fab81a602..d989fede538 100644
--- a/acceptance/pipelines/test/selection/output.txt
+++ b/acceptance/pipelines/test/selection/output.txt
@@ -21,7 +21,8 @@ Update: [UUID][1]
"method": "POST",
"path": "/api/2.0/pipelines/[UUID][0]/updates",
"body": {
- "test_only": true
+ "test_only": true,
+ "test_options": {}
}
}
diff --git a/acceptance/pipelines/test/submit-error/output.txt b/acceptance/pipelines/test/submit-error/output.txt
index 2bc638a6fcc..2951c9b22c2 100644
--- a/acceptance/pipelines/test/submit-error/output.txt
+++ b/acceptance/pipelines/test/submit-error/output.txt
@@ -10,6 +10,7 @@ Wait for it to finish, or pass its update ID to --resume to read its results
"method": "POST",
"path": "/api/2.0/pipelines/[UUID][0]/updates",
"body": {
- "test_only": true
+ "test_only": true,
+ "test_options": {}
}
}
diff --git a/acceptance/pipelines/test/test.toml b/acceptance/pipelines/test/test.toml
index 66f23523071..4a1a3bb8f55 100644
--- a/acceptance/pipelines/test/test.toml
+++ b/acceptance/pipelines/test/test.toml
@@ -1,4 +1,2 @@
RecordRequests = true
Ignore = ['.databricks']
-# The command is experimental; the gate case in arguments turns it off.
-Env.DATABRICKS_CLI_PIPELINES_TEST = "true"
diff --git a/cmd/pipelines/test.go b/cmd/pipelines/test.go
index 9e3e9aa1426..685a89f27b0 100644
--- a/cmd/pipelines/test.go
+++ b/cmd/pipelines/test.go
@@ -25,7 +25,6 @@ import (
"github.com/databricks/cli/cmd/root"
"github.com/databricks/cli/libs/cmdctx"
"github.com/databricks/cli/libs/cmdgroup"
- "github.com/databricks/cli/libs/env"
"github.com/databricks/cli/libs/flags"
"github.com/databricks/cli/libs/logdiag"
"github.com/databricks/cli/libs/shellquote"
@@ -36,18 +35,16 @@ import (
"github.com/spf13/pflag"
)
-// newStartTestUpdateRequest returns the Start Update request of a test run.
+// newStartTestUpdateRequest returns the Start Update request of a test run. It always sends
+// test_options, empty to run every test: only a request with test_options is rejected where the
+// testing API is off; without them, the service runs its older test-only update instead.
func newStartTestUpdateRequest(selection, pytestArgs []string) pipelines.StartUpdate {
- req := pipelines.StartUpdate{TestOnly: true}
- if len(selection) > 0 || len(pytestArgs) > 0 {
- req.TestOptions = &pipelines.PipelineTestOptions{TestSelection: selection, PytestArgs: pytestArgs}
+ return pipelines.StartUpdate{
+ TestOnly: true,
+ TestOptions: &pipelines.PipelineTestOptions{TestSelection: selection, PytestArgs: pytestArgs},
}
- return req
}
-// pipelinesTestEnvVar enables the experimental pipelines test command.
-const pipelinesTestEnvVar = "DATABRICKS_CLI_PIPELINES_TEST"
-
const pytestArgHint = "pass pytest arguments with --pytest-arg, for example --pytest-arg=-k --pytest-arg='orders and not slow'"
// validatePytestArgs accepts only -k and -m, each followed by its value in the next argument, as the API
@@ -542,9 +539,6 @@ type testFlags struct {
// checkTestFlags rejects the flag combinations that pipelines test does not support, before any API call.
func checkTestFlags(cmd *cobra.Command, args []string, f testFlags) error {
- if enabled, _ := env.GetBool(cmd.Context(), pipelinesTestEnvVar); !enabled {
- return fmt.Errorf("pipelines test is experimental. Set %s=true to enable it", pipelinesTestEnvVar)
- }
if cmd.ArgsLenAtDash() >= 0 {
return errors.New("the -- separator is not supported: " + pytestArgHint)
}
@@ -614,11 +608,9 @@ func notPipelineKeyError(cmd *cobra.Command, b *bundle.Bundle, key string, args
func testCommand() *cobra.Command {
cmd := &cobra.Command{
Use: "test [flags] [KEY] [TEST_SELECTION...]",
- Short: "*Experimental* Run the tests of a pipeline",
+ Short: "*Beta* Run the tests of a pipeline",
Long: `Run the tests of a pipeline in a test-only update.
-This command is experimental. Set DATABRICKS_CLI_PIPELINES_TEST=true to enable it.
-
KEY is the unique name of the pipeline, as defined in its YAML file.
It is required when test selections are given. If there is only one pipeline
in the project and no arguments are given, it is selected automatically.
diff --git a/cmd/pipelines/test_test.go b/cmd/pipelines/test_test.go
index 51553b161bd..c3be41bb9f9 100644
--- a/cmd/pipelines/test_test.go
+++ b/cmd/pipelines/test_test.go
@@ -3,6 +3,7 @@ package pipelines
import (
"bytes"
"context"
+ "encoding/json"
"errors"
"net/http"
"net/http/httptest"
@@ -81,7 +82,12 @@ func TestResolveTestSelections(t *testing.T) {
}
func TestNewStartTestUpdateRequest(t *testing.T) {
- assert.Equal(t, pipelines.StartUpdate{TestOnly: true}, newStartTestUpdateRequest(nil, nil))
+ all := newStartTestUpdateRequest(nil, nil)
+ assert.Equal(t, pipelines.StartUpdate{TestOnly: true, TestOptions: &pipelines.PipelineTestOptions{}}, all)
+ // An empty test_options must reach the service, where it is rejected if the testing API is off.
+ body, err := json.Marshal(all)
+ require.NoError(t, err)
+ assert.JSONEq(t, `{"test_only": true, "test_options": {}}`, string(body))
assert.Equal(t, pipelines.StartUpdate{
TestOnly: true,
TestOptions: &pipelines.PipelineTestOptions{TestSelection: []string{"/Workspace/tests"}, PytestArgs: []string{"-k", "orders"}},