diff --git a/.nextchanges/cli/ssh-connect-cpu-env-5.md b/.nextchanges/cli/ssh-connect-cpu-env-5.md new file mode 100644 index 00000000000..22f16f8d2f0 --- /dev/null +++ b/.nextchanges/cli/ssh-connect-cpu-env-5.md @@ -0,0 +1 @@ +* `databricks ssh connect` to serverless CPU compute now uses serverless environment version 5 by default, so the first connection after upgrading starts a new server; pass `--environment-version 4` to keep the old environment. Serverless GPU (`--accelerator`) is unchanged and stays on version 4. ([#7847](https://github.com/databricks/cli/pull/7847)) diff --git a/acceptance/ssh/connect-serverless-cpu/output.txt b/acceptance/ssh/connect-serverless-cpu/output.txt index 1a2da09b42c..b0742a5f61e 100644 --- a/acceptance/ssh/connect-serverless-cpu/output.txt +++ b/acceptance/ssh/connect-serverless-cpu/output.txt @@ -10,7 +10,7 @@ { "environment_key": "ssh_tunnel_serverless", "spec": { - "environment_version": "4" + "environment_version": "5" } } ], diff --git a/experimental/ssh/cmd/connect.go b/experimental/ssh/cmd/connect.go index d2040ad760f..866bb4ef802 100644 --- a/experimental/ssh/cmd/connect.go +++ b/experimental/ssh/cmd/connect.go @@ -1,6 +1,7 @@ package ssh import ( + "fmt" "time" "github.com/databricks/cli/cmd/root" @@ -36,7 +37,7 @@ Connect to serverless: databricks ssh connect databricks ssh connect --accelerator= # AI Runtime databricks ssh connect --base-environment= # custom base environment - databricks ssh connect --environment-version=5 # serverless environment version + databricks ssh connect --environment-version=6 # serverless environment version Connect to a dedicated cluster: databricks ssh connect --cluster=`, @@ -94,7 +95,12 @@ Connect to a dedicated cluster: cmd.Flags().BoolVar(&skipSettingsCheck, "skip-settings-check", false, "Skip checking and updating IDE settings") cmd.Flags().MarkHidden("skip-settings-check") - cmd.Flags().IntVar(&environmentVersion, "environment-version", client.DefaultEnvironmentVersion, "Serverless environment version for the SSH server job (serverless only)") + // 0 means not set: the default then depends on --accelerator, Validate can detect a real + // --environment-version + --base-environment conflict, and ToProxyCommand only persists an + // explicit version. + cmd.Flags().IntVar(&environmentVersion, "environment-version", 0, fmt.Sprintf( + "Serverless environment version for the SSH server job (serverless only; defaults to %d for CPU and %d with --accelerator)", + client.DefaultCPUEnvironmentVersion, client.DefaultGPUEnvironmentVersion)) cmd.Flags().StringVar(&baseEnvironment, "base-environment", "", "Custom base environment for serverless compute: an env.yaml path, a workspace-base-environments resource ID, or a display name") @@ -114,17 +120,9 @@ Connect to a dedicated cluster: cmd.RunE = func(cmd *cobra.Command, args []string) error { ctx := cmd.Context() wsClient := cmdctx.WorkspaceClient(ctx) - // Generate the name before resetting the version so that the default version - // maps to the same name whether it is set explicitly or implicitly. if connectionName == "" && clusterID == "" && !proxyMode { connectionName = client.GenerateDefaultConnectionName(wsClient.Config.Host, accelerator, baseEnvironment, environmentVersion) } - // Only carry an explicitly-set environment version. Leaving it at 0 otherwise - // lets the submit path default to DefaultEnvironmentVersion and lets Validate - // detect a real --environment-version + --base-environment conflict. - if !cmd.Flags().Changed("environment-version") { - environmentVersion = 0 - } // Serverless GPU compute can take much longer to provision than CPU compute, // so allow extra time for the SSH server job to start. startupTimeout := taskStartupTimeout diff --git a/experimental/ssh/cmd/connect_test.go b/experimental/ssh/cmd/connect_test.go new file mode 100644 index 00000000000..e493c77e646 --- /dev/null +++ b/experimental/ssh/cmd/connect_test.go @@ -0,0 +1,47 @@ +package ssh + +import ( + "strings" + "testing" + + "github.com/databricks/cli/experimental/ssh/internal/client" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// An IDE connection persists its ProxyCommand, and every later connection re-parses it with +// `ssh connect`. A version the user did not choose must come back as 0 so it resolves to the +// default of the compute type that --accelerator names, rather than to a version baked into +// the ProxyCommand. +func TestServerlessProxyCommandRoundTripsEnvironmentVersion(t *testing.T) { + tests := []struct { + name string + opts client.ClientOptions + }{ + {name: "CPU default", opts: client.ClientOptions{ConnectionName: "conn"}}, + {name: "GPU default", opts: client.ClientOptions{ConnectionName: "conn", Accelerator: "GPU_1xA10"}}, + {name: "CPU explicit", opts: client.ClientOptions{ConnectionName: "conn", EnvironmentVersion: 4}}, + {name: "GPU explicit", opts: client.ClientOptions{ConnectionName: "conn", Accelerator: "GPU_1xA10", EnvironmentVersion: 5}}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + proxyCommand, err := tt.opts.ToProxyCommand() + require.NoError(t, err) + + _, args, found := strings.Cut(proxyCommand, " ssh connect ") + require.True(t, found, "proxy command %q must invoke 'ssh connect'", proxyCommand) + + flags := newConnectCommand().Flags() + require.NoError(t, flags.Parse(strings.Fields(args))) + + accelerator, err := flags.GetString("accelerator") + require.NoError(t, err) + assert.Equal(t, tt.opts.Accelerator, accelerator) + + environmentVersion, err := flags.GetInt("environment-version") + require.NoError(t, err) + assert.Equal(t, tt.opts.EnvironmentVersion, environmentVersion) + }) + } +} diff --git a/experimental/ssh/internal/client/client.go b/experimental/ssh/internal/client/client.go index 5afd212131f..638330e90df 100644 --- a/experimental/ssh/internal/client/client.go +++ b/experimental/ssh/internal/client/client.go @@ -64,8 +64,12 @@ var errDriverProxyForbidden = errors.New("the workspace refused the SSH tunnel c var connectionNameRegex = regexp.MustCompile(`^[a-zA-Z0-9][a-zA-Z0-9_-]*$`) -// DefaultEnvironmentVersion is the serverless environment version used when --environment-version is not set. -const DefaultEnvironmentVersion = 4 +// Serverless environment versions used when --environment-version is not set. GPU stays on 4 +// because the standard version 5 and 6 environments ship no torch on GPU. +const ( + DefaultCPUEnvironmentVersion = 5 + DefaultGPUEnvironmentVersion = 4 +) const ( sshServerTaskKey = "start_ssh_server" @@ -213,9 +217,11 @@ func (o *ClientOptions) Validate() error { // known_hosts conflicts). The environment is folded into the hash because a // serverless server bakes in its environment at startup: distinct environments // must map to distinct connection names so they don't reuse each other's server. +// An environmentVersion of 0 stands for the default of the compute type. func GenerateDefaultConnectionName(host, accelerator, baseEnvironment string, environmentVersion int) string { // The version is hashed even when it is the default, so bumping - // DefaultEnvironmentVersion intentionally moves users to a new server. + // DefaultCPUEnvironmentVersion or DefaultGPUEnvironmentVersion intentionally + // moves users of that compute type to a new server. var hashInput string switch { case baseEnvironment != "": @@ -223,7 +229,8 @@ func GenerateDefaultConnectionName(host, accelerator, baseEnvironment string, en default: // The key keeps a version from hashing like a base environment // whose name is that number. - hashInput = host + "\x00environment_version=" + strconv.Itoa(environmentVersion) + version := cmp.Or(environmentVersion, defaultEnvironmentVersion(accelerator)) + hashInput = host + "\x00environment_version=" + strconv.Itoa(version) } h := md5.Sum([]byte(hashInput)) hashStr := hex.EncodeToString(h[:4]) @@ -245,7 +252,16 @@ func (o *ClientOptions) submittedEnvironmentVersion() int { if !o.IsServerlessMode() || o.BaseEnvironment != "" { return 0 } - return cmp.Or(o.EnvironmentVersion, DefaultEnvironmentVersion) + return cmp.Or(o.EnvironmentVersion, defaultEnvironmentVersion(o.Accelerator)) +} + +// defaultEnvironmentVersion returns the serverless environment version used when +// --environment-version is not set. +func defaultEnvironmentVersion(accelerator string) int { + if accelerator != "" { + return DefaultGPUEnvironmentVersion + } + return DefaultCPUEnvironmentVersion } // SessionIdentifier returns the unique identifier for the session. diff --git a/experimental/ssh/internal/client/client_internal_test.go b/experimental/ssh/internal/client/client_internal_test.go index 262859455b6..8b4d62cfbdb 100644 --- a/experimental/ssh/internal/client/client_internal_test.go +++ b/experimental/ssh/internal/client/client_internal_test.go @@ -489,6 +489,16 @@ func TestBuildSshTunnelEvent(t *testing.T) { EnvironmentVersion: 4, }, }, + { + name: "serverless GPU in proxy mode", + opts: ClientOptions{ConnectionName: "my-conn", Accelerator: "GPU_1xA10", ProxyMode: true}, + want: protos.SshTunnelEvent{ + ComputeType: protos.SshTunnelComputeTypeServerless, + AcceleratorType: "GPU_1xA10", + ClientMode: protos.SshTunnelClientModeProxy, + EnvironmentVersion: 4, + }, + }, { name: "proxy mode takes precedence over IDE", opts: ClientOptions{ConnectionName: "my-conn", ProxyMode: true, IDE: "vscode"}, @@ -496,7 +506,7 @@ func TestBuildSshTunnelEvent(t *testing.T) { ComputeType: protos.SshTunnelComputeTypeServerless, IdeType: "vscode", ClientMode: protos.SshTunnelClientModeProxy, - EnvironmentVersion: 4, + EnvironmentVersion: 5, }, }, { @@ -506,7 +516,7 @@ func TestBuildSshTunnelEvent(t *testing.T) { ComputeType: protos.SshTunnelComputeTypeServerless, IdeType: "cursor", ClientMode: protos.SshTunnelClientModeIDE, - EnvironmentVersion: 4, + EnvironmentVersion: 5, }, }, { diff --git a/experimental/ssh/internal/client/client_test.go b/experimental/ssh/internal/client/client_test.go index 20bc56d0115..e64e25093a2 100644 --- a/experimental/ssh/internal/client/client_test.go +++ b/experimental/ssh/internal/client/client_test.go @@ -230,22 +230,53 @@ func TestValidateServerLifecycle(t *testing.T) { func TestGenerateDefaultConnectionName(t *testing.T) { tests := []struct { - name string - host string - accelerator string - want string + name string + host string + accelerator string + environmentVersion int + want string }{ { name: "no accelerator", host: "https://my-workspace.cloud.databricks.com", - want: "databricks-cpu-2db656b0", + want: "databricks-cpu-3be751d8", }, { + name: "no accelerator with the CPU default set explicitly", + host: "https://my-workspace.cloud.databricks.com", + environmentVersion: client.DefaultCPUEnvironmentVersion, + want: "databricks-cpu-3be751d8", + }, + { + // The CPU default name from before the default moved to 5, so + // --environment-version 4 still reaches a server started back then. + name: "no accelerator with environment version 4", + host: "https://my-workspace.cloud.databricks.com", + environmentVersion: 4, + want: "databricks-cpu-2db656b0", + }, + { + // GPU names must stay byte-for-byte what they were before the CPU default + // moved, or GPU users would be moved to a new server for no reason. name: "GPU_1xA10 accelerator", host: "https://my-workspace.cloud.databricks.com", accelerator: "GPU_1xA10", want: "databricks-gpu-1xa10-2db656b0", }, + { + name: "GPU_1xA10 accelerator with the GPU default set explicitly", + host: "https://my-workspace.cloud.databricks.com", + accelerator: "GPU_1xA10", + environmentVersion: client.DefaultGPUEnvironmentVersion, + want: "databricks-gpu-1xa10-2db656b0", + }, + { + name: "GPU_1xA10 accelerator with environment version 5", + host: "https://my-workspace.cloud.databricks.com", + accelerator: "GPU_1xA10", + environmentVersion: 5, + want: "databricks-gpu-1xa10-3be751d8", + }, { name: "GPU_8xH100 accelerator", host: "https://my-workspace.cloud.databricks.com", @@ -255,18 +286,18 @@ func TestGenerateDefaultConnectionName(t *testing.T) { { name: "different host produces different name", host: "https://other-workspace.cloud.databricks.com", - want: "databricks-cpu-ebd5d001", + want: "databricks-cpu-9698f25b", }, { name: "deterministic for same input", host: "https://my-workspace.cloud.databricks.com", - want: "databricks-cpu-2db656b0", + want: "databricks-cpu-3be751d8", }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - got := client.GenerateDefaultConnectionName(tt.host, tt.accelerator, "", client.DefaultEnvironmentVersion) + got := client.GenerateDefaultConnectionName(tt.host, tt.accelerator, "", tt.environmentVersion) assert.Equal(t, tt.want, got) }) } @@ -276,25 +307,26 @@ func TestGenerateDefaultConnectionName(t *testing.T) { // ignored when an existing server for the default name is reused). t.Run("base environment differentiates the name", func(t *testing.T) { const host = "https://my-workspace.cloud.databricks.com" - base := client.GenerateDefaultConnectionName(host, "", "", client.DefaultEnvironmentVersion) - withEnv := client.GenerateDefaultConnectionName(host, "", "my-env", client.DefaultEnvironmentVersion) - otherEnv := client.GenerateDefaultConnectionName(host, "", "other-env", client.DefaultEnvironmentVersion) + base := client.GenerateDefaultConnectionName(host, "", "", 0) + withEnv := client.GenerateDefaultConnectionName(host, "", "my-env", 0) + otherEnv := client.GenerateDefaultConnectionName(host, "", "other-env", 0) assert.NotEqual(t, base, withEnv, "setting --base-environment must change the default name") assert.NotEqual(t, withEnv, otherEnv, "different environments must produce different names") - assert.Equal(t, withEnv, client.GenerateDefaultConnectionName(host, "", "my-env", client.DefaultEnvironmentVersion), "must be deterministic") + assert.Equal(t, withEnv, client.GenerateDefaultConnectionName(host, "", "my-env", 0), "must be deterministic") }) // Same for an explicit environment version: without it, a connection asking for - // version 5 would reuse a running server started on the default version. + // version 6 would reuse a running server started on the default version. t.Run("environment version differentiates the name", func(t *testing.T) { const host = "https://my-workspace.cloud.databricks.com" - base := client.GenerateDefaultConnectionName(host, "", "", client.DefaultEnvironmentVersion) - v5 := client.GenerateDefaultConnectionName(host, "", "", 5) + base := client.GenerateDefaultConnectionName(host, "", "", 0) + v4 := client.GenerateDefaultConnectionName(host, "", "", 4) v6 := client.GenerateDefaultConnectionName(host, "", "", 6) - assert.NotEqual(t, base, v5, "setting --environment-version must change the default name") - assert.NotEqual(t, v5, v6, "different versions must produce different names") - assert.NotEqual(t, v5, client.GenerateDefaultConnectionName(host, "", "5", client.DefaultEnvironmentVersion), "a version must not collide with a base environment named like it") - assert.Equal(t, v5, client.GenerateDefaultConnectionName(host, "", "", 5), "must be deterministic") + assert.NotEqual(t, base, v4, "a non-default --environment-version must change the default name") + assert.NotEqual(t, base, v6, "a non-default --environment-version must change the default name") + assert.NotEqual(t, v4, v6, "different versions must produce different names") + assert.NotEqual(t, v6, client.GenerateDefaultConnectionName(host, "", "6", 0), "a version must not collide with a base environment named like it") + assert.Equal(t, v6, client.GenerateDefaultConnectionName(host, "", "", 6), "must be deterministic") }) t.Run("base environment takes precedence over environment version", func(t *testing.T) { @@ -318,7 +350,7 @@ func TestGenerateDefaultConnectionNameMatchesRegex(t *testing.T) { for _, host := range hosts { for _, acc := range accelerators { for _, env := range environments { - name := client.GenerateDefaultConnectionName(host, acc, env, client.DefaultEnvironmentVersion) + name := client.GenerateDefaultConnectionName(host, acc, env, 0) assert.Regexp(t, nameRegex, name, "host=%q accelerator=%q environment=%q name=%q", host, acc, env, name) } } diff --git a/experimental/ssh/internal/client/submit_internal_test.go b/experimental/ssh/internal/client/submit_internal_test.go index f56043bfbb7..ff377d33333 100644 --- a/experimental/ssh/internal/client/submit_internal_test.go +++ b/experimental/ssh/internal/client/submit_internal_test.go @@ -35,17 +35,32 @@ func TestBuildSSHServerSubmitRun(t *testing.T) { t.Run("serverless environment version", func(t *testing.T) { for _, tt := range []struct { - version int - want string + name string + accelerator string + version int + proxyMode bool + want string }{ - {0, "4"}, - {5, "5"}, + {name: "CPU default", want: "5"}, + {name: "CPU explicit", version: 4, want: "4"}, + {name: "GPU default", accelerator: "GPU_1xA10", want: "4"}, + {name: "GPU explicit", accelerator: "GPU_1xA10", version: 5, want: "5"}, + // A persisted IDE ProxyCommand carries --accelerator but no --environment-version. + {name: "GPU default in proxy mode", accelerator: "GPU_1xA10", proxyMode: true, want: "4"}, } { - opts := ClientOptions{ConnectionName: "conn", ServerTimeout: time.Hour, EnvironmentVersion: tt.version} - got := buildSSHServerSubmitRun("v1", "scope", notebookPath, "", opts) - - require.Len(t, got.Environments, 1) - assert.Equal(t, tt.want, got.Environments[0].Spec.EnvironmentVersion) + t.Run(tt.name, func(t *testing.T) { + opts := ClientOptions{ + ConnectionName: "conn", + Accelerator: tt.accelerator, + ProxyMode: tt.proxyMode, + ServerTimeout: time.Hour, + EnvironmentVersion: tt.version, + } + got := buildSSHServerSubmitRun("v1", "scope", notebookPath, "", opts) + + require.Len(t, got.Environments, 1) + assert.Equal(t, tt.want, got.Environments[0].Spec.EnvironmentVersion) + }) } })