diff --git a/bundle/config/workspace.go b/bundle/config/workspace.go index abe9f82b780..d8ea1306047 100644 --- a/bundle/config/workspace.go +++ b/bundle/config/workspace.go @@ -7,6 +7,7 @@ import ( "github.com/databricks/cli/bundle/env" "github.com/databricks/cli/libs/auth" "github.com/databricks/cli/libs/databrickscfg" + "github.com/databricks/cli/libs/httptransport" "github.com/databricks/cli/libs/liteswap" "github.com/databricks/databricks-sdk-go" "github.com/databricks/databricks-sdk-go/config" @@ -247,8 +248,9 @@ func (w *Workspace) Client(ctx context.Context) (*databricks.WorkspaceClient, er } } - // Set after EnsureResolved, so skip_verify from the profile is known. - cfg.HTTPTransport = workspaceTransport(cfg.InsecureSkipVerify) + // Set after EnsureResolved, so skip_verify from the profile is known. The token-refresh client + // the SDK builds while resolving the config therefore still uses the SDK default. + cfg.HTTPTransport = httptransport.HealthChecked(cfg.InsecureSkipVerify) return databricks.NewWorkspaceClient((*databricks.Config)(cfg)) } diff --git a/bundle/config/workspace_test.go b/bundle/config/workspace_test.go index 3fb86da2b64..bf5c66a9e50 100644 --- a/bundle/config/workspace_test.go +++ b/bundle/config/workspace_test.go @@ -9,6 +9,7 @@ import ( "github.com/databricks/cli/bundle/env" "github.com/databricks/cli/internal/testutil" "github.com/databricks/cli/libs/databrickscfg" + "github.com/databricks/cli/libs/httptransport" "github.com/databricks/databricks-sdk-go/config" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -321,3 +322,32 @@ func TestWorkspaceVerifyProfileForHost(t *testing.T) { assert.ErrorContains(t, err, "doesn’t match the host configured in the bundle") }) } + +func TestWorkspaceClientUsesHealthCheckedTransport(t *testing.T) { + setupWorkspaceTest(t) + + err := databrickscfg.SaveToProfile(t.Context(), &config.Config{ + Profile: "secure", + Host: "https://secure.cloud.databricks.test", + Token: "token", + }) + require.NoError(t, err) + // skip_verify comes from the profile, so it is only known after the config is resolved. + err = databrickscfg.SaveToProfile(t.Context(), &config.Config{ + Profile: "insecure", + Host: "https://insecure.cloud.databricks.test", + Token: "token", + InsecureSkipVerify: true, + }) + require.NoError(t, err) + + w := Workspace{Profile: "secure"} + client, err := w.Client(t.Context()) + require.NoError(t, err) + assert.Same(t, httptransport.HealthChecked(false), client.Config.HTTPTransport) + + w = Workspace{Profile: "insecure"} + client, err = w.Client(t.Context()) + require.NoError(t, err) + assert.Same(t, httptransport.HealthChecked(true), client.Config.HTTPTransport) +} diff --git a/integration/internal/acc/workspace.go b/integration/internal/acc/workspace.go index af1e67010a2..b6db9881871 100644 --- a/integration/internal/acc/workspace.go +++ b/integration/internal/acc/workspace.go @@ -5,6 +5,7 @@ import ( "sync" "github.com/databricks/cli/internal/testutil" + "github.com/databricks/cli/libs/httptransport" "github.com/databricks/databricks-sdk-go" "github.com/databricks/databricks-sdk-go/service/compute" "github.com/stretchr/testify/require" @@ -27,6 +28,9 @@ type WorkspaceT struct { // configure default credentials". A shared client authenticates once and // refreshes its token in the background. The SDK does not cache a failed // authentication, so the next call on the shared client tries again. +// +// Being shared, it also shares one connection pool for the whole test process, +// hence the health-checked transport (see libs/httptransport). var sharedWorkspaceClientState struct { mu sync.Mutex client *databricks.WorkspaceClient @@ -38,7 +42,10 @@ func sharedWorkspaceClient(t testutil.TestingT) *databricks.WorkspaceClient { defer s.mu.Unlock() if s.client == nil { - w, err := databricks.NewWorkspaceClient() + // Set before the config is resolved, so the token-refresh client uses it too. With a + // transport set, the SDK ignores skip_verify; the integration tests don't use it. + cfg := &databricks.Config{HTTPTransport: httptransport.HealthChecked(false)} + w, err := databricks.NewWorkspaceClient(cfg) require.NoError(t, err) s.client = w } diff --git a/bundle/config/transport.go b/libs/httptransport/transport.go similarity index 81% rename from bundle/config/transport.go rename to libs/httptransport/transport.go index d637bcf6396..3a72c61acec 100644 --- a/bundle/config/transport.go +++ b/libs/httptransport/transport.go @@ -1,4 +1,5 @@ -package config +// Package httptransport provides the HTTP transport for clients that talk to a Databricks workspace. +package httptransport import ( "crypto/tls" @@ -25,10 +26,6 @@ const ( // which is unexported) and adds the HTTP/2 health check. Keep the two in sync on SDK upgrades. // experimental/ssh/internal/client/releases.go (newHTTP11Transport) is another hand copy of the // same defaults, HTTP/1.1-only; check it too. -// -// The bundle workspace client uses it, and so do workspace commands run in a bundle root: -// cmd/root/auth.go builds their client from the bundle client's config. The token-refresh client -// the SDK builds while resolving the config still uses the SDK default. func newHealthCheckedTransport(insecureSkipVerify bool) *http.Transport { t := &http.Transport{ Proxy: http.ProxyFromEnvironment, @@ -60,7 +57,11 @@ var ( insecureHealthCheckedTransport = sync.OnceValue(func() *http.Transport { return newHealthCheckedTransport(true) }) ) -func workspaceTransport(insecureSkipVerify bool) *http.Transport { +// HealthChecked returns the shared health-checked transport. Use it for clients that live long +// enough to see a connection die, in particular clients shared by a whole test process, where one +// dead connection would otherwise fail every test that follows. Set it on the config before the +// config is resolved, so the SDK's token-refresh client uses it too. +func HealthChecked(insecureSkipVerify bool) *http.Transport { if insecureSkipVerify { return insecureHealthCheckedTransport() } diff --git a/bundle/config/transport_test.go b/libs/httptransport/transport_test.go similarity index 79% rename from bundle/config/transport_test.go rename to libs/httptransport/transport_test.go index f05fc509923..6ee3d400b76 100644 --- a/bundle/config/transport_test.go +++ b/libs/httptransport/transport_test.go @@ -1,4 +1,4 @@ -package config +package httptransport import ( "context" @@ -11,9 +11,7 @@ import ( "testing" "time" - "github.com/databricks/cli/libs/databrickscfg" "github.com/databricks/cli/libs/retry" - "github.com/databricks/databricks-sdk-go/config" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -179,41 +177,12 @@ func TestTransportWithoutHealthCheckReusesDeadConnection(t *testing.T) { assert.EqualValues(t, 1, proxy.accepted.Load()) } -func TestWorkspaceTransportIsShared(t *testing.T) { - secure := workspaceTransport(false) - insecure := workspaceTransport(true) - assert.Same(t, secure, workspaceTransport(false)) - assert.Same(t, insecure, workspaceTransport(true)) +func TestHealthCheckedIsShared(t *testing.T) { + secure := HealthChecked(false) + insecure := HealthChecked(true) + assert.Same(t, secure, HealthChecked(false)) + assert.Same(t, insecure, HealthChecked(true)) assert.NotSame(t, secure, insecure) assert.Nil(t, secure.TLSClientConfig) assert.True(t, insecure.TLSClientConfig.InsecureSkipVerify) } - -func TestWorkspaceClientUsesHealthCheckedTransport(t *testing.T) { - setupWorkspaceTest(t) - - err := databrickscfg.SaveToProfile(t.Context(), &config.Config{ - Profile: "secure", - Host: "https://secure.cloud.databricks.test", - Token: "token", - }) - require.NoError(t, err) - // skip_verify comes from the profile, so it is only known after the config is resolved. - err = databrickscfg.SaveToProfile(t.Context(), &config.Config{ - Profile: "insecure", - Host: "https://insecure.cloud.databricks.test", - Token: "token", - InsecureSkipVerify: true, - }) - require.NoError(t, err) - - w := Workspace{Profile: "secure"} - client, err := w.Client(t.Context()) - require.NoError(t, err) - assert.Same(t, workspaceTransport(false), client.Config.HTTPTransport) - - w = Workspace{Profile: "insecure"} - client, err = w.Client(t.Context()) - require.NoError(t, err) - assert.Same(t, workspaceTransport(true), client.Config.HTTPTransport) -} diff --git a/libs/retry/retry.go b/libs/retry/retry.go index a1ec59d31ad..a577cf13aaa 100644 --- a/libs/retry/retry.go +++ b/libs/retry/retry.go @@ -74,7 +74,7 @@ func isInactivityTimeout(err error) bool { } // isConnectionLost reports whether the HTTP/2 health check closed the connection because a ping went -// unanswered (see bundle/config/transport.go). It replaces the inactivity timeout for a request on +// unanswered (see libs/httptransport). It replaces the inactivity timeout for a request on // a dead connection, with the same uncertainty about whether the server applied it, and the retry // goes out on a new connection. // diff --git a/libs/testproxy/server.go b/libs/testproxy/server.go index be5bcfd4aa7..a4c9abf44f6 100644 --- a/libs/testproxy/server.go +++ b/libs/testproxy/server.go @@ -7,6 +7,7 @@ import ( "net/http/httptest" "github.com/databricks/cli/internal/testutil" + "github.com/databricks/cli/libs/httptransport" "github.com/databricks/cli/libs/testserver" "github.com/databricks/databricks-sdk-go/apierr" "github.com/databricks/databricks-sdk-go/config" @@ -44,6 +45,11 @@ func New(t testutil.TestingT, upstream *config.Config) *ProxyServer { t: t, } + // The proxies of all tests share the transport's connection pool, so a dead connection + // would otherwise fail requests of every test that follows (see libs/httptransport). Set + // before the config is resolved, so the token-refresh client uses it too. With a transport + // set, the SDK ignores skip_verify; the proxy doesn't use it. + upstream.HTTPTransport = httptransport.HealthChecked(false) clientCfg, err := config.HTTPClientConfigFromConfig(upstream) require.NoError(t, err) s.apiClient = httpclient.NewApiClient(clientCfg) diff --git a/libs/testproxy/server_test.go b/libs/testproxy/server_test.go new file mode 100644 index 00000000000..e0ea2b64a95 --- /dev/null +++ b/libs/testproxy/server_test.go @@ -0,0 +1,17 @@ +package testproxy + +import ( + "testing" + + "github.com/databricks/cli/libs/httptransport" + "github.com/databricks/cli/libs/testserver" + "github.com/databricks/databricks-sdk-go/config" + "github.com/stretchr/testify/assert" +) + +func TestProxyUsesHealthCheckedTransport(t *testing.T) { + server := testserver.New(t) + upstream := &config.Config{Host: server.URL, Token: "token"} + New(t, upstream) + assert.Same(t, httptransport.HealthChecked(false), upstream.HTTPTransport) +}