From 482310dfd2017d3314399e470160ae97dc702d75 Mon Sep 17 00:00:00 2001 From: "denis.bilenko" Date: Tue, 17 Nov 2026 12:31:28 +0000 Subject: [PATCH 1/4] Close dead HTTP/2 connections in the bundle workspace client The SDK's default transport uses HTTP/2 without health-check pings. When a connection dies without a FIN or RST, every request to the host keeps going out over it, including the retries added in #7521, until the OS gives up on TCP (~15 min on Linux). Give the bundle workspace client a transport with HTTP2Config.SendPingTimeout=30s and PingTimeout=15s so the dead connection is closed, and retry "http2: client connection lost" on idempotent calls. Co-authored-by: Isaac --- bundle/config/transport.go | 62 ++++++++++++ bundle/config/transport_test.go | 162 ++++++++++++++++++++++++++++++++ bundle/config/workspace.go | 6 ++ libs/retry/retry.go | 4 +- libs/retry/retry_test.go | 3 + 5 files changed, 236 insertions(+), 1 deletion(-) create mode 100644 bundle/config/transport.go create mode 100644 bundle/config/transport_test.go diff --git a/bundle/config/transport.go b/bundle/config/transport.go new file mode 100644 index 00000000000..83bbfcdf275 --- /dev/null +++ b/bundle/config/transport.go @@ -0,0 +1,62 @@ +package config + +import ( + "crypto/tls" + "net" + "net/http" + "runtime" + "sync" + "time" +) + +// http2SendPingTimeout and http2PingTimeout make the HTTP/2 client ping a connection that has been +// silent for 30s and close it if the ping is not answered within 15s. +// +// Without them, a connection that dies without a FIN or RST (seen in CI as a single process's +// requests all hanging while other processes reach the same workspace fine) stays in the pool until +// the OS gives up on TCP, which takes ~15 minutes on Linux. A request that times out only resets its +// own stream, so the retries in libs/retry go out over the same dead connection and time out too. +const ( + http2SendPingTimeout = 30 * time.Second + http2PingTimeout = 15 * time.Second +) + +// newHealthCheckedTransport mirrors the SDK's default transport (httpclient.makeDefaultTransport) +// and adds the HTTP/2 health check. +func newHealthCheckedTransport(insecureSkipVerify bool) *http.Transport { + t := &http.Transport{ + Proxy: http.ProxyFromEnvironment, + DialContext: (&net.Dialer{ + Timeout: 30 * time.Second, + KeepAlive: 30 * time.Second, + }).DialContext, + ForceAttemptHTTP2: true, + MaxIdleConns: 100, + MaxIdleConnsPerHost: runtime.GOMAXPROCS(0) + 1, + IdleConnTimeout: 180 * time.Second, + TLSHandshakeTimeout: 30 * time.Second, + ExpectContinueTimeout: 1 * time.Second, + HTTP2: &http.HTTP2Config{ + SendPingTimeout: http2SendPingTimeout, + PingTimeout: http2PingTimeout, + }, + } + if insecureSkipVerify { + // The SDK only applies InsecureSkipVerify to its own transport, so we carry it over. + t.TLSClientConfig = &tls.Config{InsecureSkipVerify: true} //nolint:gosec // the user opted in via insecure_skip_verify + } + return t +} + +// The transports are shared, like the SDK's default one, so clients reuse pooled connections. +var ( + healthCheckedTransport = sync.OnceValue(func() *http.Transport { return newHealthCheckedTransport(false) }) + insecureHealthCheckedTransport = sync.OnceValue(func() *http.Transport { return newHealthCheckedTransport(true) }) +) + +func workspaceTransport(insecureSkipVerify bool) *http.Transport { + if insecureSkipVerify { + return insecureHealthCheckedTransport() + } + return healthCheckedTransport() +} diff --git a/bundle/config/transport_test.go b/bundle/config/transport_test.go new file mode 100644 index 00000000000..1dcc8bbd968 --- /dev/null +++ b/bundle/config/transport_test.go @@ -0,0 +1,162 @@ +package config + +import ( + "context" + "io" + "net" + "net/http" + "net/http/httptest" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// blackholeProxy forwards TCP connections to target until blackhole is called. After that, the +// connections it already accepted stop forwarding bytes in both directions but stay open, like a +// connection silently dropped by a NAT or load balancer. New connections still work. +type blackholeProxy struct { + ln net.Listener + target string + mu sync.Mutex + conns []*atomic.Bool + accepted atomic.Int32 +} + +func newBlackholeProxy(t *testing.T, target string) *blackholeProxy { + ln, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + p := &blackholeProxy{ln: ln, target: target} + t.Cleanup(func() { ln.Close() }) + go p.serve() + return p +} + +func (p *blackholeProxy) serve() { + for { + client, err := p.ln.Accept() + if err != nil { + return + } + p.accepted.Add(1) + server, err := net.Dial("tcp", p.target) + if err != nil { + client.Close() + continue + } + dead := &atomic.Bool{} + p.mu.Lock() + p.conns = append(p.conns, dead) + p.mu.Unlock() + go pipe(client, server, dead) + go pipe(server, client, dead) + } +} + +func pipe(dst, src net.Conn, dead *atomic.Bool) { + buf := make([]byte, 32*1024) + for { + n, err := src.Read(buf) + if err != nil { + return + } + if dead.Load() { + continue + } + if _, err := dst.Write(buf[:n]); err != nil { + return + } + } +} + +func (p *blackholeProxy) blackhole() { + p.mu.Lock() + defer p.mu.Unlock() + for _, c := range p.conns { + c.Store(true) + } +} + +// setupDeadConnection returns a GET helper for an HTTP/2 server behind a blackholeProxy, after one +// successful request whose connection the proxy then blackholes. +func setupDeadConnection(t *testing.T, transport *http.Transport) (*blackholeProxy, func(time.Duration) (string, error)) { + srv := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = io.WriteString(w, r.Proto) + })) + srv.EnableHTTP2 = true + srv.StartTLS() + t.Cleanup(srv.Close) + + proxy := newBlackholeProxy(t, srv.Listener.Addr().String()) + + transport.TLSClientConfig = srv.Client().Transport.(*http.Transport).TLSClientConfig.Clone() + t.Cleanup(transport.CloseIdleConnections) + client := &http.Client{Transport: transport} + url := "https://" + proxy.ln.Addr().String() + "/" + + get := func(timeout time.Duration) (string, error) { + ctx, cancel := context.WithTimeout(t.Context(), timeout) + defer cancel() + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + require.NoError(t, err) + resp, err := client.Do(req) + if err != nil { + return "", err + } + defer resp.Body.Close() + body, err := io.ReadAll(resp.Body) + return string(body), err + } + + proto, err := get(5 * time.Second) + require.NoError(t, err) + require.Equal(t, "HTTP/2.0", proto) + require.EqualValues(t, 1, proxy.accepted.Load()) + + proxy.blackhole() + return proxy, get +} + +func TestHealthCheckedTransportReplacesDeadConnection(t *testing.T) { + transport := newHealthCheckedTransport(false) + transport.HTTP2.SendPingTimeout = 200 * time.Millisecond + transport.HTTP2.PingTimeout = 200 * time.Millisecond + proxy, get := setupDeadConnection(t, transport) + + // The request on the dead connection fails: the health check closes the connection. + _, err := get(5 * time.Second) + require.ErrorContains(t, err, "http2: client connection lost") + + // The next request dials a new connection and succeeds. + proto, err := get(5 * time.Second) + require.NoError(t, err) + assert.Equal(t, "HTTP/2.0", proto) + assert.EqualValues(t, 2, proxy.accepted.Load()) +} + +// Without the health check, a timed-out request leaves the dead connection in the pool and the +// next request (a retry) is sent on it again. This is the failure the health check fixes. +func TestTransportWithoutHealthCheckReusesDeadConnection(t *testing.T) { + transport := newHealthCheckedTransport(false) + transport.HTTP2 = nil + proxy, get := setupDeadConnection(t, transport) + + _, err := get(500 * time.Millisecond) + require.ErrorIs(t, err, context.DeadlineExceeded) + _, err = get(500 * time.Millisecond) + require.ErrorIs(t, err, context.DeadlineExceeded) + assert.EqualValues(t, 1, proxy.accepted.Load()) +} + +func TestWorkspaceTransportIsShared(t *testing.T) { + secure := workspaceTransport(false) + insecure := workspaceTransport(true) + assert.Same(t, secure, workspaceTransport(false)) + assert.Same(t, insecure, workspaceTransport(true)) + assert.NotSame(t, secure, insecure) + assert.Nil(t, secure.TLSClientConfig) + assert.True(t, insecure.TLSClientConfig.InsecureSkipVerify) +} diff --git a/bundle/config/workspace.go b/bundle/config/workspace.go index bed0d0bf69d..5df36a5f6cb 100644 --- a/bundle/config/workspace.go +++ b/bundle/config/workspace.go @@ -247,5 +247,11 @@ func (w *Workspace) Client(ctx context.Context) (*databricks.WorkspaceClient, er } } + // Set after EnsureResolved, so InsecureSkipVerify from a profile or env var is known. + // A transport set by the caller (e.g. in tests) is kept. + if cfg.HTTPTransport == nil { + cfg.HTTPTransport = workspaceTransport(cfg.InsecureSkipVerify) + } + return databricks.NewWorkspaceClient((*databricks.Config)(cfg)) } diff --git a/libs/retry/retry.go b/libs/retry/retry.go index 95bf5a7250e..2ea07232422 100644 --- a/libs/retry/retry.go +++ b/libs/retry/retry.go @@ -79,7 +79,9 @@ func isInactivityTimeout(err error) bool { // processing the request. // - unexpected EOF, where the connection was cut mid-response. Observed on GET /clusters/get and // POST /workspace-files/import-file. -var transientReadMessages = []string{"REFUSED_STREAM", "unexpected EOF"} +// - http2: client connection lost, where the HTTP/2 health check found the connection dead and +// closed it (see bundle/config/transport.go). The retry goes out on a new connection. +var transientReadMessages = []string{"REFUSED_STREAM", "unexpected EOF", "http2: client connection lost"} // IsTransientRead returns true for errors worth retrying on an idempotent call (a read or an // overwriting write): the transient server errors IsTransient covers, plus the transport errors diff --git a/libs/retry/retry_test.go b/libs/retry/retry_test.go index 396ad9e1803..496e857b4b6 100644 --- a/libs/retry/retry_test.go +++ b/libs/retry/retry_test.go @@ -47,6 +47,9 @@ func TestIsTransientRead(t *testing.T) { // The connection cut mid-response, seen on GET /clusters/get. unexpectedEOF := &url.Error{Op: "Get", URL: "https://example.test/api/2.1/clusters/get", Err: io.ErrUnexpectedEOF} assert.True(t, IsTransientRead(ctx, unexpectedEOF)) + // The HTTP/2 health check closed a dead connection; the retry dials a new one. + connLost := &url.Error{Op: "Post", URL: "https://example.test/api/2.0/workspace-files/import-file/a?overwrite=true", Err: errors.New("http2: client connection lost")} + assert.True(t, IsTransientRead(ctx, connLost)) // Transport errors the SDK already retries for its full RetryTimeout must not be retried again. connRefused := &url.Error{Op: "Get", URL: "https://example.test/api/2.0/workspace/get-status", Err: errors.New("dial tcp 10.0.0.1:443: connect: connection refused")} assert.False(t, IsTransientRead(ctx, connRefused)) From 7c297f11a20de4a0798c46425848200b669f91f3 Mon Sep 17 00:00:00 2001 From: "denis.bilenko" Date: Tue, 17 Nov 2026 12:45:35 +0000 Subject: [PATCH 2/4] Add changelog entry cloud: TestAccept/bundle/invariant/no_drift/DMS=/INPUT_CONFIG=pipeline_allow_duplicate_names.yml.tmpl TestAccept/bundle/invariant/no_drift/DMS=/INPUT_CONFIG=job.yml.tmpl TestAccept/bundle/invariant/delete_idempotent/DMS=/INPUT_CONFIG=pipeline_allow_duplicate_names.yml.tmpl Co-authored-by: Isaac --- .nextchanges/bundles/7723.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 .nextchanges/bundles/7723.md diff --git a/.nextchanges/bundles/7723.md b/.nextchanges/bundles/7723.md new file mode 100644 index 00000000000..f9b2b1c2c67 --- /dev/null +++ b/.nextchanges/bundles/7723.md @@ -0,0 +1 @@ +* Close dead HTTP/2 connections during bundle commands, so retries after a request timeout go out on a new connection instead of hanging until the OS drops the old one. ([#7723](https://github.com/databricks/cli/pull/7723)) From a7d3bedd2b52f6f49ee4de4c45c8e9bb7819cbb4 Mon Sep 17 00:00:00 2001 From: "denis.bilenko" Date: Tue, 17 Nov 2026 13:19:12 +0000 Subject: [PATCH 3/4] Address review: retry connection-lost on all calls, harden tests - IsTransient (not only IsTransientRead) retries "http2: client connection lost": the health check now ends a request on a dead connection before the inactivity timeout, which IsTransient already retries. - Set the transport unconditionally; the nil guard could never be false. - Test proxy closes its connections; the health-check test tolerates the connection being closed before the request is sent; test that Workspace.Client picks the transport after insecure_skip_verify is resolved. cloud: TestAccept/bundle/invariant/no_drift/DMS=/INPUT_CONFIG=pipeline_allow_duplicate_names.yml.tmpl TestAccept/bundle/invariant/no_drift/DMS=/INPUT_CONFIG=job.yml.tmpl TestAccept/bundle/invariant/delete_idempotent/DMS=/INPUT_CONFIG=pipeline_allow_duplicate_names.yml.tmpl Co-authored-by: Isaac --- bundle/config/transport.go | 7 ++- bundle/config/transport_test.go | 97 ++++++++++++++++++++++++++------- bundle/config/workspace.go | 5 +- libs/retry/retry.go | 19 +++++-- libs/retry/retry_test.go | 6 ++ 5 files changed, 102 insertions(+), 32 deletions(-) diff --git a/bundle/config/transport.go b/bundle/config/transport.go index 83bbfcdf275..05c782de602 100644 --- a/bundle/config/transport.go +++ b/bundle/config/transport.go @@ -21,8 +21,11 @@ const ( http2PingTimeout = 15 * time.Second ) -// newHealthCheckedTransport mirrors the SDK's default transport (httpclient.makeDefaultTransport) -// and adds the HTTP/2 health check. +// newHealthCheckedTransport mirrors the SDK's default transport (httpclient.makeDefaultTransport, +// which is unexported) and adds the HTTP/2 health check. Keep the two in sync on SDK upgrades. +// +// Only the bundle workspace client uses it. The token-refresh client the SDK builds while +// resolving the config still uses the SDK default. func newHealthCheckedTransport(insecureSkipVerify bool) *http.Transport { t := &http.Transport{ Proxy: http.ProxyFromEnvironment, diff --git a/bundle/config/transport_test.go b/bundle/config/transport_test.go index 1dcc8bbd968..eec5cab775e 100644 --- a/bundle/config/transport_test.go +++ b/bundle/config/transport_test.go @@ -11,6 +11,9 @@ import ( "testing" "time" + "github.com/databricks/cli/libs/databrickscfg" + "github.com/databricks/cli/libs/retry" + "github.com/databricks/databricks-sdk-go/config" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -22,15 +25,20 @@ type blackholeProxy struct { ln net.Listener target string mu sync.Mutex - conns []*atomic.Bool + links []*proxyLink accepted atomic.Int32 } +type proxyLink struct { + client, server net.Conn + dead atomic.Bool +} + func newBlackholeProxy(t *testing.T, target string) *blackholeProxy { ln, err := net.Listen("tcp", "127.0.0.1:0") require.NoError(t, err) p := &blackholeProxy{ln: ln, target: target} - t.Cleanup(func() { ln.Close() }) + t.Cleanup(p.close) go p.serve() return p } @@ -41,29 +49,30 @@ func (p *blackholeProxy) serve() { if err != nil { return } - p.accepted.Add(1) server, err := net.Dial("tcp", p.target) if err != nil { client.Close() continue } - dead := &atomic.Bool{} + p.accepted.Add(1) + l := &proxyLink{client: client, server: server} p.mu.Lock() - p.conns = append(p.conns, dead) + p.links = append(p.links, l) p.mu.Unlock() - go pipe(client, server, dead) - go pipe(server, client, dead) + go l.pipe(server, client) + go l.pipe(client, server) } } -func pipe(dst, src net.Conn, dead *atomic.Bool) { +func (l *proxyLink) pipe(dst, src net.Conn) { + defer l.close() buf := make([]byte, 32*1024) for { n, err := src.Read(buf) if err != nil { return } - if dead.Load() { + if l.dead.Load() { continue } if _, err := dst.Write(buf[:n]); err != nil { @@ -72,11 +81,25 @@ func pipe(dst, src net.Conn, dead *atomic.Bool) { } } +func (l *proxyLink) close() { + l.client.Close() + l.server.Close() +} + func (p *blackholeProxy) blackhole() { p.mu.Lock() defer p.mu.Unlock() - for _, c := range p.conns { - c.Store(true) + for _, l := range p.links { + l.dead.Store(true) + } +} + +func (p *blackholeProxy) close() { + p.ln.Close() + p.mu.Lock() + defer p.mu.Unlock() + for _, l := range p.links { + l.close() } } @@ -122,23 +145,28 @@ func setupDeadConnection(t *testing.T, transport *http.Transport) (*blackholePro func TestHealthCheckedTransportReplacesDeadConnection(t *testing.T) { transport := newHealthCheckedTransport(false) - transport.HTTP2.SendPingTimeout = 200 * time.Millisecond - transport.HTTP2.PingTimeout = 200 * time.Millisecond + transport.HTTP2.SendPingTimeout = time.Second + transport.HTTP2.PingTimeout = time.Second proxy, get := setupDeadConnection(t, transport) - // The request on the dead connection fails: the health check closes the connection. - _, err := get(5 * time.Second) - require.ErrorContains(t, err, "http2: client connection lost") + // A request sent on the dead connection fails once the health check closes it, with an error + // libs/retry retries. If the health check closed the connection before the request was sent, + // the request already goes out on a new connection. + _, err := get(10 * time.Second) + if err != nil { + require.ErrorContains(t, err, "http2: client connection lost") + assert.True(t, retry.IsTransient(t.Context(), err)) + } - // The next request dials a new connection and succeeds. - proto, err := get(5 * time.Second) + proto, err := get(10 * time.Second) require.NoError(t, err) assert.Equal(t, "HTTP/2.0", proto) assert.EqualValues(t, 2, proxy.accepted.Load()) } -// Without the health check, a timed-out request leaves the dead connection in the pool and the -// next request (a retry) is sent on it again. This is the failure the health check fixes. +// Documents the net/http behavior the health check works around: without it, a timed-out request +// leaves the dead connection in the pool and the next request (a retry) is sent on it again. If a +// Go release stops reusing such connections, this test fails and can be removed. func TestTransportWithoutHealthCheckReusesDeadConnection(t *testing.T) { transport := newHealthCheckedTransport(false) transport.HTTP2 = nil @@ -160,3 +188,32 @@ func TestWorkspaceTransportIsShared(t *testing.T) { assert.Nil(t, secure.TLSClientConfig) assert.True(t, insecure.TLSClientConfig.InsecureSkipVerify) } + +func TestWorkspaceClientUsesHealthCheckedTransport(t *testing.T) { + setupWorkspaceTest(t) + + err := databrickscfg.SaveToProfile(t.Context(), &config.Config{ + Profile: "secure", + Host: "https://secure.cloud.databricks.test", + Token: "token", + }) + require.NoError(t, err) + // insecure_skip_verify comes from the profile, so it is only known after the config is resolved. + err = databrickscfg.SaveToProfile(t.Context(), &config.Config{ + Profile: "insecure", + Host: "https://insecure.cloud.databricks.test", + Token: "token", + InsecureSkipVerify: true, + }) + require.NoError(t, err) + + w := Workspace{Profile: "secure"} + client, err := w.Client(t.Context()) + require.NoError(t, err) + assert.Same(t, workspaceTransport(false), client.Config.HTTPTransport) + + w = Workspace{Profile: "insecure"} + client, err = w.Client(t.Context()) + require.NoError(t, err) + assert.Same(t, workspaceTransport(true), client.Config.HTTPTransport) +} diff --git a/bundle/config/workspace.go b/bundle/config/workspace.go index 5df36a5f6cb..ec5ca591758 100644 --- a/bundle/config/workspace.go +++ b/bundle/config/workspace.go @@ -248,10 +248,7 @@ func (w *Workspace) Client(ctx context.Context) (*databricks.WorkspaceClient, er } // Set after EnsureResolved, so InsecureSkipVerify from a profile or env var is known. - // A transport set by the caller (e.g. in tests) is kept. - if cfg.HTTPTransport == nil { - cfg.HTTPTransport = workspaceTransport(cfg.InsecureSkipVerify) - } + cfg.HTTPTransport = workspaceTransport(cfg.InsecureSkipVerify) return databricks.NewWorkspaceClient((*databricks.Config)(cfg)) } diff --git a/libs/retry/retry.go b/libs/retry/retry.go index 2ea07232422..2e6e6618cef 100644 --- a/libs/retry/retry.go +++ b/libs/retry/retry.go @@ -38,12 +38,12 @@ func interval(ctx context.Context) time.Duration { return time.Duration(ms) * time.Millisecond } -// IsTransient returns true for 5xx and 408 errors that the SDK did not already retry, and for -// the SDK's client-side inactivity timeout. +// IsTransient returns true for 5xx and 408 errors that the SDK did not already retry, for the +// SDK's client-side inactivity timeout, and for an HTTP/2 connection closed by its health check. // The SDK retries 504s matching its allTransientErrors patterns (e.g. "deadline exceeded"); // this covers the remaining 504s like TEMPORARILY_UNAVAILABLE. func IsTransient(ctx context.Context, err error) bool { - if isInactivityTimeout(err) { + if isInactivityTimeout(err) || isConnectionLost(err) { return true } apiErr, ok := errors.AsType[*apierr.APIError](err) @@ -73,15 +73,22 @@ func isInactivityTimeout(err error) bool { return strings.HasPrefix(msg, "request timed out after ") && strings.HasSuffix(msg, " of inactivity") } +// isConnectionLost reports whether the HTTP/2 health check closed the connection because a ping went +// unanswered (see bundle/config/transport.go). It replaces the inactivity timeout for a request on +// a dead connection, with the same uncertainty about whether the server applied it, and the retry +// goes out on a new connection. +func isConnectionLost(err error) bool { + urlErr, ok := errors.AsType[*url.Error](err) + return ok && urlErr.Err.Error() == "http2: client connection lost" +} + // transientReadMessages are the transport errors, matched by message, that are worth retrying on // an idempotent call: // - REFUSED_STREAM, the HTTP/2 refusal in issue #6472, where the server refused the stream before // processing the request. // - unexpected EOF, where the connection was cut mid-response. Observed on GET /clusters/get and // POST /workspace-files/import-file. -// - http2: client connection lost, where the HTTP/2 health check found the connection dead and -// closed it (see bundle/config/transport.go). The retry goes out on a new connection. -var transientReadMessages = []string{"REFUSED_STREAM", "unexpected EOF", "http2: client connection lost"} +var transientReadMessages = []string{"REFUSED_STREAM", "unexpected EOF"} // IsTransientRead returns true for errors worth retrying on an idempotent call (a read or an // overwriting write): the transient server errors IsTransient covers, plus the transport errors diff --git a/libs/retry/retry_test.go b/libs/retry/retry_test.go index 496e857b4b6..f93c7d8ce18 100644 --- a/libs/retry/retry_test.go +++ b/libs/retry/retry_test.go @@ -34,6 +34,12 @@ func TestIsTransient(t *testing.T) { assert.True(t, IsTransientRead(ctx, inactivity)) cancelled := &url.Error{Op: "Post", URL: "https://example.test/api/2.0/workspace-files/import-file/a", Err: context.Canceled} assert.False(t, IsTransient(ctx, cancelled)) + + // The HTTP/2 health check closed a dead connection, which ends the request before the + // inactivity timeout would. + connLost := &url.Error{Op: "Post", URL: "https://example.test/api/2.0/workspace-files/import-file/a", Err: errors.New("http2: client connection lost")} + assert.True(t, IsTransient(ctx, connLost)) + assert.True(t, IsTransient(ctx, fmt.Errorf("failed to upload: %w", connLost))) } func TestIsTransientRead(t *testing.T) { From 8a92a8c4dcecd99c8c6568aef4f831f0bc18933a Mon Sep 17 00:00:00 2001 From: "denis.bilenko" Date: Wed, 18 Nov 2026 06:21:48 +0000 Subject: [PATCH 4/4] Address review: document transport scope, sync note and error matching - Note that workspace commands run in a bundle root use the transport too (cmd/root/auth.go builds their client from the bundle client's config). - Name experimental/ssh's newHTTP11Transport in the sync note. - The SDK attribute is skip_verify and has no env var. - Say why isConnectionLost matches by message. Co-authored-by: Isaac --- bundle/config/transport.go | 9 ++++++--- bundle/config/transport_test.go | 2 +- bundle/config/workspace.go | 2 +- libs/retry/retry.go | 3 +++ 4 files changed, 11 insertions(+), 5 deletions(-) diff --git a/bundle/config/transport.go b/bundle/config/transport.go index 05c782de602..d637bcf6396 100644 --- a/bundle/config/transport.go +++ b/bundle/config/transport.go @@ -23,9 +23,12 @@ const ( // newHealthCheckedTransport mirrors the SDK's default transport (httpclient.makeDefaultTransport, // which is unexported) and adds the HTTP/2 health check. Keep the two in sync on SDK upgrades. +// experimental/ssh/internal/client/releases.go (newHTTP11Transport) is another hand copy of the +// same defaults, HTTP/1.1-only; check it too. // -// Only the bundle workspace client uses it. The token-refresh client the SDK builds while -// resolving the config still uses the SDK default. +// The bundle workspace client uses it, and so do workspace commands run in a bundle root: +// cmd/root/auth.go builds their client from the bundle client's config. The token-refresh client +// the SDK builds while resolving the config still uses the SDK default. func newHealthCheckedTransport(insecureSkipVerify bool) *http.Transport { t := &http.Transport{ Proxy: http.ProxyFromEnvironment, @@ -46,7 +49,7 @@ func newHealthCheckedTransport(insecureSkipVerify bool) *http.Transport { } if insecureSkipVerify { // The SDK only applies InsecureSkipVerify to its own transport, so we carry it over. - t.TLSClientConfig = &tls.Config{InsecureSkipVerify: true} //nolint:gosec // the user opted in via insecure_skip_verify + t.TLSClientConfig = &tls.Config{InsecureSkipVerify: true} //nolint:gosec // the user opted in via skip_verify } return t } diff --git a/bundle/config/transport_test.go b/bundle/config/transport_test.go index eec5cab775e..f05fc509923 100644 --- a/bundle/config/transport_test.go +++ b/bundle/config/transport_test.go @@ -198,7 +198,7 @@ func TestWorkspaceClientUsesHealthCheckedTransport(t *testing.T) { Token: "token", }) require.NoError(t, err) - // insecure_skip_verify comes from the profile, so it is only known after the config is resolved. + // skip_verify comes from the profile, so it is only known after the config is resolved. err = databrickscfg.SaveToProfile(t.Context(), &config.Config{ Profile: "insecure", Host: "https://insecure.cloud.databricks.test", diff --git a/bundle/config/workspace.go b/bundle/config/workspace.go index ec5ca591758..abe9f82b780 100644 --- a/bundle/config/workspace.go +++ b/bundle/config/workspace.go @@ -247,7 +247,7 @@ func (w *Workspace) Client(ctx context.Context) (*databricks.WorkspaceClient, er } } - // Set after EnsureResolved, so InsecureSkipVerify from a profile or env var is known. + // Set after EnsureResolved, so skip_verify from the profile is known. cfg.HTTPTransport = workspaceTransport(cfg.InsecureSkipVerify) return databricks.NewWorkspaceClient((*databricks.Config)(cfg)) diff --git a/libs/retry/retry.go b/libs/retry/retry.go index 2e6e6618cef..a1ec59d31ad 100644 --- a/libs/retry/retry.go +++ b/libs/retry/retry.go @@ -77,6 +77,9 @@ func isInactivityTimeout(err error) bool { // unanswered (see bundle/config/transport.go). It replaces the inactivity timeout for a request on // a dead connection, with the same uncertainty about whether the server applied it, and the retry // goes out on a new connection. +// +// We match the message because net/http builds it with errors.New in closeForLostPing and exports +// no sentinel error or type for it. func isConnectionLost(err error) bool { urlErr, ok := errors.AsType[*url.Error](err) return ok && urlErr.Err.Error() == "http2: client connection lost"