diff --git a/.nextchanges/bundles/7723.md b/.nextchanges/bundles/7723.md new file mode 100644 index 00000000000..f9b2b1c2c67 --- /dev/null +++ b/.nextchanges/bundles/7723.md @@ -0,0 +1 @@ +* Close dead HTTP/2 connections during bundle commands, so retries after a request timeout go out on a new connection instead of hanging until the OS drops the old one. ([#7723](https://github.com/databricks/cli/pull/7723)) diff --git a/bundle/config/transport.go b/bundle/config/transport.go new file mode 100644 index 00000000000..d637bcf6396 --- /dev/null +++ b/bundle/config/transport.go @@ -0,0 +1,68 @@ +package config + +import ( + "crypto/tls" + "net" + "net/http" + "runtime" + "sync" + "time" +) + +// http2SendPingTimeout and http2PingTimeout make the HTTP/2 client ping a connection that has been +// silent for 30s and close it if the ping is not answered within 15s. +// +// Without them, a connection that dies without a FIN or RST (seen in CI as a single process's +// requests all hanging while other processes reach the same workspace fine) stays in the pool until +// the OS gives up on TCP, which takes ~15 minutes on Linux. A request that times out only resets its +// own stream, so the retries in libs/retry go out over the same dead connection and time out too. +const ( + http2SendPingTimeout = 30 * time.Second + http2PingTimeout = 15 * time.Second +) + +// newHealthCheckedTransport mirrors the SDK's default transport (httpclient.makeDefaultTransport, +// which is unexported) and adds the HTTP/2 health check. Keep the two in sync on SDK upgrades. +// experimental/ssh/internal/client/releases.go (newHTTP11Transport) is another hand copy of the +// same defaults, HTTP/1.1-only; check it too. +// +// The bundle workspace client uses it, and so do workspace commands run in a bundle root: +// cmd/root/auth.go builds their client from the bundle client's config. The token-refresh client +// the SDK builds while resolving the config still uses the SDK default. +func newHealthCheckedTransport(insecureSkipVerify bool) *http.Transport { + t := &http.Transport{ + Proxy: http.ProxyFromEnvironment, + DialContext: (&net.Dialer{ + Timeout: 30 * time.Second, + KeepAlive: 30 * time.Second, + }).DialContext, + ForceAttemptHTTP2: true, + MaxIdleConns: 100, + MaxIdleConnsPerHost: runtime.GOMAXPROCS(0) + 1, + IdleConnTimeout: 180 * time.Second, + TLSHandshakeTimeout: 30 * time.Second, + ExpectContinueTimeout: 1 * time.Second, + HTTP2: &http.HTTP2Config{ + SendPingTimeout: http2SendPingTimeout, + PingTimeout: http2PingTimeout, + }, + } + if insecureSkipVerify { + // The SDK only applies InsecureSkipVerify to its own transport, so we carry it over. + t.TLSClientConfig = &tls.Config{InsecureSkipVerify: true} //nolint:gosec // the user opted in via skip_verify + } + return t +} + +// The transports are shared, like the SDK's default one, so clients reuse pooled connections. +var ( + healthCheckedTransport = sync.OnceValue(func() *http.Transport { return newHealthCheckedTransport(false) }) + insecureHealthCheckedTransport = sync.OnceValue(func() *http.Transport { return newHealthCheckedTransport(true) }) +) + +func workspaceTransport(insecureSkipVerify bool) *http.Transport { + if insecureSkipVerify { + return insecureHealthCheckedTransport() + } + return healthCheckedTransport() +} diff --git a/bundle/config/transport_test.go b/bundle/config/transport_test.go new file mode 100644 index 00000000000..f05fc509923 --- /dev/null +++ b/bundle/config/transport_test.go @@ -0,0 +1,219 @@ +package config + +import ( + "context" + "io" + "net" + "net/http" + "net/http/httptest" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/databricks/cli/libs/databrickscfg" + "github.com/databricks/cli/libs/retry" + "github.com/databricks/databricks-sdk-go/config" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// blackholeProxy forwards TCP connections to target until blackhole is called. After that, the +// connections it already accepted stop forwarding bytes in both directions but stay open, like a +// connection silently dropped by a NAT or load balancer. New connections still work. +type blackholeProxy struct { + ln net.Listener + target string + mu sync.Mutex + links []*proxyLink + accepted atomic.Int32 +} + +type proxyLink struct { + client, server net.Conn + dead atomic.Bool +} + +func newBlackholeProxy(t *testing.T, target string) *blackholeProxy { + ln, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + p := &blackholeProxy{ln: ln, target: target} + t.Cleanup(p.close) + go p.serve() + return p +} + +func (p *blackholeProxy) serve() { + for { + client, err := p.ln.Accept() + if err != nil { + return + } + server, err := net.Dial("tcp", p.target) + if err != nil { + client.Close() + continue + } + p.accepted.Add(1) + l := &proxyLink{client: client, server: server} + p.mu.Lock() + p.links = append(p.links, l) + p.mu.Unlock() + go l.pipe(server, client) + go l.pipe(client, server) + } +} + +func (l *proxyLink) pipe(dst, src net.Conn) { + defer l.close() + buf := make([]byte, 32*1024) + for { + n, err := src.Read(buf) + if err != nil { + return + } + if l.dead.Load() { + continue + } + if _, err := dst.Write(buf[:n]); err != nil { + return + } + } +} + +func (l *proxyLink) close() { + l.client.Close() + l.server.Close() +} + +func (p *blackholeProxy) blackhole() { + p.mu.Lock() + defer p.mu.Unlock() + for _, l := range p.links { + l.dead.Store(true) + } +} + +func (p *blackholeProxy) close() { + p.ln.Close() + p.mu.Lock() + defer p.mu.Unlock() + for _, l := range p.links { + l.close() + } +} + +// setupDeadConnection returns a GET helper for an HTTP/2 server behind a blackholeProxy, after one +// successful request whose connection the proxy then blackholes. +func setupDeadConnection(t *testing.T, transport *http.Transport) (*blackholeProxy, func(time.Duration) (string, error)) { + srv := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = io.WriteString(w, r.Proto) + })) + srv.EnableHTTP2 = true + srv.StartTLS() + t.Cleanup(srv.Close) + + proxy := newBlackholeProxy(t, srv.Listener.Addr().String()) + + transport.TLSClientConfig = srv.Client().Transport.(*http.Transport).TLSClientConfig.Clone() + t.Cleanup(transport.CloseIdleConnections) + client := &http.Client{Transport: transport} + url := "https://" + proxy.ln.Addr().String() + "/" + + get := func(timeout time.Duration) (string, error) { + ctx, cancel := context.WithTimeout(t.Context(), timeout) + defer cancel() + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + require.NoError(t, err) + resp, err := client.Do(req) + if err != nil { + return "", err + } + defer resp.Body.Close() + body, err := io.ReadAll(resp.Body) + return string(body), err + } + + proto, err := get(5 * time.Second) + require.NoError(t, err) + require.Equal(t, "HTTP/2.0", proto) + require.EqualValues(t, 1, proxy.accepted.Load()) + + proxy.blackhole() + return proxy, get +} + +func TestHealthCheckedTransportReplacesDeadConnection(t *testing.T) { + transport := newHealthCheckedTransport(false) + transport.HTTP2.SendPingTimeout = time.Second + transport.HTTP2.PingTimeout = time.Second + proxy, get := setupDeadConnection(t, transport) + + // A request sent on the dead connection fails once the health check closes it, with an error + // libs/retry retries. If the health check closed the connection before the request was sent, + // the request already goes out on a new connection. + _, err := get(10 * time.Second) + if err != nil { + require.ErrorContains(t, err, "http2: client connection lost") + assert.True(t, retry.IsTransient(t.Context(), err)) + } + + proto, err := get(10 * time.Second) + require.NoError(t, err) + assert.Equal(t, "HTTP/2.0", proto) + assert.EqualValues(t, 2, proxy.accepted.Load()) +} + +// Documents the net/http behavior the health check works around: without it, a timed-out request +// leaves the dead connection in the pool and the next request (a retry) is sent on it again. If a +// Go release stops reusing such connections, this test fails and can be removed. +func TestTransportWithoutHealthCheckReusesDeadConnection(t *testing.T) { + transport := newHealthCheckedTransport(false) + transport.HTTP2 = nil + proxy, get := setupDeadConnection(t, transport) + + _, err := get(500 * time.Millisecond) + require.ErrorIs(t, err, context.DeadlineExceeded) + _, err = get(500 * time.Millisecond) + require.ErrorIs(t, err, context.DeadlineExceeded) + assert.EqualValues(t, 1, proxy.accepted.Load()) +} + +func TestWorkspaceTransportIsShared(t *testing.T) { + secure := workspaceTransport(false) + insecure := workspaceTransport(true) + assert.Same(t, secure, workspaceTransport(false)) + assert.Same(t, insecure, workspaceTransport(true)) + assert.NotSame(t, secure, insecure) + assert.Nil(t, secure.TLSClientConfig) + assert.True(t, insecure.TLSClientConfig.InsecureSkipVerify) +} + +func TestWorkspaceClientUsesHealthCheckedTransport(t *testing.T) { + setupWorkspaceTest(t) + + err := databrickscfg.SaveToProfile(t.Context(), &config.Config{ + Profile: "secure", + Host: "https://secure.cloud.databricks.test", + Token: "token", + }) + require.NoError(t, err) + // skip_verify comes from the profile, so it is only known after the config is resolved. + err = databrickscfg.SaveToProfile(t.Context(), &config.Config{ + Profile: "insecure", + Host: "https://insecure.cloud.databricks.test", + Token: "token", + InsecureSkipVerify: true, + }) + require.NoError(t, err) + + w := Workspace{Profile: "secure"} + client, err := w.Client(t.Context()) + require.NoError(t, err) + assert.Same(t, workspaceTransport(false), client.Config.HTTPTransport) + + w = Workspace{Profile: "insecure"} + client, err = w.Client(t.Context()) + require.NoError(t, err) + assert.Same(t, workspaceTransport(true), client.Config.HTTPTransport) +} diff --git a/bundle/config/workspace.go b/bundle/config/workspace.go index bed0d0bf69d..abe9f82b780 100644 --- a/bundle/config/workspace.go +++ b/bundle/config/workspace.go @@ -247,5 +247,8 @@ func (w *Workspace) Client(ctx context.Context) (*databricks.WorkspaceClient, er } } + // Set after EnsureResolved, so skip_verify from the profile is known. + cfg.HTTPTransport = workspaceTransport(cfg.InsecureSkipVerify) + return databricks.NewWorkspaceClient((*databricks.Config)(cfg)) } diff --git a/libs/retry/retry.go b/libs/retry/retry.go index 95bf5a7250e..a1ec59d31ad 100644 --- a/libs/retry/retry.go +++ b/libs/retry/retry.go @@ -38,12 +38,12 @@ func interval(ctx context.Context) time.Duration { return time.Duration(ms) * time.Millisecond } -// IsTransient returns true for 5xx and 408 errors that the SDK did not already retry, and for -// the SDK's client-side inactivity timeout. +// IsTransient returns true for 5xx and 408 errors that the SDK did not already retry, for the +// SDK's client-side inactivity timeout, and for an HTTP/2 connection closed by its health check. // The SDK retries 504s matching its allTransientErrors patterns (e.g. "deadline exceeded"); // this covers the remaining 504s like TEMPORARILY_UNAVAILABLE. func IsTransient(ctx context.Context, err error) bool { - if isInactivityTimeout(err) { + if isInactivityTimeout(err) || isConnectionLost(err) { return true } apiErr, ok := errors.AsType[*apierr.APIError](err) @@ -73,6 +73,18 @@ func isInactivityTimeout(err error) bool { return strings.HasPrefix(msg, "request timed out after ") && strings.HasSuffix(msg, " of inactivity") } +// isConnectionLost reports whether the HTTP/2 health check closed the connection because a ping went +// unanswered (see bundle/config/transport.go). It replaces the inactivity timeout for a request on +// a dead connection, with the same uncertainty about whether the server applied it, and the retry +// goes out on a new connection. +// +// We match the message because net/http builds it with errors.New in closeForLostPing and exports +// no sentinel error or type for it. +func isConnectionLost(err error) bool { + urlErr, ok := errors.AsType[*url.Error](err) + return ok && urlErr.Err.Error() == "http2: client connection lost" +} + // transientReadMessages are the transport errors, matched by message, that are worth retrying on // an idempotent call: // - REFUSED_STREAM, the HTTP/2 refusal in issue #6472, where the server refused the stream before diff --git a/libs/retry/retry_test.go b/libs/retry/retry_test.go index 396ad9e1803..f93c7d8ce18 100644 --- a/libs/retry/retry_test.go +++ b/libs/retry/retry_test.go @@ -34,6 +34,12 @@ func TestIsTransient(t *testing.T) { assert.True(t, IsTransientRead(ctx, inactivity)) cancelled := &url.Error{Op: "Post", URL: "https://example.test/api/2.0/workspace-files/import-file/a", Err: context.Canceled} assert.False(t, IsTransient(ctx, cancelled)) + + // The HTTP/2 health check closed a dead connection, which ends the request before the + // inactivity timeout would. + connLost := &url.Error{Op: "Post", URL: "https://example.test/api/2.0/workspace-files/import-file/a", Err: errors.New("http2: client connection lost")} + assert.True(t, IsTransient(ctx, connLost)) + assert.True(t, IsTransient(ctx, fmt.Errorf("failed to upload: %w", connLost))) } func TestIsTransientRead(t *testing.T) { @@ -47,6 +53,9 @@ func TestIsTransientRead(t *testing.T) { // The connection cut mid-response, seen on GET /clusters/get. unexpectedEOF := &url.Error{Op: "Get", URL: "https://example.test/api/2.1/clusters/get", Err: io.ErrUnexpectedEOF} assert.True(t, IsTransientRead(ctx, unexpectedEOF)) + // The HTTP/2 health check closed a dead connection; the retry dials a new one. + connLost := &url.Error{Op: "Post", URL: "https://example.test/api/2.0/workspace-files/import-file/a?overwrite=true", Err: errors.New("http2: client connection lost")} + assert.True(t, IsTransientRead(ctx, connLost)) // Transport errors the SDK already retries for its full RetryTimeout must not be retried again. connRefused := &url.Error{Op: "Get", URL: "https://example.test/api/2.0/workspace/get-status", Err: errors.New("dial tcp 10.0.0.1:443: connect: connection refused")} assert.False(t, IsTransientRead(ctx, connRefused))