Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .nextchanges/bundles/7723.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
* Close dead HTTP/2 connections during bundle commands, so retries after a request timeout go out on a new connection instead of hanging until the OS drops the old one. ([#7723](https://github.com/databricks/cli/pull/7723))
68 changes: 68 additions & 0 deletions bundle/config/transport.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,68 @@
package config

import (
"crypto/tls"
"net"
"net/http"
"runtime"
"sync"
"time"
)

// http2SendPingTimeout and http2PingTimeout make the HTTP/2 client ping a connection that has been
// silent for 30s and close it if the ping is not answered within 15s.
//
// Without them, a connection that dies without a FIN or RST (seen in CI as a single process's
// requests all hanging while other processes reach the same workspace fine) stays in the pool until
// the OS gives up on TCP, which takes ~15 minutes on Linux. A request that times out only resets its
// own stream, so the retries in libs/retry go out over the same dead connection and time out too.
const (
http2SendPingTimeout = 30 * time.Second
http2PingTimeout = 15 * time.Second
)

// newHealthCheckedTransport mirrors the SDK's default transport (httpclient.makeDefaultTransport,
// which is unexported) and adds the HTTP/2 health check. Keep the two in sync on SDK upgrades.
Comment thread
denik marked this conversation as resolved.
Resolved
// experimental/ssh/internal/client/releases.go (newHTTP11Transport) is another hand copy of the
// same defaults, HTTP/1.1-only; check it too.
//
// The bundle workspace client uses it, and so do workspace commands run in a bundle root:
// cmd/root/auth.go builds their client from the bundle client's config. The token-refresh client
// the SDK builds while resolving the config still uses the SDK default.
func newHealthCheckedTransport(insecureSkipVerify bool) *http.Transport {
t := &http.Transport{
Proxy: http.ProxyFromEnvironment,
DialContext: (&net.Dialer{
Timeout: 30 * time.Second,
KeepAlive: 30 * time.Second,
}).DialContext,
ForceAttemptHTTP2: true,
MaxIdleConns: 100,
MaxIdleConnsPerHost: runtime.GOMAXPROCS(0) + 1,
IdleConnTimeout: 180 * time.Second,
TLSHandshakeTimeout: 30 * time.Second,
ExpectContinueTimeout: 1 * time.Second,
HTTP2: &http.HTTP2Config{
SendPingTimeout: http2SendPingTimeout,
PingTimeout: http2PingTimeout,
},
}
if insecureSkipVerify {
// The SDK only applies InsecureSkipVerify to its own transport, so we carry it over.
t.TLSClientConfig = &tls.Config{InsecureSkipVerify: true} //nolint:gosec // the user opted in via skip_verify
}
return t
}

// The transports are shared, like the SDK's default one, so clients reuse pooled connections.
var (
healthCheckedTransport = sync.OnceValue(func() *http.Transport { return newHealthCheckedTransport(false) })
insecureHealthCheckedTransport = sync.OnceValue(func() *http.Transport { return newHealthCheckedTransport(true) })
)

func workspaceTransport(insecureSkipVerify bool) *http.Transport {
if insecureSkipVerify {
return insecureHealthCheckedTransport()
}
return healthCheckedTransport()
}
219 changes: 219 additions & 0 deletions bundle/config/transport_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,219 @@
package config

import (
"context"
"io"
"net"
"net/http"
"net/http/httptest"
"sync"
"sync/atomic"
"testing"
"time"

"github.com/databricks/cli/libs/databrickscfg"
"github.com/databricks/cli/libs/retry"
"github.com/databricks/databricks-sdk-go/config"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)

// blackholeProxy forwards TCP connections to target until blackhole is called. After that, the
// connections it already accepted stop forwarding bytes in both directions but stay open, like a
// connection silently dropped by a NAT or load balancer. New connections still work.
type blackholeProxy struct {
ln net.Listener
target string
mu sync.Mutex
links []*proxyLink
accepted atomic.Int32
}

type proxyLink struct {
client, server net.Conn
dead atomic.Bool
}

func newBlackholeProxy(t *testing.T, target string) *blackholeProxy {
ln, err := net.Listen("tcp", "127.0.0.1:0")
require.NoError(t, err)
p := &blackholeProxy{ln: ln, target: target}
t.Cleanup(p.close)
go p.serve()
return p
}

func (p *blackholeProxy) serve() {
for {
client, err := p.ln.Accept()
if err != nil {
return
}
server, err := net.Dial("tcp", p.target)
if err != nil {
client.Close()
continue
}
p.accepted.Add(1)
l := &proxyLink{client: client, server: server}
p.mu.Lock()
p.links = append(p.links, l)
p.mu.Unlock()
go l.pipe(server, client)
go l.pipe(client, server)
}
}

func (l *proxyLink) pipe(dst, src net.Conn) {
defer l.close()
buf := make([]byte, 32*1024)
for {
n, err := src.Read(buf)
if err != nil {
return
}
if l.dead.Load() {
continue
}
if _, err := dst.Write(buf[:n]); err != nil {
return
}
}
}

func (l *proxyLink) close() {
l.client.Close()
l.server.Close()
}

func (p *blackholeProxy) blackhole() {
p.mu.Lock()
defer p.mu.Unlock()
for _, l := range p.links {
l.dead.Store(true)
}
}

func (p *blackholeProxy) close() {
p.ln.Close()
p.mu.Lock()
defer p.mu.Unlock()
for _, l := range p.links {
l.close()
}
}

// setupDeadConnection returns a GET helper for an HTTP/2 server behind a blackholeProxy, after one
// successful request whose connection the proxy then blackholes.
func setupDeadConnection(t *testing.T, transport *http.Transport) (*blackholeProxy, func(time.Duration) (string, error)) {
srv := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
_, _ = io.WriteString(w, r.Proto)
}))
srv.EnableHTTP2 = true
srv.StartTLS()
t.Cleanup(srv.Close)

proxy := newBlackholeProxy(t, srv.Listener.Addr().String())

transport.TLSClientConfig = srv.Client().Transport.(*http.Transport).TLSClientConfig.Clone()
t.Cleanup(transport.CloseIdleConnections)
client := &http.Client{Transport: transport}
url := "https://" + proxy.ln.Addr().String() + "/"

get := func(timeout time.Duration) (string, error) {
ctx, cancel := context.WithTimeout(t.Context(), timeout)
defer cancel()
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
require.NoError(t, err)
resp, err := client.Do(req)
if err != nil {
return "", err
}
defer resp.Body.Close()
body, err := io.ReadAll(resp.Body)
return string(body), err
}

proto, err := get(5 * time.Second)
require.NoError(t, err)
require.Equal(t, "HTTP/2.0", proto)
require.EqualValues(t, 1, proxy.accepted.Load())

proxy.blackhole()
return proxy, get
}

func TestHealthCheckedTransportReplacesDeadConnection(t *testing.T) {
transport := newHealthCheckedTransport(false)
transport.HTTP2.SendPingTimeout = time.Second
transport.HTTP2.PingTimeout = time.Second
proxy, get := setupDeadConnection(t, transport)

// A request sent on the dead connection fails once the health check closes it, with an error
// libs/retry retries. If the health check closed the connection before the request was sent,
// the request already goes out on a new connection.
_, err := get(10 * time.Second)
if err != nil {
require.ErrorContains(t, err, "http2: client connection lost")
assert.True(t, retry.IsTransient(t.Context(), err))
}

proto, err := get(10 * time.Second)
require.NoError(t, err)
assert.Equal(t, "HTTP/2.0", proto)
assert.EqualValues(t, 2, proxy.accepted.Load())
}

// Documents the net/http behavior the health check works around: without it, a timed-out request
// leaves the dead connection in the pool and the next request (a retry) is sent on it again. If a
// Go release stops reusing such connections, this test fails and can be removed.
func TestTransportWithoutHealthCheckReusesDeadConnection(t *testing.T) {
transport := newHealthCheckedTransport(false)
transport.HTTP2 = nil
proxy, get := setupDeadConnection(t, transport)

_, err := get(500 * time.Millisecond)
require.ErrorIs(t, err, context.DeadlineExceeded)
_, err = get(500 * time.Millisecond)
require.ErrorIs(t, err, context.DeadlineExceeded)
assert.EqualValues(t, 1, proxy.accepted.Load())
}

func TestWorkspaceTransportIsShared(t *testing.T) {
secure := workspaceTransport(false)
insecure := workspaceTransport(true)
assert.Same(t, secure, workspaceTransport(false))
assert.Same(t, insecure, workspaceTransport(true))
assert.NotSame(t, secure, insecure)
assert.Nil(t, secure.TLSClientConfig)
assert.True(t, insecure.TLSClientConfig.InsecureSkipVerify)
}

func TestWorkspaceClientUsesHealthCheckedTransport(t *testing.T) {
setupWorkspaceTest(t)

err := databrickscfg.SaveToProfile(t.Context(), &config.Config{
Profile: "secure",
Host: "https://secure.cloud.databricks.test",
Token: "token",
})
require.NoError(t, err)
// skip_verify comes from the profile, so it is only known after the config is resolved.
err = databrickscfg.SaveToProfile(t.Context(), &config.Config{
Profile: "insecure",
Host: "https://insecure.cloud.databricks.test",
Token: "token",
InsecureSkipVerify: true,
})
require.NoError(t, err)

w := Workspace{Profile: "secure"}
client, err := w.Client(t.Context())
require.NoError(t, err)
assert.Same(t, workspaceTransport(false), client.Config.HTTPTransport)

w = Workspace{Profile: "insecure"}
client, err = w.Client(t.Context())
require.NoError(t, err)
assert.Same(t, workspaceTransport(true), client.Config.HTTPTransport)
}
3 changes: 3 additions & 0 deletions bundle/config/workspace.go
Original file line number Diff line number Diff line change
Expand Up @@ -247,5 +247,8 @@ func (w *Workspace) Client(ctx context.Context) (*databricks.WorkspaceClient, er
}
}

// Set after EnsureResolved, so skip_verify from the profile is known.
cfg.HTTPTransport = workspaceTransport(cfg.InsecureSkipVerify)

return databricks.NewWorkspaceClient((*databricks.Config)(cfg))
}
18 changes: 15 additions & 3 deletions libs/retry/retry.go
Original file line number Diff line number Diff line change
Expand Up @@ -38,12 +38,12 @@ func interval(ctx context.Context) time.Duration {
return time.Duration(ms) * time.Millisecond
}

// IsTransient returns true for 5xx and 408 errors that the SDK did not already retry, and for
// the SDK's client-side inactivity timeout.
// IsTransient returns true for 5xx and 408 errors that the SDK did not already retry, for the
// SDK's client-side inactivity timeout, and for an HTTP/2 connection closed by its health check.
// The SDK retries 504s matching its allTransientErrors patterns (e.g. "deadline exceeded");
// this covers the remaining 504s like TEMPORARILY_UNAVAILABLE.
func IsTransient(ctx context.Context, err error) bool {
if isInactivityTimeout(err) {
if isInactivityTimeout(err) || isConnectionLost(err) {
return true
}
apiErr, ok := errors.AsType[*apierr.APIError](err)
Expand Down Expand Up @@ -73,6 +73,18 @@ func isInactivityTimeout(err error) bool {
return strings.HasPrefix(msg, "request timed out after ") && strings.HasSuffix(msg, " of inactivity")
}

// isConnectionLost reports whether the HTTP/2 health check closed the connection because a ping went
// unanswered (see bundle/config/transport.go). It replaces the inactivity timeout for a request on
// a dead connection, with the same uncertainty about whether the server applied it, and the retry
// goes out on a new connection.
//
// We match the message because net/http builds it with errors.New in closeForLostPing and exports
// no sentinel error or type for it.
func isConnectionLost(err error) bool {
urlErr, ok := errors.AsType[*url.Error](err)
return ok && urlErr.Err.Error() == "http2: client connection lost"
Comment thread
denik marked this conversation as resolved.
Resolved
}

// transientReadMessages are the transport errors, matched by message, that are worth retrying on
// an idempotent call:
// - REFUSED_STREAM, the HTTP/2 refusal in issue #6472, where the server refused the stream before
Expand Down
9 changes: 9 additions & 0 deletions libs/retry/retry_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,12 @@ func TestIsTransient(t *testing.T) {
assert.True(t, IsTransientRead(ctx, inactivity))
cancelled := &url.Error{Op: "Post", URL: "https://example.test/api/2.0/workspace-files/import-file/a", Err: context.Canceled}
assert.False(t, IsTransient(ctx, cancelled))

// The HTTP/2 health check closed a dead connection, which ends the request before the
// inactivity timeout would.
connLost := &url.Error{Op: "Post", URL: "https://example.test/api/2.0/workspace-files/import-file/a", Err: errors.New("http2: client connection lost")}
assert.True(t, IsTransient(ctx, connLost))
assert.True(t, IsTransient(ctx, fmt.Errorf("failed to upload: %w", connLost)))
}

func TestIsTransientRead(t *testing.T) {
Expand All @@ -47,6 +53,9 @@ func TestIsTransientRead(t *testing.T) {
// The connection cut mid-response, seen on GET /clusters/get.
unexpectedEOF := &url.Error{Op: "Get", URL: "https://example.test/api/2.1/clusters/get", Err: io.ErrUnexpectedEOF}
assert.True(t, IsTransientRead(ctx, unexpectedEOF))
// The HTTP/2 health check closed a dead connection; the retry dials a new one.
connLost := &url.Error{Op: "Post", URL: "https://example.test/api/2.0/workspace-files/import-file/a?overwrite=true", Err: errors.New("http2: client connection lost")}
assert.True(t, IsTransientRead(ctx, connLost))
// Transport errors the SDK already retries for its full RetryTimeout must not be retried again.
connRefused := &url.Error{Op: "Get", URL: "https://example.test/api/2.0/workspace/get-status", Err: errors.New("dial tcp 10.0.0.1:443: connect: connection refused")}
assert.False(t, IsTransientRead(ctx, connRefused))
Expand Down
Loading