Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .nextchanges/cli/files-api-empty-errors.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
* Retry read-only Files API calls (stat, read, list) on UC Volumes on HTTP 500, and show the HTTP status and path for Files API errors that have no message instead of an empty `databricks-api error:`. ([#7839](https://github.com/databricks/cli/pull/7839))
112 changes: 96 additions & 16 deletions libs/filer/files_client.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import (
"bytes"
"cmp"
"context"
"encoding/json"
"errors"
"fmt"
"io"
Expand Down Expand Up @@ -58,6 +59,16 @@ var filesRetryStatusCodes = []int{
http.StatusGatewayTimeout, // 504
}

// filesReadRetryStatusCodes add 500 to filesRetryStatusCodes for read-only
// calls (metadata, download, list): a retry cannot apply anything twice there,
// so a transient server error is worth retrying too. Writes and deletes are not
// retried on 500 because the failed attempt may have been applied.
var filesReadRetryStatusCodes = append(slices.Clone(filesRetryStatusCodes), http.StatusInternalServerError)

// filesMaxServerErrorRetries bounds the retries on 500 within one call, so a
// 500 that does not clear (e.g. on one broken path) still fails in seconds.
const filesMaxServerErrorRetries = 3

// filesMaxRetries bounds the retries of one call. Non-streaming files/v2 calls
// also have a 5 minute SDK timeout, but the single-shot upload retried by the
// filer itself does not.
Expand All @@ -80,6 +91,23 @@ func (r *cappedRetrier) IsRetriable(err error) (time.Duration, bool) {
return r.Retrier.IsRetriable(err)
}

// readRetrier is a cappedRetrier that also stops after
// filesMaxServerErrorRetries retries on 500.
type readRetrier struct {
cappedRetrier
serverErrors int
}

func (r *readRetrier) IsRetriable(err error) (time.Duration, bool) {
if httpStatus(err) == http.StatusInternalServerError {
if r.serverErrors >= filesMaxServerErrorRetries {
return 0, false
}
r.serverErrors++
}
return r.cappedRetrier.IsRetriable(err)
}

// As of 19th Feb 2024, the Files API backend has a rate limit of 10 concurrent
// requests and 100 QPS. We limit the number of concurrent requests to 5 to
// avoid hitting the rate limit.
Expand Down Expand Up @@ -204,11 +232,63 @@ func (w *FilesClient) newRetrier() ops.Retrier {
return &cappedRetrier{Retrier: apiretry.NewRetrier(w.retryBackoff, apiretry.RetrierConfig{StatusCodes: filesRetryStatusCodes})}
}

func (w *FilesClient) newReadRetrier() ops.Retrier {
inner := apiretry.NewRetrier(w.retryBackoff, apiretry.RetrierConfig{StatusCodes: filesReadRetryStatusCodes})
return &readRetrier{Retrier: inner}

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[blocking] readRetrier embeds cappedRetrier, which in turn embeds ops.Retrier. Retrier is therefore a promoted field of readRetrier, and Go forbids promoted fields as keys in composite literals ("Promoted fields act as if they were ordinary fields of a struct except that they cannot be used as field names in composite literals of the struct"). This is a compile error. Compare line 232 where cappedRetrier{Retrier: ...} works because Retrier is a direct field there. Should be &readRetrier{cappedRetrier: cappedRetrier{Retrier: inner}}.

}

// retry is the call option that retries a files/v2 call on throttling.
func (w *FilesClient) retry() call.Option {
return call.WithRetrier(w.newRetrier)
}

// readRetry is retry for read-only calls, which are also retried on 500.
func (w *FilesClient) readRetry() call.Option {
return call.WithRetrier(w.newReadRetrier)
}

// filesAPIError describes a Files API error that carries no message. The files/v2
// error text is "databricks-api error: " plus the message from the JSON body, so
// an error with no body (every failed HEAD) or a non-JSON one (e.g. from a proxy)
// would otherwise print as "databricks-api error: " with no status or path.
type filesAPIError struct {
op string
absPath string
err *apierr.APIError
}

func (e filesAPIError) Error() string {
status := e.err.HTTPStatusCode()
msg := fmt.Sprintf("%s %s: Files API returned HTTP %d", e.op, e.absPath, status)
// Non-standard codes such as 520 have no status text.
if text := http.StatusText(status); text != "" {
msg += " " + text
}
// A JSON body can carry an error_code without a message.
var body struct {
ErrorCode string `json:"error_code"`
}
if json.Unmarshal(e.err.HTTPBody(), &body) == nil && body.ErrorCode != "" {

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[nit] json.Unmarshal(e.err.HTTPBody(), &body) runs each time Error() is called (logging, fmt.Errorf wrapping, test assertions). Consider extracting the error_code once in describeFilesError and storing it in filesAPIError.

msg += " (" + body.ErrorCode + ")"
}
return msg
}

func (e filesAPIError) Unwrap() error {
return e.err
}

// describeFilesError returns err, or a [filesAPIError] for it if err is a
// files/v2 HTTP error without a message.
func describeFilesError(err error, op, absPath string) error {
// Only a bare error: a wrapped one already has context from its wrapper.
aerr, ok := err.(*apierr.APIError) //nolint:errorlint // must not match wrapped errors
if !ok || aerr.Message() != "" || aerr.HTTPStatusCode() < 0 {
return err
}
return filesAPIError{op: op, absPath: absPath, err: aerr}
}

func NewFilesClient(ctx context.Context, w *databricks.WorkspaceClient, root string) (Filer, error) {
c, err := newFilesAPIClient(ctx, w.Config)
if err != nil {
Expand Down Expand Up @@ -269,13 +349,13 @@ func (w *FilesClient) Write(ctx context.Context, name string, reader io.Reader,
// Check that target path exists if CreateParentDirectories mode is not set
if !slices.Contains(mode, CreateParentDirectories) {
dir := path.Dir(absPath)
_, err := w.client.GetDirectoryMetadata(ctx, files.GetDirectoryMetadataRequest{DirectoryPath: &dir}, w.retry())
_, err := w.client.GetDirectoryMetadata(ctx, files.GetDirectoryMetadataRequest{DirectoryPath: &dir}, w.readRetry())
if err != nil {
// This API returns a 404 if the directory doesn't exist.
if httpStatus(err) == http.StatusNotFound {
return noSuchDirectoryError{dir}
}
return err
return describeFilesError(err, "stat", dir)
}
}

Expand All @@ -301,7 +381,7 @@ func (w *FilesClient) Write(ctx context.Context, name string, reader io.Reader,
opts = append(opts, files.WithProgress(fn))
}
_, uerr := w.client.Upload(ctx, absPath, reader, opts...)
return mapUploadError(uerr, absPath)
return describeFilesError(mapUploadError(uerr, absPath), "write", absPath)
}

err = w.uploadFile(ctx, absPath, reader, overwrite)
Expand All @@ -313,7 +393,7 @@ func (w *FilesClient) Write(ctx context.Context, name string, reader io.Reader,
if httpStatus(err) == http.StatusConflict {
return fileAlreadyExistsError{absPath}
}
return err
return describeFilesError(err, "write", absPath)
}

// filesReplayBufferSize is the largest non-seekable upload that is buffered in
Expand Down Expand Up @@ -443,7 +523,7 @@ func (w *FilesClient) Read(ctx context.Context, name string) (io.ReadCloser, err
return nil, err
}

resp, err := w.client.DownloadFile(ctx, files.DownloadFileRequest{FilePath: &absPath}, w.retry())
resp, err := w.client.DownloadFile(ctx, files.DownloadFileRequest{FilePath: &absPath}, w.readRetry())

// Return early on success.
if err == nil {
Expand All @@ -461,7 +541,7 @@ func (w *FilesClient) Read(ctx context.Context, name string) (io.ReadCloser, err
return nil, fileDoesNotExistError{absPath}
}

return nil, err
return nil, describeFilesError(err, "read", absPath)
}

func (w *FilesClient) deleteFile(ctx context.Context, name string) error {
Expand All @@ -487,7 +567,7 @@ func (w *FilesClient) deleteFile(ctx context.Context, name string) error {
return fileDoesNotExistError{absPath}
}

return err
return describeFilesError(err, "delete", absPath)
}

func (w *FilesClient) deleteDirectory(ctx context.Context, name string) error {
Expand All @@ -514,7 +594,7 @@ func (w *FilesClient) deleteDirectory(ctx context.Context, name string) error {
if info := aerr.Details().ErrorInfo; info != nil && info.Reason == "FILES_API_DIRECTORY_IS_NOT_EMPTY" {
return directoryNotEmptyError{absPath}
}
return err
return describeFilesError(err, "delete", absPath)
}

// On GCS-backed storage a directory created implicitly is just a key prefix
Expand All @@ -525,7 +605,7 @@ func (w *FilesClient) deleteDirectory(ctx context.Context, name string) error {
if httpStatus(err) == http.StatusNotFound {
return noSuchDirectoryError{absPath}
}
return err
return describeFilesError(err, "delete", absPath)
}

func (w *FilesClient) recursiveDelete(ctx context.Context, name string) error {
Expand Down Expand Up @@ -625,7 +705,7 @@ func (w *FilesClient) ReadDir(ctx context.Context, name string) ([]fs.DirEntry,
var entries []fs.DirEntry
for entry, err := range w.client.ListDirectoryContentsIter(ctx, files.ListDirectoryContentsRequest{
DirectoryPath: &absPath,
}, w.retry()) {
}, w.readRetry()) {
if err != nil {
// This API returns a 404 if the specified path does not exist.
if httpStatus(err) == http.StatusNotFound {
Expand All @@ -637,7 +717,7 @@ func (w *FilesClient) ReadDir(ctx context.Context, name string) ([]fs.DirEntry,
// No file or directory exists at the specified path. Return no such directory error.
return nil, noSuchDirectoryError{absPath}
}
return nil, err
return nil, describeFilesError(err, "list", absPath)
}

entries = append(entries, filesApiDirEntry{
Expand Down Expand Up @@ -669,7 +749,7 @@ func (w *FilesClient) Mkdir(ctx context.Context, name string) error {
return fileAlreadyExistsError{absPath}
}

return err
return describeFilesError(err, "mkdir", absPath)
}

// Get file metadata for a file using the Files API.
Expand All @@ -679,7 +759,7 @@ func (w *FilesClient) statFile(ctx context.Context, name string) (fs.FileInfo, e
return nil, err
}

resp, err := w.client.GetFileMetadata(ctx, files.GetFileMetadataRequest{FilePath: &absPath}, w.retry())
resp, err := w.client.GetFileMetadata(ctx, files.GetFileMetadataRequest{FilePath: &absPath}, w.readRetry())

// If the HEAD requests succeeds, the file exists.
if err == nil {
Expand All @@ -695,7 +775,7 @@ func (w *FilesClient) statFile(ctx context.Context, name string) (fs.FileInfo, e
return nil, fileDoesNotExistError{absPath}
}

return nil, err
return nil, describeFilesError(err, "stat", absPath)
}

// Get file metadata for a directory using the Files API.
Expand All @@ -705,7 +785,7 @@ func (w *FilesClient) statDir(ctx context.Context, name string) (fs.FileInfo, er
return nil, err
}

_, err = w.client.GetDirectoryMetadata(ctx, files.GetDirectoryMetadataRequest{DirectoryPath: &absPath}, w.retry())
_, err = w.client.GetDirectoryMetadata(ctx, files.GetDirectoryMetadataRequest{DirectoryPath: &absPath}, w.readRetry())

// If the HEAD requests succeeds, the directory exists.
if err == nil {
Expand All @@ -717,7 +797,7 @@ func (w *FilesClient) statDir(ctx context.Context, name string) (fs.FileInfo, er
return nil, noSuchDirectoryError{absPath}
}

return nil, err
return nil, describeFilesError(err, "stat", absPath)
}

func (w *FilesClient) Stat(ctx context.Context, name string) (fs.FileInfo, error) {
Expand Down
106 changes: 103 additions & 3 deletions libs/filer/files_client_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import (
"fmt"
"io"
"io/fs"
"net/http"
"os"
"path/filepath"
"strings"
Expand All @@ -18,6 +19,7 @@ import (
"github.com/databricks/cli/libs/env"
"github.com/databricks/cli/libs/testserver"
"github.com/databricks/databricks-sdk-go"
"github.com/databricks/sdk-go/core/apierr"
"github.com/databricks/sdk-go/core/ops"
files "github.com/databricks/sdk-go/files/v2"
"github.com/stretchr/testify/assert"
Expand Down Expand Up @@ -232,15 +234,20 @@ var throttledResponse = testserver.Response{

// throttled wraps a handler so that its first n requests are throttled.
func throttled(n int, handler func(req testserver.Request) any) func(req testserver.Request) any {
return failFirst(n, throttledResponse, handler)
}

// failFirst wraps a handler so that its first n requests get resp.
func failFirst(n int, resp testserver.Response, handler func(req testserver.Request) any) func(req testserver.Request) any {
var mu sync.Mutex
count := 0
return func(req testserver.Request) any {
mu.Lock()
count++
throttle := count <= n
fail := count <= n
mu.Unlock()
if throttle {
return throttledResponse
if fail {
return resp
}
return handler(req)
}
Expand Down Expand Up @@ -413,3 +420,96 @@ func TestFilesClientStopsRetryingAfterLimit(t *testing.T) {
assert.ErrorContains(t, err, "AWS S3 is throttling requests")
assert.Equal(t, filesMaxRetries+1, requests)
}

// failing wraps a handler so that its first n requests fail with status and no
// body, as a failed HEAD (or a proxy error page) does.
func failing(n, status int, handler func(req testserver.Request) any) func(req testserver.Request) any {
return failFirst(n, testserver.Response{StatusCode: status, Body: []byte{}}, handler)
}

func TestFilesClientRetriesReadOnlyCallsOn500(t *testing.T) {
ctx := t.Context()
server := testserver.New(t)
server.Handle("HEAD", "/api/2.0/fs/directories/{path...}", failing(2, 500, ok))
server.Handle("HEAD", "/api/2.0/fs/files/{path...}", failing(2, 500, ok))
server.Handle("PUT", "/api/2.0/fs/files/{path...}", ok)
server.Handle("GET", "/api/2.0/fs/files/{path...}", failing(2, 500, func(testserver.Request) any {
return testserver.Response{Body: []byte("abc")}
}))
f := newFilesClientWithFastRetries(t, server)

// Write checks that the parent directory exists with a HEAD.
require.NoError(t, f.Write(ctx, "hello", strings.NewReader("abc")))

_, err := f.statFile(ctx, "hello")
require.NoError(t, err)

r, err := f.Read(ctx, "hello")
require.NoError(t, err)
data, err := io.ReadAll(r)
require.NoError(t, err)
assert.Equal(t, "abc", string(data))
require.NoError(t, r.Close())
}

func TestFilesClientDoesNotRetryDeleteOn500(t *testing.T) {
ctx := t.Context()
server := testserver.New(t)
requests := 0
server.Handle("DELETE", "/api/2.0/fs/files/{path...}", func(req testserver.Request) any {
requests++
return testserver.Response{StatusCode: 500, Body: []byte{}}
})
f := newFilesClientWithFastRetries(t, server)

err := f.deleteFile(ctx, "hello")
assert.EqualError(t, err, "delete /Volumes/main/schema/vol/hello: Files API returned HTTP 500 Internal Server Error")
assert.Equal(t, 1, requests)
}

func TestFilesClientDescribesErrorWithoutMessage(t *testing.T) {
ctx := t.Context()
server := testserver.New(t)
server.Handle("HEAD", "/api/2.0/fs/directories/{path...}", failing(filesMaxRetries+1, 403, ok))
f := newFilesClientWithFastRetries(t, server)

_, err := f.Stat(ctx, "dir")
assert.EqualError(t, err, "stat /Volumes/main/schema/vol/dir: Files API returned HTTP 403 Forbidden")
// The status stays available to callers that map errors by it.
assert.Equal(t, http.StatusForbidden, httpStatus(err))
}

func TestDescribeFilesErrorLeavesOtherErrors(t *testing.T) {
withMessage := apierr.FromHTTPError(500, nil, []byte(`{"error_code":"INTERNAL_ERROR","message":"boom"}`))
assert.Equal(t, "databricks-api error: boom", describeFilesError(withMessage, "stat", "/p").Error())

// A wrapped error already has context from its wrapper.
wrapped := fmt.Errorf("context: %w", apierr.FromHTTPError(500, nil, nil))
assert.Equal(t, wrapped, describeFilesError(wrapped, "stat", "/p"))

assert.NoError(t, describeFilesError(nil, "stat", "/p"))
}

func TestFilesClientStopsRetrying500(t *testing.T) {
ctx := t.Context()
server := testserver.New(t)
requests := 0
head := failing(filesMaxServerErrorRetries+1, 500, ok)
server.Handle("HEAD", "/api/2.0/fs/files/{path...}", func(req testserver.Request) any {
requests++
return head(req)
})
f := newFilesClientWithFastRetries(t, server)

_, err := f.statFile(ctx, "hello")
assert.EqualError(t, err, "stat /Volumes/main/schema/vol/hello: Files API returned HTTP 500 Internal Server Error")
assert.Equal(t, filesMaxServerErrorRetries+1, requests)
}

func TestFilesAPIErrorText(t *testing.T) {
errorCodeOnly := apierr.FromHTTPError(403, nil, []byte(`{"error_code":"PERMISSION_DENIED"}`))
assert.EqualError(t, describeFilesError(errorCodeOnly, "read", "/p"), "read /p: Files API returned HTTP 403 Forbidden (PERMISSION_DENIED)")

nonStandard := apierr.FromHTTPError(520, nil, []byte("<html>oops</html>"))
assert.EqualError(t, describeFilesError(nonStandard, "read", "/p"), "read /p: Files API returned HTTP 520")
}
Loading