From 9b8a6d13cda15adc9340100f0518b3c817f8c18e Mon Sep 17 00:00:00 2001 From: "denis.bilenko" Date: Fri, 20 Nov 2026 08:21:42 +0000 Subject: [PATCH 1/2] filer: describe Files API errors without a message, retry reads on 500 files/v2 errors print as "databricks-api error: " plus the message from the JSON body, so a failed HEAD (no body) or a non-JSON error page prints nothing else. Show the operation, path and HTTP status instead, and retry read-only calls (metadata, download, list) on 500 as well. Co-authored-by: Isaac --- .nextchanges/cli/files-api-empty-errors.md | 1 + libs/filer/files_client.go | 77 +++++++++++++++----- libs/filer/files_client_test.go | 82 ++++++++++++++++++++++ 3 files changed, 144 insertions(+), 16 deletions(-) create mode 100644 .nextchanges/cli/files-api-empty-errors.md diff --git a/.nextchanges/cli/files-api-empty-errors.md b/.nextchanges/cli/files-api-empty-errors.md new file mode 100644 index 00000000000..170295adcca --- /dev/null +++ b/.nextchanges/cli/files-api-empty-errors.md @@ -0,0 +1 @@ +* Retry read-only Files API calls (stat, read, list) on UC Volumes on HTTP 500, and show the HTTP status and path for Files API errors that have no message instead of an empty `databricks-api error:`. diff --git a/libs/filer/files_client.go b/libs/filer/files_client.go index 9ce13f53286..5927a13c0e6 100644 --- a/libs/filer/files_client.go +++ b/libs/filer/files_client.go @@ -58,6 +58,12 @@ var filesRetryStatusCodes = []int{ http.StatusGatewayTimeout, // 504 } +// filesReadRetryStatusCodes add 500 to filesRetryStatusCodes for read-only +// calls (metadata, download, list): a retry cannot apply anything twice there, +// so a transient server error is worth retrying too. Writes and deletes are not +// retried on 500 because the failed attempt may have been applied. +var filesReadRetryStatusCodes = append(slices.Clone(filesRetryStatusCodes), http.StatusInternalServerError) + // filesMaxRetries bounds the retries of one call. Non-streaming files/v2 calls // also have a 5 minute SDK timeout, but the single-shot upload retried by the // filer itself does not. @@ -201,7 +207,11 @@ type FilesClient struct { } func (w *FilesClient) newRetrier() ops.Retrier { - return &cappedRetrier{Retrier: apiretry.NewRetrier(w.retryBackoff, apiretry.RetrierConfig{StatusCodes: filesRetryStatusCodes})} + return w.newRetrierFor(filesRetryStatusCodes) +} + +func (w *FilesClient) newRetrierFor(statusCodes []int) ops.Retrier { + return &cappedRetrier{Retrier: apiretry.NewRetrier(w.retryBackoff, apiretry.RetrierConfig{StatusCodes: statusCodes})} } // retry is the call option that retries a files/v2 call on throttling. @@ -209,6 +219,41 @@ func (w *FilesClient) retry() call.Option { return call.WithRetrier(w.newRetrier) } +// readRetry is retry for read-only calls, which are also retried on 500. +func (w *FilesClient) readRetry() call.Option { + return call.WithRetrier(func() ops.Retrier { return w.newRetrierFor(filesReadRetryStatusCodes) }) +} + +// filesAPIError describes a Files API error that carries no message. The files/v2 +// error text is "databricks-api error: " plus the message from the JSON body, so +// an error with no body (every failed HEAD) or a non-JSON one (e.g. from a proxy) +// would otherwise print as "databricks-api error: " with no status or path. +type filesAPIError struct { + op string + absPath string + err *apierr.APIError +} + +func (e filesAPIError) Error() string { + status := e.err.HTTPStatusCode() + return fmt.Sprintf("%s %s: Files API returned HTTP %d %s", e.op, e.absPath, status, http.StatusText(status)) +} + +func (e filesAPIError) Unwrap() error { + return e.err +} + +// describeFilesError returns err, or a [filesAPIError] for it if err is a +// files/v2 HTTP error without a message. +func describeFilesError(err error, op, absPath string) error { + // Only a bare error: a wrapped one already has context from its wrapper. + aerr, ok := err.(*apierr.APIError) //nolint:errorlint // must not match wrapped errors + if !ok || aerr.Message() != "" || aerr.HTTPStatusCode() < 0 { + return err + } + return filesAPIError{op: op, absPath: absPath, err: aerr} +} + func NewFilesClient(ctx context.Context, w *databricks.WorkspaceClient, root string) (Filer, error) { c, err := newFilesAPIClient(ctx, w.Config) if err != nil { @@ -269,13 +314,13 @@ func (w *FilesClient) Write(ctx context.Context, name string, reader io.Reader, // Check that target path exists if CreateParentDirectories mode is not set if !slices.Contains(mode, CreateParentDirectories) { dir := path.Dir(absPath) - _, err := w.client.GetDirectoryMetadata(ctx, files.GetDirectoryMetadataRequest{DirectoryPath: &dir}, w.retry()) + _, err := w.client.GetDirectoryMetadata(ctx, files.GetDirectoryMetadataRequest{DirectoryPath: &dir}, w.readRetry()) if err != nil { // This API returns a 404 if the directory doesn't exist. if httpStatus(err) == http.StatusNotFound { return noSuchDirectoryError{dir} } - return err + return describeFilesError(err, "stat", dir) } } @@ -313,7 +358,7 @@ func (w *FilesClient) Write(ctx context.Context, name string, reader io.Reader, if httpStatus(err) == http.StatusConflict { return fileAlreadyExistsError{absPath} } - return err + return describeFilesError(err, "write", absPath) } // filesReplayBufferSize is the largest non-seekable upload that is buffered in @@ -443,7 +488,7 @@ func (w *FilesClient) Read(ctx context.Context, name string) (io.ReadCloser, err return nil, err } - resp, err := w.client.DownloadFile(ctx, files.DownloadFileRequest{FilePath: &absPath}, w.retry()) + resp, err := w.client.DownloadFile(ctx, files.DownloadFileRequest{FilePath: &absPath}, w.readRetry()) // Return early on success. if err == nil { @@ -461,7 +506,7 @@ func (w *FilesClient) Read(ctx context.Context, name string) (io.ReadCloser, err return nil, fileDoesNotExistError{absPath} } - return nil, err + return nil, describeFilesError(err, "read", absPath) } func (w *FilesClient) deleteFile(ctx context.Context, name string) error { @@ -487,7 +532,7 @@ func (w *FilesClient) deleteFile(ctx context.Context, name string) error { return fileDoesNotExistError{absPath} } - return err + return describeFilesError(err, "delete", absPath) } func (w *FilesClient) deleteDirectory(ctx context.Context, name string) error { @@ -514,7 +559,7 @@ func (w *FilesClient) deleteDirectory(ctx context.Context, name string) error { if info := aerr.Details().ErrorInfo; info != nil && info.Reason == "FILES_API_DIRECTORY_IS_NOT_EMPTY" { return directoryNotEmptyError{absPath} } - return err + return describeFilesError(err, "delete", absPath) } // On GCS-backed storage a directory created implicitly is just a key prefix @@ -525,7 +570,7 @@ func (w *FilesClient) deleteDirectory(ctx context.Context, name string) error { if httpStatus(err) == http.StatusNotFound { return noSuchDirectoryError{absPath} } - return err + return describeFilesError(err, "delete", absPath) } func (w *FilesClient) recursiveDelete(ctx context.Context, name string) error { @@ -625,7 +670,7 @@ func (w *FilesClient) ReadDir(ctx context.Context, name string) ([]fs.DirEntry, var entries []fs.DirEntry for entry, err := range w.client.ListDirectoryContentsIter(ctx, files.ListDirectoryContentsRequest{ DirectoryPath: &absPath, - }, w.retry()) { + }, w.readRetry()) { if err != nil { // This API returns a 404 if the specified path does not exist. if httpStatus(err) == http.StatusNotFound { @@ -637,7 +682,7 @@ func (w *FilesClient) ReadDir(ctx context.Context, name string) ([]fs.DirEntry, // No file or directory exists at the specified path. Return no such directory error. return nil, noSuchDirectoryError{absPath} } - return nil, err + return nil, describeFilesError(err, "list", absPath) } entries = append(entries, filesApiDirEntry{ @@ -669,7 +714,7 @@ func (w *FilesClient) Mkdir(ctx context.Context, name string) error { return fileAlreadyExistsError{absPath} } - return err + return describeFilesError(err, "mkdir", absPath) } // Get file metadata for a file using the Files API. @@ -679,7 +724,7 @@ func (w *FilesClient) statFile(ctx context.Context, name string) (fs.FileInfo, e return nil, err } - resp, err := w.client.GetFileMetadata(ctx, files.GetFileMetadataRequest{FilePath: &absPath}, w.retry()) + resp, err := w.client.GetFileMetadata(ctx, files.GetFileMetadataRequest{FilePath: &absPath}, w.readRetry()) // If the HEAD requests succeeds, the file exists. if err == nil { @@ -695,7 +740,7 @@ func (w *FilesClient) statFile(ctx context.Context, name string) (fs.FileInfo, e return nil, fileDoesNotExistError{absPath} } - return nil, err + return nil, describeFilesError(err, "stat", absPath) } // Get file metadata for a directory using the Files API. @@ -705,7 +750,7 @@ func (w *FilesClient) statDir(ctx context.Context, name string) (fs.FileInfo, er return nil, err } - _, err = w.client.GetDirectoryMetadata(ctx, files.GetDirectoryMetadataRequest{DirectoryPath: &absPath}, w.retry()) + _, err = w.client.GetDirectoryMetadata(ctx, files.GetDirectoryMetadataRequest{DirectoryPath: &absPath}, w.readRetry()) // If the HEAD requests succeeds, the directory exists. if err == nil { @@ -717,7 +762,7 @@ func (w *FilesClient) statDir(ctx context.Context, name string) (fs.FileInfo, er return nil, noSuchDirectoryError{absPath} } - return nil, err + return nil, describeFilesError(err, "stat", absPath) } func (w *FilesClient) Stat(ctx context.Context, name string) (fs.FileInfo, error) { diff --git a/libs/filer/files_client_test.go b/libs/filer/files_client_test.go index 0434f673150..5b2c3754aae 100644 --- a/libs/filer/files_client_test.go +++ b/libs/filer/files_client_test.go @@ -6,6 +6,7 @@ import ( "fmt" "io" "io/fs" + "net/http" "os" "path/filepath" "strings" @@ -18,6 +19,7 @@ import ( "github.com/databricks/cli/libs/env" "github.com/databricks/cli/libs/testserver" "github.com/databricks/databricks-sdk-go" + "github.com/databricks/sdk-go/core/apierr" "github.com/databricks/sdk-go/core/ops" files "github.com/databricks/sdk-go/files/v2" "github.com/stretchr/testify/assert" @@ -413,3 +415,83 @@ func TestFilesClientStopsRetryingAfterLimit(t *testing.T) { assert.ErrorContains(t, err, "AWS S3 is throttling requests") assert.Equal(t, filesMaxRetries+1, requests) } + +// failing wraps a handler so that its first n requests fail with status and no +// body, as a failed HEAD (or a proxy error page) does. +func failing(n, status int, handler func(req testserver.Request) any) func(req testserver.Request) any { + var mu sync.Mutex + count := 0 + return func(req testserver.Request) any { + mu.Lock() + count++ + fail := count <= n + mu.Unlock() + if fail { + return testserver.Response{StatusCode: status, Body: []byte{}} + } + return handler(req) + } +} + +func TestFilesClientRetriesReadOnlyCallsOn500(t *testing.T) { + ctx := t.Context() + server := testserver.New(t) + server.Handle("HEAD", "/api/2.0/fs/directories/{path...}", failing(2, 500, ok)) + server.Handle("HEAD", "/api/2.0/fs/files/{path...}", failing(2, 500, ok)) + server.Handle("PUT", "/api/2.0/fs/files/{path...}", ok) + server.Handle("GET", "/api/2.0/fs/files/{path...}", failing(2, 500, func(testserver.Request) any { + return testserver.Response{Body: []byte("abc")} + })) + f := newFilesClientWithFastRetries(t, server) + + // Write checks that the parent directory exists with a HEAD. + require.NoError(t, f.Write(ctx, "hello", strings.NewReader("abc"))) + + _, err := f.statFile(ctx, "hello") + require.NoError(t, err) + + r, err := f.Read(ctx, "hello") + require.NoError(t, err) + data, err := io.ReadAll(r) + require.NoError(t, err) + assert.Equal(t, "abc", string(data)) + require.NoError(t, r.Close()) +} + +func TestFilesClientDoesNotRetryDeleteOn500(t *testing.T) { + ctx := t.Context() + server := testserver.New(t) + requests := 0 + server.Handle("DELETE", "/api/2.0/fs/files/{path...}", func(req testserver.Request) any { + requests++ + return testserver.Response{StatusCode: 500, Body: []byte{}} + }) + f := newFilesClientWithFastRetries(t, server) + + err := f.deleteFile(ctx, "hello") + assert.EqualError(t, err, "delete /Volumes/main/schema/vol/hello: Files API returned HTTP 500 Internal Server Error") + assert.Equal(t, 1, requests) +} + +func TestFilesClientDescribesErrorWithoutMessage(t *testing.T) { + ctx := t.Context() + server := testserver.New(t) + server.Handle("HEAD", "/api/2.0/fs/directories/{path...}", failing(filesMaxRetries+1, 403, ok)) + f := newFilesClientWithFastRetries(t, server) + + _, err := f.Stat(ctx, "dir") + assert.EqualError(t, err, "stat /Volumes/main/schema/vol/dir: Files API returned HTTP 403 Forbidden") + // The status stays available to callers that map errors by it. + assert.Equal(t, http.StatusForbidden, httpStatus(err)) +} + +func TestDescribeFilesErrorLeavesOtherErrors(t *testing.T) { + withMessage := apierr.FromHTTPError(500, nil, []byte(`{"error_code":"INTERNAL_ERROR","message":"boom"}`)) + assert.Equal(t, "databricks-api error: boom", describeFilesError(withMessage, "stat", "/p").Error()) + + // A wrapped error already has context from its wrapper. + wrapped := fmt.Errorf("context: %w", apierr.FromHTTPError(500, nil, nil)) + assert.Equal(t, wrapped, describeFilesError(wrapped, "stat", "/p")) + + assert.NoError(t, describeFilesError(nil, "stat", "/p")) +} From 3fd4cb7d2b878df7727453d5841e709248f35f77 Mon Sep 17 00:00:00 2001 From: "denis.bilenko" Date: Fri, 20 Nov 2026 08:45:30 +0000 Subject: [PATCH 2/2] filer: cap 500 retries on reads, cover multipart uploads, show error_code Address review: stop after 3 retries on 500 so a 500 that does not clear fails in seconds; describe errors from the multipart upload path too; include the body's error_code when there is one; no trailing space for non-standard status codes; changelog PR link. Co-authored-by: Isaac --- .nextchanges/cli/files-api-empty-errors.md | 2 +- libs/filer/files_client.go | 47 ++++++++++++++++++--- libs/filer/files_client_test.go | 48 +++++++++++++++------- 3 files changed, 75 insertions(+), 22 deletions(-) diff --git a/.nextchanges/cli/files-api-empty-errors.md b/.nextchanges/cli/files-api-empty-errors.md index 170295adcca..d6f1d919f5f 100644 --- a/.nextchanges/cli/files-api-empty-errors.md +++ b/.nextchanges/cli/files-api-empty-errors.md @@ -1 +1 @@ -* Retry read-only Files API calls (stat, read, list) on UC Volumes on HTTP 500, and show the HTTP status and path for Files API errors that have no message instead of an empty `databricks-api error:`. +* Retry read-only Files API calls (stat, read, list) on UC Volumes on HTTP 500, and show the HTTP status and path for Files API errors that have no message instead of an empty `databricks-api error:`. ([#7839](https://github.com/databricks/cli/pull/7839)) diff --git a/libs/filer/files_client.go b/libs/filer/files_client.go index 5927a13c0e6..b6fd7c4fe4a 100644 --- a/libs/filer/files_client.go +++ b/libs/filer/files_client.go @@ -5,6 +5,7 @@ import ( "bytes" "cmp" "context" + "encoding/json" "errors" "fmt" "io" @@ -64,6 +65,10 @@ var filesRetryStatusCodes = []int{ // retried on 500 because the failed attempt may have been applied. var filesReadRetryStatusCodes = append(slices.Clone(filesRetryStatusCodes), http.StatusInternalServerError) +// filesMaxServerErrorRetries bounds the retries on 500 within one call, so a +// 500 that does not clear (e.g. on one broken path) still fails in seconds. +const filesMaxServerErrorRetries = 3 + // filesMaxRetries bounds the retries of one call. Non-streaming files/v2 calls // also have a 5 minute SDK timeout, but the single-shot upload retried by the // filer itself does not. @@ -86,6 +91,23 @@ func (r *cappedRetrier) IsRetriable(err error) (time.Duration, bool) { return r.Retrier.IsRetriable(err) } +// readRetrier is a cappedRetrier that also stops after +// filesMaxServerErrorRetries retries on 500. +type readRetrier struct { + cappedRetrier + serverErrors int +} + +func (r *readRetrier) IsRetriable(err error) (time.Duration, bool) { + if httpStatus(err) == http.StatusInternalServerError { + if r.serverErrors >= filesMaxServerErrorRetries { + return 0, false + } + r.serverErrors++ + } + return r.cappedRetrier.IsRetriable(err) +} + // As of 19th Feb 2024, the Files API backend has a rate limit of 10 concurrent // requests and 100 QPS. We limit the number of concurrent requests to 5 to // avoid hitting the rate limit. @@ -207,11 +229,12 @@ type FilesClient struct { } func (w *FilesClient) newRetrier() ops.Retrier { - return w.newRetrierFor(filesRetryStatusCodes) + return &cappedRetrier{Retrier: apiretry.NewRetrier(w.retryBackoff, apiretry.RetrierConfig{StatusCodes: filesRetryStatusCodes})} } -func (w *FilesClient) newRetrierFor(statusCodes []int) ops.Retrier { - return &cappedRetrier{Retrier: apiretry.NewRetrier(w.retryBackoff, apiretry.RetrierConfig{StatusCodes: statusCodes})} +func (w *FilesClient) newReadRetrier() ops.Retrier { + inner := apiretry.NewRetrier(w.retryBackoff, apiretry.RetrierConfig{StatusCodes: filesReadRetryStatusCodes}) + return &readRetrier{Retrier: inner} } // retry is the call option that retries a files/v2 call on throttling. @@ -221,7 +244,7 @@ func (w *FilesClient) retry() call.Option { // readRetry is retry for read-only calls, which are also retried on 500. func (w *FilesClient) readRetry() call.Option { - return call.WithRetrier(func() ops.Retrier { return w.newRetrierFor(filesReadRetryStatusCodes) }) + return call.WithRetrier(w.newReadRetrier) } // filesAPIError describes a Files API error that carries no message. The files/v2 @@ -236,7 +259,19 @@ type filesAPIError struct { func (e filesAPIError) Error() string { status := e.err.HTTPStatusCode() - return fmt.Sprintf("%s %s: Files API returned HTTP %d %s", e.op, e.absPath, status, http.StatusText(status)) + msg := fmt.Sprintf("%s %s: Files API returned HTTP %d", e.op, e.absPath, status) + // Non-standard codes such as 520 have no status text. + if text := http.StatusText(status); text != "" { + msg += " " + text + } + // A JSON body can carry an error_code without a message. + var body struct { + ErrorCode string `json:"error_code"` + } + if json.Unmarshal(e.err.HTTPBody(), &body) == nil && body.ErrorCode != "" { + msg += " (" + body.ErrorCode + ")" + } + return msg } func (e filesAPIError) Unwrap() error { @@ -346,7 +381,7 @@ func (w *FilesClient) Write(ctx context.Context, name string, reader io.Reader, opts = append(opts, files.WithProgress(fn)) } _, uerr := w.client.Upload(ctx, absPath, reader, opts...) - return mapUploadError(uerr, absPath) + return describeFilesError(mapUploadError(uerr, absPath), "write", absPath) } err = w.uploadFile(ctx, absPath, reader, overwrite) diff --git a/libs/filer/files_client_test.go b/libs/filer/files_client_test.go index 5b2c3754aae..3330083ad04 100644 --- a/libs/filer/files_client_test.go +++ b/libs/filer/files_client_test.go @@ -234,15 +234,20 @@ var throttledResponse = testserver.Response{ // throttled wraps a handler so that its first n requests are throttled. func throttled(n int, handler func(req testserver.Request) any) func(req testserver.Request) any { + return failFirst(n, throttledResponse, handler) +} + +// failFirst wraps a handler so that its first n requests get resp. +func failFirst(n int, resp testserver.Response, handler func(req testserver.Request) any) func(req testserver.Request) any { var mu sync.Mutex count := 0 return func(req testserver.Request) any { mu.Lock() count++ - throttle := count <= n + fail := count <= n mu.Unlock() - if throttle { - return throttledResponse + if fail { + return resp } return handler(req) } @@ -419,18 +424,7 @@ func TestFilesClientStopsRetryingAfterLimit(t *testing.T) { // failing wraps a handler so that its first n requests fail with status and no // body, as a failed HEAD (or a proxy error page) does. func failing(n, status int, handler func(req testserver.Request) any) func(req testserver.Request) any { - var mu sync.Mutex - count := 0 - return func(req testserver.Request) any { - mu.Lock() - count++ - fail := count <= n - mu.Unlock() - if fail { - return testserver.Response{StatusCode: status, Body: []byte{}} - } - return handler(req) - } + return failFirst(n, testserver.Response{StatusCode: status, Body: []byte{}}, handler) } func TestFilesClientRetriesReadOnlyCallsOn500(t *testing.T) { @@ -495,3 +489,27 @@ func TestDescribeFilesErrorLeavesOtherErrors(t *testing.T) { assert.NoError(t, describeFilesError(nil, "stat", "/p")) } + +func TestFilesClientStopsRetrying500(t *testing.T) { + ctx := t.Context() + server := testserver.New(t) + requests := 0 + head := failing(filesMaxServerErrorRetries+1, 500, ok) + server.Handle("HEAD", "/api/2.0/fs/files/{path...}", func(req testserver.Request) any { + requests++ + return head(req) + }) + f := newFilesClientWithFastRetries(t, server) + + _, err := f.statFile(ctx, "hello") + assert.EqualError(t, err, "stat /Volumes/main/schema/vol/hello: Files API returned HTTP 500 Internal Server Error") + assert.Equal(t, filesMaxServerErrorRetries+1, requests) +} + +func TestFilesAPIErrorText(t *testing.T) { + errorCodeOnly := apierr.FromHTTPError(403, nil, []byte(`{"error_code":"PERMISSION_DENIED"}`)) + assert.EqualError(t, describeFilesError(errorCodeOnly, "read", "/p"), "read /p: Files API returned HTTP 403 Forbidden (PERMISSION_DENIED)") + + nonStandard := apierr.FromHTTPError(520, nil, []byte("oops")) + assert.EqualError(t, describeFilesError(nonStandard, "read", "/p"), "read /p: Files API returned HTTP 520") +}