From 3b8984e4be7a6ea2427eb815131f1f876bb4e68c Mon Sep 17 00:00:00 2001 From: "denis.bilenko" Date: Fri, 13 Nov 2026 16:13:53 +0000 Subject: [PATCH 1/5] filer: retry small non-seekable uploads to UC Volumes fs cp from a Volume or DBFS to a Volume streams the source download into the Files API upload. #7559 retries uploads only for seekable readers, so a single 503 (e.g. "Azure is temporarily unavailable; try again later.") on that upload still failed the copy. Buffer non-seekable bodies of up to 8 MiB so they can be replayed; larger streams keep one attempt. Co-authored-by: Isaac --- libs/filer/files_client.go | 32 +++++++++++++++++++++++++------- libs/filer/files_client_test.go | 28 +++++++++++++++++++++++++--- 2 files changed, 50 insertions(+), 10 deletions(-) diff --git a/libs/filer/files_client.go b/libs/filer/files_client.go index 25f8e8de344..fa4646095f8 100644 --- a/libs/filer/files_client.go +++ b/libs/filer/files_client.go @@ -1,6 +1,7 @@ package filer import ( + "bytes" "cmp" "context" "errors" @@ -312,18 +313,35 @@ func (w *FilesClient) Write(ctx context.Context, name string, reader io.Reader, return err } +// filesReplayBufferSize is the largest non-seekable upload that is buffered in +// memory so it can be retried. fs cp between remote paths (Volumes or DBFS to +// Volumes) streams the source download into the upload, so without the buffer +// a single 503 fails the copy; larger streams are not buffered to keep memory +// bounded and get one attempt. +const filesReplayBufferSize = 8 << 20 + // uploadFile sends reader in a single PUT. files/v2 rejects a retrier on this // call because it cannot replay a streamed body, so a seekable reader is rewound -// to its starting offset and retried here. Other readers get one attempt; that -// includes pipes, which are *os.File and so implement io.Seeker but fail Seek. +// to its starting offset and retried here. A non-seekable reader (including a +// pipe, which is an *os.File whose Seek fails) is buffered if it is at most +// filesReplayBufferSize bytes; a longer one gets one attempt. func (w *FilesClient) uploadFile(ctx context.Context, absPath string, reader io.Reader, overwrite bool) error { seeker, ok := reader.(io.Seeker) - if !ok { - return w.putFile(ctx, absPath, io.NopCloser(reader), overwrite) + var start int64 + var err error + if ok { + start, err = seeker.Seek(0, io.SeekCurrent) } - start, err := seeker.Seek(0, io.SeekCurrent) - if err != nil { - return w.putFile(ctx, absPath, io.NopCloser(reader), overwrite) + if !ok || err != nil { + head, err := io.ReadAll(io.LimitReader(reader, filesReplayBufferSize+1)) + if err != nil { + return err + } + if len(head) > filesReplayBufferSize { + return w.putFile(ctx, absPath, io.NopCloser(io.MultiReader(bytes.NewReader(head), reader)), overwrite) + } + buffered := bytes.NewReader(head) + reader, seeker, start = buffered, buffered, 0 } return ops.Execute(ctx, func(ctx context.Context) error { if _, err := seeker.Seek(start, io.SeekStart); err != nil { diff --git a/libs/filer/files_client_test.go b/libs/filer/files_client_test.go index 0d621d3e542..96d67dd04b9 100644 --- a/libs/filer/files_client_test.go +++ b/libs/filer/files_client_test.go @@ -312,14 +312,36 @@ func TestFilesClientRetriesThrottledUploadFromStartOffset(t *testing.T) { assert.Equal(t, []string{"abc", "abc", "abc"}, uploads) } -func TestFilesClientDoesNotRetryNonSeekableUpload(t *testing.T) { +func TestFilesClientRetriesSmallNonSeekableUpload(t *testing.T) { ctx := t.Context() server := testserver.New(t) - server.Handle("PUT", "/api/2.0/fs/files/{path...}", throttled(1, ok)) + var uploads []string + put := throttled(2, ok) + server.Handle("PUT", "/api/2.0/fs/files/{path...}", func(req testserver.Request) any { + uploads = append(uploads, string(req.Body)) + return put(req) + }) + f := newFilesClientWithFastRetries(t, server) + + require.NoError(t, f.Write(ctx, "hello", onlyReader{strings.NewReader("abc")}, CreateParentDirectories)) + assert.Equal(t, []string{"abc", "abc", "abc"}, uploads) +} + +func TestFilesClientDoesNotRetryLargeNonSeekableUpload(t *testing.T) { + ctx := t.Context() + server := testserver.New(t) + var sizes []int + put := throttled(1, ok) + server.Handle("PUT", "/api/2.0/fs/files/{path...}", func(req testserver.Request) any { + sizes = append(sizes, len(req.Body)) + return put(req) + }) f := newFilesClientWithFastRetries(t, server) - err := f.Write(ctx, "hello", onlyReader{strings.NewReader("abc")}, CreateParentDirectories) + content := strings.Repeat("x", filesReplayBufferSize+1) + err := f.Write(ctx, "hello", onlyReader{strings.NewReader(content)}, CreateParentDirectories) assert.ErrorContains(t, err, "AWS S3 is throttling requests") + assert.Equal(t, []int{len(content)}, sizes) } func TestFilesClientStopsRetryingAfterLimit(t *testing.T) { From a6fbc0ecb9ebd2200987d2aae8fccd4852cdcd7b Mon Sep 17 00:00:00 2001 From: "denis.bilenko" Date: Fri, 13 Nov 2026 16:14:48 +0000 Subject: [PATCH 2/5] Add changelog entry Co-authored-by: Isaac --- .nextchanges/cli/7607.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 .nextchanges/cli/7607.md diff --git a/.nextchanges/cli/7607.md b/.nextchanges/cli/7607.md new file mode 100644 index 00000000000..268ce6ee6b9 --- /dev/null +++ b/.nextchanges/cli/7607.md @@ -0,0 +1 @@ +* Retry `fs cp` uploads to UC Volumes from another Volume or from DBFS (files up to 8 MiB) when the Files API returns a transient error such as HTTP 503. ([#7607](https://github.com/databricks/cli/pull/7607)) From 4a696ee9074811b521c87c5cc204d871b3232b1a Mon Sep 17 00:00:00 2001 From: "denis.bilenko" Date: Fri, 13 Nov 2026 16:20:39 +0000 Subject: [PATCH 3/5] filer: split replay buffering into a helper, test boundary and pipe Co-authored-by: Isaac --- libs/filer/files_client.go | 61 +++++++++++++++++++-------------- libs/filer/files_client_test.go | 55 +++++++++++++++++++++-------- 2 files changed, 77 insertions(+), 39 deletions(-) diff --git a/libs/filer/files_client.go b/libs/filer/files_client.go index fa4646095f8..05957c34b10 100644 --- a/libs/filer/files_client.go +++ b/libs/filer/files_client.go @@ -5,6 +5,7 @@ import ( "cmp" "context" "errors" + "fmt" "io" "io/fs" "net/http" @@ -283,7 +284,8 @@ func (w *FilesClient) Write(ctx context.Context, name string, reader io.Reader, // upload engine, which sends small files in a single PUT and splits large ones // into parts. Non-seekable streams and the flag-off case use the single-shot // UploadFile endpoint below: the former to avoid buffering the whole stream in - // memory, the latter to keep the default behavior a plain PUT. The engine + // memory (uploadFile buffers only small ones, to retry them), the latter to + // keep the default behavior a plain PUT. The engine // recovers an io.ReaderAt for concurrent positioned reads when the source // provides one (a local file). if MultipartUploadEnabled(ctx) && isSeekable(reader) { @@ -316,38 +318,47 @@ func (w *FilesClient) Write(ctx context.Context, name string, reader io.Reader, // filesReplayBufferSize is the largest non-seekable upload that is buffered in // memory so it can be retried. fs cp between remote paths (Volumes or DBFS to // Volumes) streams the source download into the upload, so without the buffer -// a single 503 fails the copy; larger streams are not buffered to keep memory -// bounded and get one attempt. -const filesReplayBufferSize = 8 << 20 +// a single 503 fails the copy. Larger streams get one attempt, so memory per +// upload stays bounded. A variable so tests can lower it. +var filesReplayBufferSize = 8 << 20 + +// replayable returns a reader that can be rewound to start before each upload +// attempt. A non-seekable reader (including a pipe, which is an *os.File whose +// Seek fails) is read into memory if it is at most filesReplayBufferSize bytes; +// otherwise stream holds the bytes read so far followed by the +// rest of reader, to be sent once. +func replayable(reader io.Reader) (rs io.ReadSeeker, start int64, stream io.Reader, err error) { + if s, ok := reader.(io.ReadSeeker); ok { + if start, err := s.Seek(0, io.SeekCurrent); err == nil { + return s, start, nil, nil + } + } + head, err := io.ReadAll(io.LimitReader(reader, int64(filesReplayBufferSize)+1)) + if err != nil { + return nil, 0, nil, fmt.Errorf("reading upload body: %w", err) + } + if len(head) > filesReplayBufferSize { + return nil, 0, io.MultiReader(bytes.NewReader(head), reader), nil + } + return bytes.NewReader(head), 0, nil, nil +} // uploadFile sends reader in a single PUT. files/v2 rejects a retrier on this -// call because it cannot replay a streamed body, so a seekable reader is rewound -// to its starting offset and retried here. A non-seekable reader (including a -// pipe, which is an *os.File whose Seek fails) is buffered if it is at most -// filesReplayBufferSize bytes; a longer one gets one attempt. +// call because it cannot replay a streamed body, so the filer retries it here +// with a reader from replayable, rewound to its starting offset each attempt. func (w *FilesClient) uploadFile(ctx context.Context, absPath string, reader io.Reader, overwrite bool) error { - seeker, ok := reader.(io.Seeker) - var start int64 - var err error - if ok { - start, err = seeker.Seek(0, io.SeekCurrent) - } - if !ok || err != nil { - head, err := io.ReadAll(io.LimitReader(reader, filesReplayBufferSize+1)) - if err != nil { - return err - } - if len(head) > filesReplayBufferSize { - return w.putFile(ctx, absPath, io.NopCloser(io.MultiReader(bytes.NewReader(head), reader)), overwrite) - } - buffered := bytes.NewReader(head) - reader, seeker, start = buffered, buffered, 0 + seeker, start, stream, err := replayable(reader) + if err != nil { + return err + } + if stream != nil { + return w.putFile(ctx, absPath, io.NopCloser(stream), overwrite) } return ops.Execute(ctx, func(ctx context.Context) error { if _, err := seeker.Seek(start, io.SeekStart); err != nil { return err } - body := &attemptBody{r: reader} + body := &attemptBody{r: seeker} err := w.putFile(ctx, absPath, body, overwrite) body.Close() return err diff --git a/libs/filer/files_client_test.go b/libs/filer/files_client_test.go index 96d67dd04b9..270394175e5 100644 --- a/libs/filer/files_client_test.go +++ b/libs/filer/files_client_test.go @@ -312,36 +312,63 @@ func TestFilesClientRetriesThrottledUploadFromStartOffset(t *testing.T) { assert.Equal(t, []string{"abc", "abc", "abc"}, uploads) } -func TestFilesClientRetriesSmallNonSeekableUpload(t *testing.T) { - ctx := t.Context() - server := testserver.New(t) +// uploadsWithOneThrottle records each PUT body; the first PUT is throttled. +func uploadsWithOneThrottle(server *testserver.Server) *[]string { var uploads []string - put := throttled(2, ok) + put := throttled(1, ok) server.Handle("PUT", "/api/2.0/fs/files/{path...}", func(req testserver.Request) any { uploads = append(uploads, string(req.Body)) return put(req) }) + return &uploads +} + +func TestFilesClientRetriesSmallNonSeekableUpload(t *testing.T) { + ctx := t.Context() + server := testserver.New(t) + uploads := uploadsWithOneThrottle(server) f := newFilesClientWithFastRetries(t, server) + setReplayBufferSize(t, 3) + // Exactly filesReplayBufferSize bytes still fit in the buffer. require.NoError(t, f.Write(ctx, "hello", onlyReader{strings.NewReader("abc")}, CreateParentDirectories)) - assert.Equal(t, []string{"abc", "abc", "abc"}, uploads) + assert.Equal(t, []string{"abc", "abc"}, *uploads) +} + +func TestFilesClientRetriesSmallPipeUpload(t *testing.T) { + ctx := t.Context() + server := testserver.New(t) + uploads := uploadsWithOneThrottle(server) + f := newFilesClientWithFastRetries(t, server) + + // A pipe is an *os.File whose Seek fails. + pr, pw, err := os.Pipe() + require.NoError(t, err) + _, err = pw.WriteString("abc") + require.NoError(t, err) + require.NoError(t, pw.Close()) + defer pr.Close() + + require.NoError(t, f.Write(ctx, "hello", pr, CreateParentDirectories)) + assert.Equal(t, []string{"abc", "abc"}, *uploads) } func TestFilesClientDoesNotRetryLargeNonSeekableUpload(t *testing.T) { ctx := t.Context() server := testserver.New(t) - var sizes []int - put := throttled(1, ok) - server.Handle("PUT", "/api/2.0/fs/files/{path...}", func(req testserver.Request) any { - sizes = append(sizes, len(req.Body)) - return put(req) - }) + uploads := uploadsWithOneThrottle(server) f := newFilesClientWithFastRetries(t, server) + setReplayBufferSize(t, 2) - content := strings.Repeat("x", filesReplayBufferSize+1) - err := f.Write(ctx, "hello", onlyReader{strings.NewReader(content)}, CreateParentDirectories) + err := f.Write(ctx, "hello", onlyReader{strings.NewReader("abc")}, CreateParentDirectories) assert.ErrorContains(t, err, "AWS S3 is throttling requests") - assert.Equal(t, []int{len(content)}, sizes) + assert.Equal(t, []string{"abc"}, *uploads) +} + +func setReplayBufferSize(t *testing.T, n int) { + old := filesReplayBufferSize + filesReplayBufferSize = n + t.Cleanup(func() { filesReplayBufferSize = old }) } func TestFilesClientStopsRetryingAfterLimit(t *testing.T) { From 5270942bb68566bdc15476f7161b30180e2d3b06 Mon Sep 17 00:00:00 2001 From: "denis.bilenko" Date: Tue, 17 Nov 2026 11:34:01 +0000 Subject: [PATCH 4/5] filer: read buffered upload sources in 1 MiB chunks A DBFS read handle makes one /dbfs/read call per Read, and io.ReadAll asks for small growing chunks, so buffering a 1 MiB DBFS file took ~20 calls. Read through a 1 MiB bufio.Reader instead. Also update the cmd/fs comment that said Volume writes are not retried. Co-authored-by: Isaac --- cmd/fs/helpers.go | 6 ++++-- libs/filer/files_client.go | 12 ++++++++++-- libs/filer/files_client_test.go | 27 +++++++++++++++++++++++++++ 3 files changed, 41 insertions(+), 4 deletions(-) diff --git a/cmd/fs/helpers.go b/cmd/fs/helpers.go index 5a42f41b903..9a0c64764b4 100644 --- a/cmd/fs/helpers.go +++ b/cmd/fs/helpers.go @@ -52,8 +52,10 @@ func filerForPath(ctx context.Context, fullPath string) (filer.Filer, string, er return nil, "", err } - // Writes are not retried: a write without overwrite may fail on the file its own earlier - // attempt created, and an overwrite would need to buffer the streamed source to replay it. + // Writes are not retried here: a write without overwrite may fail on the file its own + // earlier attempt created, and replaying a streamed source needs buffering. The Files API + // filer (Volumes) retries its own uploads when the source is seekable or small enough + // to buffer; DBFS writes get one attempt. return retry.FilerExceptWrites(f), path, nil } diff --git a/libs/filer/files_client.go b/libs/filer/files_client.go index 05957c34b10..6625483f563 100644 --- a/libs/filer/files_client.go +++ b/libs/filer/files_client.go @@ -1,6 +1,7 @@ package filer import ( + "bufio" "bytes" "cmp" "context" @@ -322,6 +323,9 @@ func (w *FilesClient) Write(ctx context.Context, name string, reader io.Reader, // upload stays bounded. A variable so tests can lower it. var filesReplayBufferSize = 8 << 20 +// filesReplayReadSize is the size of the reads replayable makes from the source. +const filesReplayReadSize = 1 << 20 + // replayable returns a reader that can be rewound to start before each upload // attempt. A non-seekable reader (including a pipe, which is an *os.File whose // Seek fails) is read into memory if it is at most filesReplayBufferSize bytes; @@ -333,12 +337,16 @@ func replayable(reader io.Reader) (rs io.ReadSeeker, start int64, stream io.Read return s, start, nil, nil } } - head, err := io.ReadAll(io.LimitReader(reader, int64(filesReplayBufferSize)+1)) + // io.ReadAll asks for small, growing chunks, and a DBFS read handle makes one + // API call per Read; a 1 MiB buffer (the DBFS block size) keeps that to one + // call per MiB. + br := bufio.NewReaderSize(reader, filesReplayReadSize) + head, err := io.ReadAll(io.LimitReader(br, int64(filesReplayBufferSize)+1)) if err != nil { return nil, 0, nil, fmt.Errorf("reading upload body: %w", err) } if len(head) > filesReplayBufferSize { - return nil, 0, io.MultiReader(bytes.NewReader(head), reader), nil + return nil, 0, io.MultiReader(bytes.NewReader(head), br), nil } return bytes.NewReader(head), 0, nil, nil } diff --git a/libs/filer/files_client_test.go b/libs/filer/files_client_test.go index 270394175e5..0434f673150 100644 --- a/libs/filer/files_client_test.go +++ b/libs/filer/files_client_test.go @@ -365,6 +365,33 @@ func TestFilesClientDoesNotRetryLargeNonSeekableUpload(t *testing.T) { assert.Equal(t, []string{"abc"}, *uploads) } +// countingReader counts Read calls on a non-seekable reader, modelling a DBFS +// read handle that makes one API call per Read. +type countingReader struct { + r io.Reader + calls int +} + +func (c *countingReader) Read(p []byte) (int, error) { + c.calls++ + return c.r.Read(p) +} + +func TestFilesClientBuffersUploadInLargeReads(t *testing.T) { + ctx := t.Context() + server := testserver.New(t) + uploads := uploadsWithOneThrottle(server) + f := newFilesClientWithFastRetries(t, server) + + content := strings.Repeat("x", filesReplayReadSize) + src := &countingReader{r: strings.NewReader(content)} + require.NoError(t, f.Write(ctx, "hello", src, CreateParentDirectories)) + assert.Len(t, *uploads, 2) + // One Read fills the buffer and one more returns EOF; io.ReadAll's own + // growing chunks would take about 20. + assert.LessOrEqual(t, src.calls, 3) +} + func setReplayBufferSize(t *testing.T, n int) { old := filesReplayBufferSize filesReplayBufferSize = n From ea52c08f3d26ee81d80b837a8a510f2a1f32b007 Mon Sep 17 00:00:00 2001 From: "denis.bilenko" Date: Wed, 18 Nov 2026 11:20:19 +0000 Subject: [PATCH 5/5] filer: spell out replay buffer sizes and where they come from Co-authored-by: Isaac --- libs/filer/files_client.go | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/libs/filer/files_client.go b/libs/filer/files_client.go index 6625483f563..9ce13f53286 100644 --- a/libs/filer/files_client.go +++ b/libs/filer/files_client.go @@ -319,12 +319,17 @@ func (w *FilesClient) Write(ctx context.Context, name string, reader io.Reader, // filesReplayBufferSize is the largest non-seekable upload that is buffered in // memory so it can be retried. fs cp between remote paths (Volumes or DBFS to // Volumes) streams the source download into the upload, so without the buffer -// a single 503 fails the copy. Larger streams get one attempt, so memory per -// upload stays bounded. A variable so tests can lower it. -var filesReplayBufferSize = 8 << 20 - -// filesReplayReadSize is the size of the reads replayable makes from the source. -const filesReplayReadSize = 1 << 20 +// a single 503 fails the copy. Larger streams get one attempt. +// +// 8 MiB is not an API limit; it caps memory. With fs cp's default concurrency +// of 8, at most 64 MiB is buffered at once, and most files copied with fs cp +// (code, notebooks, configs) are well below it. A variable so tests can lower it. +var filesReplayBufferSize = 8 * 1024 * 1024 // 8 MiB + +// filesReplayReadSize is the size of the reads replayable makes from the +// source. It matches the DBFS read limit: /api/2.0/dbfs/read returns at most +// 1 MiB per call, and the SDK's DBFS read handle makes one call per Read. +const filesReplayReadSize = 1024 * 1024 // 1 MiB // replayable returns a reader that can be rewound to start before each upload // attempt. A non-seekable reader (including a pipe, which is an *os.File whose @@ -337,9 +342,8 @@ func replayable(reader io.Reader) (rs io.ReadSeeker, start int64, stream io.Read return s, start, nil, nil } } - // io.ReadAll asks for small, growing chunks, and a DBFS read handle makes one - // API call per Read; a 1 MiB buffer (the DBFS block size) keeps that to one - // call per MiB. + // io.ReadAll asks for small, growing chunks; reading through + // filesReplayReadSize keeps a DBFS source to one API call per MiB. br := bufio.NewReaderSize(reader, filesReplayReadSize) head, err := io.ReadAll(io.LimitReader(br, int64(filesReplayBufferSize)+1)) if err != nil {