diff --git a/.nextchanges/cli/7607.md b/.nextchanges/cli/7607.md new file mode 100644 index 00000000000..268ce6ee6b9 --- /dev/null +++ b/.nextchanges/cli/7607.md @@ -0,0 +1 @@ +* Retry `fs cp` uploads to UC Volumes from another Volume or from DBFS (files up to 8 MiB) when the Files API returns a transient error such as HTTP 503. ([#7607](https://github.com/databricks/cli/pull/7607)) diff --git a/cmd/fs/helpers.go b/cmd/fs/helpers.go index 5a42f41b903..9a0c64764b4 100644 --- a/cmd/fs/helpers.go +++ b/cmd/fs/helpers.go @@ -52,8 +52,10 @@ func filerForPath(ctx context.Context, fullPath string) (filer.Filer, string, er return nil, "", err } - // Writes are not retried: a write without overwrite may fail on the file its own earlier - // attempt created, and an overwrite would need to buffer the streamed source to replay it. + // Writes are not retried here: a write without overwrite may fail on the file its own + // earlier attempt created, and replaying a streamed source needs buffering. The Files API + // filer (Volumes) retries its own uploads when the source is seekable or small enough + // to buffer; DBFS writes get one attempt. return retry.FilerExceptWrites(f), path, nil } diff --git a/libs/filer/files_client.go b/libs/filer/files_client.go index 25f8e8de344..9ce13f53286 100644 --- a/libs/filer/files_client.go +++ b/libs/filer/files_client.go @@ -1,9 +1,12 @@ package filer import ( + "bufio" + "bytes" "cmp" "context" "errors" + "fmt" "io" "io/fs" "net/http" @@ -282,7 +285,8 @@ func (w *FilesClient) Write(ctx context.Context, name string, reader io.Reader, // upload engine, which sends small files in a single PUT and splits large ones // into parts. Non-seekable streams and the flag-off case use the single-shot // UploadFile endpoint below: the former to avoid buffering the whole stream in - // memory, the latter to keep the default behavior a plain PUT. The engine + // memory (uploadFile buffers only small ones, to retry them), the latter to + // keep the default behavior a plain PUT. The engine // recovers an io.ReaderAt for concurrent positioned reads when the source // provides one (a local file). if MultipartUploadEnabled(ctx) && isSeekable(reader) { @@ -312,24 +316,61 @@ func (w *FilesClient) Write(ctx context.Context, name string, reader io.Reader, return err } +// filesReplayBufferSize is the largest non-seekable upload that is buffered in +// memory so it can be retried. fs cp between remote paths (Volumes or DBFS to +// Volumes) streams the source download into the upload, so without the buffer +// a single 503 fails the copy. Larger streams get one attempt. +// +// 8 MiB is not an API limit; it caps memory. With fs cp's default concurrency +// of 8, at most 64 MiB is buffered at once, and most files copied with fs cp +// (code, notebooks, configs) are well below it. A variable so tests can lower it. +var filesReplayBufferSize = 8 * 1024 * 1024 // 8 MiB + +// filesReplayReadSize is the size of the reads replayable makes from the +// source. It matches the DBFS read limit: /api/2.0/dbfs/read returns at most +// 1 MiB per call, and the SDK's DBFS read handle makes one call per Read. +const filesReplayReadSize = 1024 * 1024 // 1 MiB + +// replayable returns a reader that can be rewound to start before each upload +// attempt. A non-seekable reader (including a pipe, which is an *os.File whose +// Seek fails) is read into memory if it is at most filesReplayBufferSize bytes; +// otherwise stream holds the bytes read so far followed by the +// rest of reader, to be sent once. +func replayable(reader io.Reader) (rs io.ReadSeeker, start int64, stream io.Reader, err error) { + if s, ok := reader.(io.ReadSeeker); ok { + if start, err := s.Seek(0, io.SeekCurrent); err == nil { + return s, start, nil, nil + } + } + // io.ReadAll asks for small, growing chunks; reading through + // filesReplayReadSize keeps a DBFS source to one API call per MiB. + br := bufio.NewReaderSize(reader, filesReplayReadSize) + head, err := io.ReadAll(io.LimitReader(br, int64(filesReplayBufferSize)+1)) + if err != nil { + return nil, 0, nil, fmt.Errorf("reading upload body: %w", err) + } + if len(head) > filesReplayBufferSize { + return nil, 0, io.MultiReader(bytes.NewReader(head), br), nil + } + return bytes.NewReader(head), 0, nil, nil +} + // uploadFile sends reader in a single PUT. files/v2 rejects a retrier on this -// call because it cannot replay a streamed body, so a seekable reader is rewound -// to its starting offset and retried here. Other readers get one attempt; that -// includes pipes, which are *os.File and so implement io.Seeker but fail Seek. +// call because it cannot replay a streamed body, so the filer retries it here +// with a reader from replayable, rewound to its starting offset each attempt. func (w *FilesClient) uploadFile(ctx context.Context, absPath string, reader io.Reader, overwrite bool) error { - seeker, ok := reader.(io.Seeker) - if !ok { - return w.putFile(ctx, absPath, io.NopCloser(reader), overwrite) - } - start, err := seeker.Seek(0, io.SeekCurrent) + seeker, start, stream, err := replayable(reader) if err != nil { - return w.putFile(ctx, absPath, io.NopCloser(reader), overwrite) + return err + } + if stream != nil { + return w.putFile(ctx, absPath, io.NopCloser(stream), overwrite) } return ops.Execute(ctx, func(ctx context.Context) error { if _, err := seeker.Seek(start, io.SeekStart); err != nil { return err } - body := &attemptBody{r: reader} + body := &attemptBody{r: seeker} err := w.putFile(ctx, absPath, body, overwrite) body.Close() return err diff --git a/libs/filer/files_client_test.go b/libs/filer/files_client_test.go index 0d621d3e542..0434f673150 100644 --- a/libs/filer/files_client_test.go +++ b/libs/filer/files_client_test.go @@ -312,14 +312,90 @@ func TestFilesClientRetriesThrottledUploadFromStartOffset(t *testing.T) { assert.Equal(t, []string{"abc", "abc", "abc"}, uploads) } -func TestFilesClientDoesNotRetryNonSeekableUpload(t *testing.T) { +// uploadsWithOneThrottle records each PUT body; the first PUT is throttled. +func uploadsWithOneThrottle(server *testserver.Server) *[]string { + var uploads []string + put := throttled(1, ok) + server.Handle("PUT", "/api/2.0/fs/files/{path...}", func(req testserver.Request) any { + uploads = append(uploads, string(req.Body)) + return put(req) + }) + return &uploads +} + +func TestFilesClientRetriesSmallNonSeekableUpload(t *testing.T) { + ctx := t.Context() + server := testserver.New(t) + uploads := uploadsWithOneThrottle(server) + f := newFilesClientWithFastRetries(t, server) + setReplayBufferSize(t, 3) + + // Exactly filesReplayBufferSize bytes still fit in the buffer. + require.NoError(t, f.Write(ctx, "hello", onlyReader{strings.NewReader("abc")}, CreateParentDirectories)) + assert.Equal(t, []string{"abc", "abc"}, *uploads) +} + +func TestFilesClientRetriesSmallPipeUpload(t *testing.T) { + ctx := t.Context() + server := testserver.New(t) + uploads := uploadsWithOneThrottle(server) + f := newFilesClientWithFastRetries(t, server) + + // A pipe is an *os.File whose Seek fails. + pr, pw, err := os.Pipe() + require.NoError(t, err) + _, err = pw.WriteString("abc") + require.NoError(t, err) + require.NoError(t, pw.Close()) + defer pr.Close() + + require.NoError(t, f.Write(ctx, "hello", pr, CreateParentDirectories)) + assert.Equal(t, []string{"abc", "abc"}, *uploads) +} + +func TestFilesClientDoesNotRetryLargeNonSeekableUpload(t *testing.T) { ctx := t.Context() server := testserver.New(t) - server.Handle("PUT", "/api/2.0/fs/files/{path...}", throttled(1, ok)) + uploads := uploadsWithOneThrottle(server) f := newFilesClientWithFastRetries(t, server) + setReplayBufferSize(t, 2) err := f.Write(ctx, "hello", onlyReader{strings.NewReader("abc")}, CreateParentDirectories) assert.ErrorContains(t, err, "AWS S3 is throttling requests") + assert.Equal(t, []string{"abc"}, *uploads) +} + +// countingReader counts Read calls on a non-seekable reader, modelling a DBFS +// read handle that makes one API call per Read. +type countingReader struct { + r io.Reader + calls int +} + +func (c *countingReader) Read(p []byte) (int, error) { + c.calls++ + return c.r.Read(p) +} + +func TestFilesClientBuffersUploadInLargeReads(t *testing.T) { + ctx := t.Context() + server := testserver.New(t) + uploads := uploadsWithOneThrottle(server) + f := newFilesClientWithFastRetries(t, server) + + content := strings.Repeat("x", filesReplayReadSize) + src := &countingReader{r: strings.NewReader(content)} + require.NoError(t, f.Write(ctx, "hello", src, CreateParentDirectories)) + assert.Len(t, *uploads, 2) + // One Read fills the buffer and one more returns EOF; io.ReadAll's own + // growing chunks would take about 20. + assert.LessOrEqual(t, src.calls, 3) +} + +func setReplayBufferSize(t *testing.T, n int) { + old := filesReplayBufferSize + filesReplayBufferSize = n + t.Cleanup(func() { filesReplayBufferSize = old }) } func TestFilesClientStopsRetryingAfterLimit(t *testing.T) {