diff --git a/.nextchanges/cli/7763.md b/.nextchanges/cli/7763.md new file mode 100644 index 00000000000..011367cfb3b --- /dev/null +++ b/.nextchanges/cli/7763.md @@ -0,0 +1 @@ +* Retry DBFS uploads (`fs cp` to `dbfs:/`) once when the backend loses the upload stream ("No such stream with handle"). ([#7763](https://github.com/databricks/cli/pull/7763)) diff --git a/libs/filer/dbfs_client.go b/libs/filer/dbfs_client.go index f5d6c5993e4..2b4d63671fa 100644 --- a/libs/filer/dbfs_client.go +++ b/libs/filer/dbfs_client.go @@ -1,6 +1,7 @@ package filer import ( + "bytes" "cmp" "context" "errors" @@ -12,6 +13,7 @@ import ( "strings" "time" + "github.com/databricks/cli/libs/log" "github.com/databricks/databricks-sdk-go" "github.com/databricks/databricks-sdk-go/apierr" "github.com/databricks/databricks-sdk-go/service/files" @@ -114,6 +116,26 @@ func (w *DbfsClient) Write(ctx context.Context, name string, reader io.Reader, m } } + src := newDbfsUploadSource(reader) + err = w.upload(ctx, absPath, fileMode, src) + if !isDbfsStreamLost(err) { + return err + } + + // The backend forgot the stream handle mid-upload, so the file is incomplete. + // Restart the whole create/add-block/close sequence once. The first create + // succeeded, so whatever is at the path now is this upload's own partial file; + // overwrite it even if the caller did not ask to overwrite an existing file. + // A file another writer created at the same path in between is replaced too. + if !src.rewind() { + return err + } + log.Debugf(ctx, "dbfs write to %s lost its stream handle, restarting the upload: %s", absPath, err) + return w.upload(ctx, absPath, fileMode|files.FileModeOverwrite, src) +} + +// upload writes the content of src to absPath with one DBFS stream (create, add-block, close). +func (w *DbfsClient) upload(ctx context.Context, absPath string, fileMode files.FileMode, src io.Reader) error { handle, err := w.workspaceClient.Dbfs.Open(ctx, absPath, fileMode) if err != nil { aerr, ok := errors.AsType[*apierr.APIError](err) @@ -131,7 +153,7 @@ func (w *DbfsClient) Write(ctx context.Context, name string, reader io.Reader, m return err } - _, err = io.Copy(handle, reader) + _, err = io.Copy(handle, src) cerr := handle.Close() if err == nil { err = cerr @@ -140,6 +162,83 @@ func (w *DbfsClient) Write(ctx context.Context, name string, reader io.Reader, m return err } +// isDbfsStreamLost reports whether err says that the DBFS backend no longer knows +// the stream handle of an upload. It happens transiently in CI on every cloud, +// for files of a few bytes, a few seconds after the handle was created. +func isDbfsStreamLost(err error) bool { + aerr, ok := errors.AsType[*apierr.APIError](err) + return ok && aerr.ErrorCode == "RESOURCE_DOES_NOT_EXIST" && strings.HasPrefix(aerr.Message, "No such stream with handle") +} + +// dbfsReplayBufferSize is the largest non-seekable source that a DBFS upload +// keeps in memory so it can restart after losing its stream handle. It caps +// memory, it is not an API limit. An upload that has read more than this from a +// non-seekable source is not restarted. +// A variable so tests can lower it. +var dbfsReplayBufferSize = 8 * 1024 * 1024 // 8 MiB + +// dbfsUploadSource reads an upload's source and can rewind it to the start once. +// A seekable source is rewound with Seek. A non-seekable one (a remote file +// download, or a pipe, which is an *os.File whose Seek fails) is copied into a +// buffer while it is read, up to dbfsReplayBufferSize bytes. +type dbfsUploadSource struct { + r io.Reader + + seeker io.Seeker + start int64 + + buf bytes.Buffer + overflow bool + replay io.Reader +} + +func newDbfsUploadSource(r io.Reader) *dbfsUploadSource { + src := &dbfsUploadSource{r: r} + if s, ok := r.(io.Seeker); ok { + if start, err := s.Seek(0, io.SeekCurrent); err == nil { + src.seeker = s + src.start = start + } + } + return src +} + +func (s *dbfsUploadSource) Read(p []byte) (int, error) { + if s.replay != nil { + return s.replay.Read(p) + } + n, err := s.r.Read(p) + if s.seeker == nil && !s.overflow { + if s.buf.Len()+n > dbfsReplayBufferSize { + s.overflow = true + s.buf = bytes.Buffer{} + } else { + s.buf.Write(p[:n]) + } + } + return n, err +} + +// rewind makes the next Read start from the beginning of the source again. +// It returns false if that is not possible. It works only once. +func (s *dbfsUploadSource) rewind() bool { + if s.replay != nil { + return false + } + if s.seeker != nil { + if _, err := s.seeker.Seek(s.start, io.SeekStart); err != nil { + return false + } + s.replay = s.r + return true + } + if s.overflow { + return false + } + s.replay = io.MultiReader(bytes.NewReader(s.buf.Bytes()), s.r) + return true +} + func (w *DbfsClient) Read(ctx context.Context, name string) (io.ReadCloser, error) { absPath, err := w.root.Join(name) if err != nil { diff --git a/libs/filer/dbfs_client_test.go b/libs/filer/dbfs_client_test.go new file mode 100644 index 00000000000..c15dcb47107 --- /dev/null +++ b/libs/filer/dbfs_client_test.go @@ -0,0 +1,192 @@ +package filer + +import ( + "bytes" + "io" + "strings" + "sync" + "testing" + + "github.com/databricks/cli/libs/testserver" + "github.com/databricks/databricks-sdk-go" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// dbfsServer serves the DBFS API from the test server's fake workspace. The +// first dropAddBlock add-block calls and the first dropClose close calls find +// their stream handle gone, like the real backend does when it loses a stream. +type dbfsServer struct { + mu sync.Mutex + dropAddBlock int + dropClose int + creates []string +} + +// dropHandle closes the request's stream on the fake and then runs handler, which +// fails with the backend's "No such stream with handle" error. Closing commits +// the partial file, so a restarted upload without overwrite would conflict with it. +func dropHandle(req testserver.Request, handler func(testserver.Request) testserver.Response) testserver.Response { + req.Workspace.DbfsClose(req) + return handler(req) +} + +func newDbfsClientWithServer(t *testing.T, d *dbfsServer) *DbfsClient { + t.Helper() + server := testserver.New(t) + server.Handle("GET", "/api/2.0/dbfs/get-status", func(req testserver.Request) any { + return req.Workspace.DbfsGetStatus(req) + }) + server.Handle("POST", "/api/2.0/dbfs/mkdirs", func(req testserver.Request) any { + return req.Workspace.DbfsMkdirs(req) + }) + server.Handle("GET", "/api/2.0/dbfs/read", func(req testserver.Request) any { + return req.Workspace.DbfsRead(req) + }) + server.Handle("POST", "/api/2.0/dbfs/create", func(req testserver.Request) any { + d.mu.Lock() + d.creates = append(d.creates, string(req.Body)) + d.mu.Unlock() + return req.Workspace.DbfsCreate(req) + }) + server.Handle("POST", "/api/2.0/dbfs/add-block", func(req testserver.Request) any { + d.mu.Lock() + drop := d.dropAddBlock > 0 + d.dropAddBlock-- + d.mu.Unlock() + if drop { + return dropHandle(req, req.Workspace.DbfsAddBlock) + } + return req.Workspace.DbfsAddBlock(req) + }) + server.Handle("POST", "/api/2.0/dbfs/close", func(req testserver.Request) any { + d.mu.Lock() + drop := d.dropClose > 0 + d.dropClose-- + d.mu.Unlock() + if drop { + return dropHandle(req, req.Workspace.DbfsClose) + } + return req.Workspace.DbfsClose(req) + }) + + client, err := databricks.NewWorkspaceClient(&databricks.Config{ + Host: server.URL, + Token: "testtoken", + }) + require.NoError(t, err) + + f, err := NewDbfsClient(client, "/tmp/root") + require.NoError(t, err) + return f.(*DbfsClient) +} + +func readDbfsFile(t *testing.T, f *DbfsClient, name string) string { + t.Helper() + r, err := f.Read(t.Context(), name) + require.NoError(t, err) + b, err := io.ReadAll(r) + require.NoError(t, err) + return string(b) +} + +func TestDbfsClientRestartsUploadWhenAddBlockLosesHandle(t *testing.T) { + d := &dbfsServer{dropAddBlock: 1} + f := newDbfsClientWithServer(t, d) + + // The source is a non-seekable stream and the write does not overwrite: the + // restart must replay the buffered bytes and replace its own partial file. + err := f.Write(t.Context(), "hello", onlyReader{strings.NewReader("abc")}, CreateParentDirectories) + require.NoError(t, err) + assert.Equal(t, "abc", readDbfsFile(t, f, "hello")) + assert.Equal(t, []string{ + `{"path":"/tmp/root/hello"}`, + `{"overwrite":true,"path":"/tmp/root/hello"}`, + }, d.creates) +} + +func TestDbfsClientRestartsUploadWhenCloseLosesHandle(t *testing.T) { + d := &dbfsServer{dropClose: 1} + f := newDbfsClientWithServer(t, d) + + err := f.Write(t.Context(), "hello", onlyReader{strings.NewReader("abc")}, CreateParentDirectories) + require.NoError(t, err) + assert.Equal(t, "abc", readDbfsFile(t, f, "hello")) + assert.Len(t, d.creates, 2) +} + +func TestDbfsClientRestartsSeekableUploadFromStartOffset(t *testing.T) { + d := &dbfsServer{dropAddBlock: 1} + f := newDbfsClientWithServer(t, d) + + r := bytes.NewReader([]byte("xxabc")) + _, err := r.Seek(2, io.SeekStart) + require.NoError(t, err) + + // The seekable source is not buffered, even when it exceeds the buffer size. + setDbfsReplayBufferSize(t, 1) + err = f.Write(t.Context(), "hello", r, CreateParentDirectories, OverwriteIfExists) + require.NoError(t, err) + assert.Equal(t, "abc", readDbfsFile(t, f, "hello")) + assert.Len(t, d.creates, 2) +} + +func TestDbfsClientRestartsUploadOnlyOnce(t *testing.T) { + d := &dbfsServer{dropAddBlock: 2} + f := newDbfsClientWithServer(t, d) + + err := f.Write(t.Context(), "hello", onlyReader{strings.NewReader("abc")}, CreateParentDirectories) + assert.ErrorContains(t, err, "No such stream with handle") + assert.Len(t, d.creates, 2) +} + +func TestDbfsClientDoesNotRestartLargeNonSeekableUpload(t *testing.T) { + d := &dbfsServer{dropAddBlock: 1} + f := newDbfsClientWithServer(t, d) + setDbfsReplayBufferSize(t, 2) + + err := f.Write(t.Context(), "hello", onlyReader{strings.NewReader("abc")}, CreateParentDirectories) + assert.ErrorContains(t, err, "No such stream with handle") + assert.Len(t, d.creates, 1) +} + +func TestDbfsClientRestartsLargeNonSeekableUploadThatFailsEarly(t *testing.T) { + d := &dbfsServer{dropAddBlock: 1} + f := newDbfsClientWithServer(t, d) + + // The first add-block fails after one 1 MiB block was read: the buffered + // block is replayed and the rest of the source is streamed after it. + content := strings.Repeat("a", 1024*1024) + strings.Repeat("b", 1024*1024) + "c" + setDbfsReplayBufferSize(t, 1024*1024+10) + err := f.Write(t.Context(), "hello", onlyReader{strings.NewReader(content)}, CreateParentDirectories) + require.NoError(t, err) + assert.Equal(t, content, readDbfsFile(t, f, "hello")) + assert.Len(t, d.creates, 2) +} + +func TestDbfsClientRestartsNonSeekableUploadOfBufferSize(t *testing.T) { + d := &dbfsServer{dropAddBlock: 1} + f := newDbfsClientWithServer(t, d) + setDbfsReplayBufferSize(t, 3) + + err := f.Write(t.Context(), "hello", onlyReader{strings.NewReader("abc")}, CreateParentDirectories) + require.NoError(t, err) + assert.Equal(t, "abc", readDbfsFile(t, f, "hello")) +} + +func TestDbfsClientDoesNotRestartOnOtherErrors(t *testing.T) { + d := &dbfsServer{} + f := newDbfsClientWithServer(t, d) + + require.NoError(t, f.Write(t.Context(), "hello", strings.NewReader("abc"), CreateParentDirectories)) + err := f.Write(t.Context(), "hello", strings.NewReader("def"), CreateParentDirectories) + assert.ErrorIs(t, err, fileAlreadyExistsError{"/tmp/root/hello"}) + assert.Len(t, d.creates, 2) + assert.Equal(t, "abc", readDbfsFile(t, f, "hello")) +} + +func setDbfsReplayBufferSize(t *testing.T, n int) { + old := dbfsReplayBufferSize + dbfsReplayBufferSize = n + t.Cleanup(func() { dbfsReplayBufferSize = old }) +} diff --git a/libs/testserver/dbfs.go b/libs/testserver/dbfs.go index 712db80920e..34508edb7aa 100644 --- a/libs/testserver/dbfs.go +++ b/libs/testserver/dbfs.go @@ -185,7 +185,7 @@ func (s *FakeWorkspace) DbfsAddBlock(req Request) Response { defer s.LockUnlock()() h, ok := s.dbfsState().handles[request.Handle] if !ok { - return dbfsError(404, "RESOURCE_DOES_NOT_EXIST", fmt.Sprintf("No handle %d.", request.Handle)) + return dbfsError(404, "RESOURCE_DOES_NOT_EXIST", fmt.Sprintf("No such stream with handle: %d.", request.Handle)) } if len(data) > dbfsMaxBlockSize { return dbfsError(400, "MAX_BLOCK_SIZE_EXCEEDED", fmt.Sprintf("Block size %d exceeds the maximum of %d bytes.", len(data), dbfsMaxBlockSize)) @@ -203,7 +203,7 @@ func (s *FakeWorkspace) DbfsClose(req Request) Response { d := s.dbfsState() h, ok := d.handles[request.Handle] if !ok { - return dbfsError(404, "RESOURCE_DOES_NOT_EXIST", fmt.Sprintf("No handle %d.", request.Handle)) + return dbfsError(404, "RESOURCE_DOES_NOT_EXIST", fmt.Sprintf("No such stream with handle: %d.", request.Handle)) } delete(d.handles, request.Handle) d.files[h.path] = dbfsFile{data: h.data, modTime: nowMilli()}