Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .nextchanges/cli/7763.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
* Retry DBFS uploads (`fs cp` to `dbfs:/`) once when the backend loses the upload stream ("No such stream with handle"). ([#7763](https://github.com/databricks/cli/pull/7763))
101 changes: 100 additions & 1 deletion libs/filer/dbfs_client.go
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package filer

import (
"bytes"
"cmp"
"context"
"errors"
Expand All @@ -12,6 +13,7 @@ import (
"strings"
"time"

"github.com/databricks/cli/libs/log"
"github.com/databricks/databricks-sdk-go"
"github.com/databricks/databricks-sdk-go/apierr"
"github.com/databricks/databricks-sdk-go/service/files"
Expand Down Expand Up @@ -114,6 +116,26 @@ func (w *DbfsClient) Write(ctx context.Context, name string, reader io.Reader, m
}
}

src := newDbfsUploadSource(reader)
err = w.upload(ctx, absPath, fileMode, src)
if !isDbfsStreamLost(err) {
return err
}

// The backend forgot the stream handle mid-upload, so the file is incomplete.
// Restart the whole create/add-block/close sequence once. The first create
// succeeded, so whatever is at the path now is this upload's own partial file;
// overwrite it even if the caller did not ask to overwrite an existing file.
// A file another writer created at the same path in between is replaced too.
if !src.rewind() {
return err
}
log.Debugf(ctx, "dbfs write to %s lost its stream handle, restarting the upload: %s", absPath, err)
return w.upload(ctx, absPath, fileMode|files.FileModeOverwrite, src)
}

// upload writes the content of src to absPath with one DBFS stream (create, add-block, close).
func (w *DbfsClient) upload(ctx context.Context, absPath string, fileMode files.FileMode, src io.Reader) error {
handle, err := w.workspaceClient.Dbfs.Open(ctx, absPath, fileMode)
if err != nil {
aerr, ok := errors.AsType[*apierr.APIError](err)
Expand All @@ -131,7 +153,7 @@ func (w *DbfsClient) Write(ctx context.Context, name string, reader io.Reader, m
return err
}

_, err = io.Copy(handle, reader)
_, err = io.Copy(handle, src)
cerr := handle.Close()
if err == nil {
err = cerr
Expand All @@ -140,6 +162,83 @@ func (w *DbfsClient) Write(ctx context.Context, name string, reader io.Reader, m
return err
}

// isDbfsStreamLost reports whether err says that the DBFS backend no longer knows
// the stream handle of an upload. It happens transiently in CI on every cloud,
// for files of a few bytes, a few seconds after the handle was created.
func isDbfsStreamLost(err error) bool {
aerr, ok := errors.AsType[*apierr.APIError](err)
return ok && aerr.ErrorCode == "RESOURCE_DOES_NOT_EXIST" && strings.HasPrefix(aerr.Message, "No such stream with handle")
}

// dbfsReplayBufferSize is the largest non-seekable source that a DBFS upload
// keeps in memory so it can restart after losing its stream handle. It caps
// memory, it is not an API limit. An upload that has read more than this from a
// non-seekable source is not restarted.
// A variable so tests can lower it.
var dbfsReplayBufferSize = 8 * 1024 * 1024 // 8 MiB

// dbfsUploadSource reads an upload's source and can rewind it to the start once.
// A seekable source is rewound with Seek. A non-seekable one (a remote file
// download, or a pipe, which is an *os.File whose Seek fails) is copied into a
// buffer while it is read, up to dbfsReplayBufferSize bytes.
type dbfsUploadSource struct {
r io.Reader

seeker io.Seeker
start int64

buf bytes.Buffer
overflow bool
replay io.Reader
}

func newDbfsUploadSource(r io.Reader) *dbfsUploadSource {
src := &dbfsUploadSource{r: r}
if s, ok := r.(io.Seeker); ok {
if start, err := s.Seek(0, io.SeekCurrent); err == nil {
src.seeker = s
src.start = start
}
}
return src
}

func (s *dbfsUploadSource) Read(p []byte) (int, error) {
if s.replay != nil {
return s.replay.Read(p)
}
n, err := s.r.Read(p)
if s.seeker == nil && !s.overflow {
if s.buf.Len()+n > dbfsReplayBufferSize {
s.overflow = true
s.buf = bytes.Buffer{}
} else {
s.buf.Write(p[:n])
}
}
return n, err
}

// rewind makes the next Read start from the beginning of the source again.
// It returns false if that is not possible. It works only once.
func (s *dbfsUploadSource) rewind() bool {
if s.replay != nil {
return false
}
if s.seeker != nil {
if _, err := s.seeker.Seek(s.start, io.SeekStart); err != nil {
return false
}
s.replay = s.r
return true
}
if s.overflow {
return false
}
s.replay = io.MultiReader(bytes.NewReader(s.buf.Bytes()), s.r)
return true
}

func (w *DbfsClient) Read(ctx context.Context, name string) (io.ReadCloser, error) {
absPath, err := w.root.Join(name)
if err != nil {
Expand Down
192 changes: 192 additions & 0 deletions libs/filer/dbfs_client_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,192 @@
package filer

import (
"bytes"
"io"
"strings"
"sync"
"testing"

"github.com/databricks/cli/libs/testserver"
"github.com/databricks/databricks-sdk-go"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)

// dbfsServer serves the DBFS API from the test server's fake workspace. The
// first dropAddBlock add-block calls and the first dropClose close calls find
// their stream handle gone, like the real backend does when it loses a stream.
type dbfsServer struct {
mu sync.Mutex
dropAddBlock int
dropClose int
creates []string
}

// dropHandle closes the request's stream on the fake and then runs handler, which
// fails with the backend's "No such stream with handle" error. Closing commits
// the partial file, so a restarted upload without overwrite would conflict with it.
func dropHandle(req testserver.Request, handler func(testserver.Request) testserver.Response) testserver.Response {
req.Workspace.DbfsClose(req)
return handler(req)
}

func newDbfsClientWithServer(t *testing.T, d *dbfsServer) *DbfsClient {
t.Helper()
server := testserver.New(t)
server.Handle("GET", "/api/2.0/dbfs/get-status", func(req testserver.Request) any {
return req.Workspace.DbfsGetStatus(req)
})
server.Handle("POST", "/api/2.0/dbfs/mkdirs", func(req testserver.Request) any {
return req.Workspace.DbfsMkdirs(req)
})
server.Handle("GET", "/api/2.0/dbfs/read", func(req testserver.Request) any {
return req.Workspace.DbfsRead(req)
})
server.Handle("POST", "/api/2.0/dbfs/create", func(req testserver.Request) any {
d.mu.Lock()
d.creates = append(d.creates, string(req.Body))
d.mu.Unlock()
return req.Workspace.DbfsCreate(req)
})
server.Handle("POST", "/api/2.0/dbfs/add-block", func(req testserver.Request) any {
d.mu.Lock()
drop := d.dropAddBlock > 0
d.dropAddBlock--
d.mu.Unlock()
if drop {
return dropHandle(req, req.Workspace.DbfsAddBlock)
}
return req.Workspace.DbfsAddBlock(req)
})
server.Handle("POST", "/api/2.0/dbfs/close", func(req testserver.Request) any {
d.mu.Lock()
drop := d.dropClose > 0
d.dropClose--
d.mu.Unlock()
if drop {
return dropHandle(req, req.Workspace.DbfsClose)
}
return req.Workspace.DbfsClose(req)
})

client, err := databricks.NewWorkspaceClient(&databricks.Config{
Host: server.URL,
Token: "testtoken",
})
require.NoError(t, err)

f, err := NewDbfsClient(client, "/tmp/root")
require.NoError(t, err)
return f.(*DbfsClient)
}

func readDbfsFile(t *testing.T, f *DbfsClient, name string) string {
t.Helper()
r, err := f.Read(t.Context(), name)
require.NoError(t, err)
b, err := io.ReadAll(r)
require.NoError(t, err)
return string(b)
}

func TestDbfsClientRestartsUploadWhenAddBlockLosesHandle(t *testing.T) {
d := &dbfsServer{dropAddBlock: 1}
f := newDbfsClientWithServer(t, d)

// The source is a non-seekable stream and the write does not overwrite: the
// restart must replay the buffered bytes and replace its own partial file.
err := f.Write(t.Context(), "hello", onlyReader{strings.NewReader("abc")}, CreateParentDirectories)
require.NoError(t, err)
assert.Equal(t, "abc", readDbfsFile(t, f, "hello"))
assert.Equal(t, []string{
`{"path":"/tmp/root/hello"}`,
`{"overwrite":true,"path":"/tmp/root/hello"}`,
}, d.creates)
}

func TestDbfsClientRestartsUploadWhenCloseLosesHandle(t *testing.T) {
d := &dbfsServer{dropClose: 1}
f := newDbfsClientWithServer(t, d)

err := f.Write(t.Context(), "hello", onlyReader{strings.NewReader("abc")}, CreateParentDirectories)
require.NoError(t, err)
assert.Equal(t, "abc", readDbfsFile(t, f, "hello"))
assert.Len(t, d.creates, 2)
}

func TestDbfsClientRestartsSeekableUploadFromStartOffset(t *testing.T) {
d := &dbfsServer{dropAddBlock: 1}
f := newDbfsClientWithServer(t, d)

r := bytes.NewReader([]byte("xxabc"))
_, err := r.Seek(2, io.SeekStart)
require.NoError(t, err)

// The seekable source is not buffered, even when it exceeds the buffer size.
setDbfsReplayBufferSize(t, 1)
err = f.Write(t.Context(), "hello", r, CreateParentDirectories, OverwriteIfExists)
require.NoError(t, err)
assert.Equal(t, "abc", readDbfsFile(t, f, "hello"))
assert.Len(t, d.creates, 2)
}

func TestDbfsClientRestartsUploadOnlyOnce(t *testing.T) {
d := &dbfsServer{dropAddBlock: 2}
f := newDbfsClientWithServer(t, d)

err := f.Write(t.Context(), "hello", onlyReader{strings.NewReader("abc")}, CreateParentDirectories)
assert.ErrorContains(t, err, "No such stream with handle")
assert.Len(t, d.creates, 2)
}

func TestDbfsClientDoesNotRestartLargeNonSeekableUpload(t *testing.T) {
d := &dbfsServer{dropAddBlock: 1}
f := newDbfsClientWithServer(t, d)
setDbfsReplayBufferSize(t, 2)

err := f.Write(t.Context(), "hello", onlyReader{strings.NewReader("abc")}, CreateParentDirectories)
assert.ErrorContains(t, err, "No such stream with handle")
assert.Len(t, d.creates, 1)
}

func TestDbfsClientRestartsLargeNonSeekableUploadThatFailsEarly(t *testing.T) {
d := &dbfsServer{dropAddBlock: 1}
f := newDbfsClientWithServer(t, d)

// The first add-block fails after one 1 MiB block was read: the buffered
// block is replayed and the rest of the source is streamed after it.
content := strings.Repeat("a", 1024*1024) + strings.Repeat("b", 1024*1024) + "c"
setDbfsReplayBufferSize(t, 1024*1024+10)
err := f.Write(t.Context(), "hello", onlyReader{strings.NewReader(content)}, CreateParentDirectories)
require.NoError(t, err)
assert.Equal(t, content, readDbfsFile(t, f, "hello"))
assert.Len(t, d.creates, 2)
}

func TestDbfsClientRestartsNonSeekableUploadOfBufferSize(t *testing.T) {
d := &dbfsServer{dropAddBlock: 1}
f := newDbfsClientWithServer(t, d)
setDbfsReplayBufferSize(t, 3)

err := f.Write(t.Context(), "hello", onlyReader{strings.NewReader("abc")}, CreateParentDirectories)
require.NoError(t, err)
assert.Equal(t, "abc", readDbfsFile(t, f, "hello"))
}

func TestDbfsClientDoesNotRestartOnOtherErrors(t *testing.T) {
d := &dbfsServer{}
f := newDbfsClientWithServer(t, d)

require.NoError(t, f.Write(t.Context(), "hello", strings.NewReader("abc"), CreateParentDirectories))
err := f.Write(t.Context(), "hello", strings.NewReader("def"), CreateParentDirectories)
assert.ErrorIs(t, err, fileAlreadyExistsError{"/tmp/root/hello"})
assert.Len(t, d.creates, 2)
assert.Equal(t, "abc", readDbfsFile(t, f, "hello"))
}

func setDbfsReplayBufferSize(t *testing.T, n int) {
old := dbfsReplayBufferSize
dbfsReplayBufferSize = n
t.Cleanup(func() { dbfsReplayBufferSize = old })
}
4 changes: 2 additions & 2 deletions libs/testserver/dbfs.go
Original file line number Diff line number Diff line change
Expand Up @@ -185,7 +185,7 @@ func (s *FakeWorkspace) DbfsAddBlock(req Request) Response {
defer s.LockUnlock()()
h, ok := s.dbfsState().handles[request.Handle]
if !ok {
return dbfsError(404, "RESOURCE_DOES_NOT_EXIST", fmt.Sprintf("No handle %d.", request.Handle))
return dbfsError(404, "RESOURCE_DOES_NOT_EXIST", fmt.Sprintf("No such stream with handle: %d.", request.Handle))
}
if len(data) > dbfsMaxBlockSize {
return dbfsError(400, "MAX_BLOCK_SIZE_EXCEEDED", fmt.Sprintf("Block size %d exceeds the maximum of %d bytes.", len(data), dbfsMaxBlockSize))
Expand All @@ -203,7 +203,7 @@ func (s *FakeWorkspace) DbfsClose(req Request) Response {
d := s.dbfsState()
h, ok := d.handles[request.Handle]
if !ok {
return dbfsError(404, "RESOURCE_DOES_NOT_EXIST", fmt.Sprintf("No handle %d.", request.Handle))
return dbfsError(404, "RESOURCE_DOES_NOT_EXIST", fmt.Sprintf("No such stream with handle: %d.", request.Handle))
}
delete(d.handles, request.Handle)
d.files[h.path] = dbfsFile{data: h.data, modTime: nowMilli()}
Expand Down
Loading