Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .nextchanges/cli/fs-cp-r-skip-parent-check.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
* `databricks fs cp -r` no longer checks that each file's parent directory exists before writing it, because the copy has already created that directory. This saves one request per file: a copy to a Volume sends one request per small file instead of two. ([#7711](https://github.com/databricks/cli/pull/7711))
1 change: 1 addition & 0 deletions acceptance/cmd/fs/cp-r-requests/out.test.toml
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
Cloud = false
33 changes: 33 additions & 0 deletions acceptance/cmd/fs/cp-r-requests/output.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@

>>> [CLI] fs cp -r localdir dbfs:/Volumes/main/default/fs-cp-r-requests/dst
localdir/a.txt -> dbfs:/Volumes/main/default/fs-cp-r-requests/dst/a.txt
localdir/b.txt -> dbfs:/Volumes/main/default/fs-cp-r-requests/dst/b.txt
localdir/sub/c.txt -> dbfs:/Volumes/main/default/fs-cp-r-requests/dst/sub/c.txt

=== Recursive copy: a directory PUT per directory, a file PUT per file, no directory HEAD per file

>>> print_requests.py --sort //api/2.0/fs/
{"method":"PUT","path":"/api/2.0/fs/directories/Volumes/main/default/fs-cp-r-requests/dst"}
{"method":"PUT","path":"/api/2.0/fs/directories/Volumes/main/default/fs-cp-r-requests/dst/sub"}
{"method":"PUT","path":"/api/2.0/fs/files/Volumes/main/default/fs-cp-r-requests/dst/a.txt"}
{"method":"PUT","path":"/api/2.0/fs/files/Volumes/main/default/fs-cp-r-requests/dst/b.txt"}
{"method":"PUT","path":"/api/2.0/fs/files/Volumes/main/default/fs-cp-r-requests/dst/sub/c.txt"}

=== Single-file copy still stats the target and checks its parent directory

>>> [CLI] fs cp localdir/a.txt dbfs:/Volumes/main/default/fs-cp-r-requests/dst/sub/single.txt
localdir/a.txt -> dbfs:/Volumes/main/default/fs-cp-r-requests/dst/sub/single.txt

>>> print_requests.py --sort //api/2.0/fs/
{"method":"HEAD","path":"/api/2.0/fs/directories/Volumes/main/default/fs-cp-r-requests/dst/sub"}
{"method":"HEAD","path":"/api/2.0/fs/directories/Volumes/main/default/fs-cp-r-requests/dst/sub/single.txt"}
{"method":"HEAD","path":"/api/2.0/fs/files/Volumes/main/default/fs-cp-r-requests/dst/sub/single.txt"}
{"method":"PUT","path":"/api/2.0/fs/files/Volumes/main/default/fs-cp-r-requests/dst/sub/single.txt"}

=== Copied files

>>> [CLI] fs cat dbfs:/Volumes/main/default/fs-cp-r-requests/dst/a.txt
one

>>> [CLI] fs cat dbfs:/Volumes/main/default/fs-cp-r-requests/dst/sub/c.txt
three
25 changes: 25 additions & 0 deletions acceptance/cmd/fs/cp-r-requests/script
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
# fs cp -r creates each target directory during the walk, so it should not
# check the parent directory again before writing each file.
mkdir -p localdir/sub
echo one > localdir/a.txt
echo two > localdir/b.txt
echo three > localdir/sub/c.txt

cleanup() {
rm -rf localdir "$OUT_REQUESTS"
}
trap cleanup EXIT

TARGET="dbfs:/Volumes/main/default/fs-cp-r-requests/dst"
trace $CLI fs cp -r localdir ${TARGET} 2>&1 | sort

title "Recursive copy: a directory PUT per directory, a file PUT per file, no directory HEAD per file"
trace print_requests.py --sort //api/2.0/fs/ | jq -c '{method, path}'

title "Single-file copy still stats the target and checks its parent directory"
trace $CLI fs cp localdir/a.txt ${TARGET}/sub/single.txt
trace print_requests.py --sort //api/2.0/fs/ | jq -c '{method, path}'

title "Copied files"
trace $CLI fs cat ${TARGET}/a.txt
trace $CLI fs cat ${TARGET}/sub/c.txt
1 change: 1 addition & 0 deletions acceptance/cmd/fs/cp-r-requests/test.toml
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
RecordRequests = true
16 changes: 10 additions & 6 deletions cmd/fs/cp.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import (
"os"
"path"
"path/filepath"
"slices"
"strings"
"sync"

Expand Down Expand Up @@ -102,7 +103,9 @@ func (c *copy) cpDirToDir(ctx context.Context, sourceDir, targetDir string) erro
if ctx.Err() != nil {
return ctx.Err()
}
return c.cpFileToFile(ctx, sourcePath, targetPath)
// The walk already created the target directory, so skip the filer's
// per-file parent-directory check.
return c.cpFileToFile(ctx, sourcePath, targetPath, filer.CreateParentDirectories)
})
return nil
})
Expand All @@ -121,7 +124,9 @@ func (c *copy) cpFileToDir(ctx context.Context, sourcePath, targetDir string) er
return c.cpFileToFile(ctx, sourcePath, targetPath)
}

func (c *copy) cpFileToFile(ctx context.Context, sourcePath, targetPath string) error {
// cpFileToFile copies one file. extraModes are passed to Write, in addition to
// OverwriteIfExists when --overwrite is set.
func (c *copy) cpFileToFile(ctx context.Context, sourcePath, targetPath string, extraModes ...filer.WriteMode) error {
// Get reader for file at source path
r, err := c.sourceFiler.Read(ctx, sourcePath)
if err != nil {
Expand All @@ -140,12 +145,11 @@ func (c *copy) cpFileToFile(ctx context.Context, sourcePath, targetPath string)
closeProgress = stop
}

var writeErr error
modes := slices.Clone(extraModes)
if c.overwrite {
writeErr = c.targetFiler.Write(ctx, targetPath, r, filer.OverwriteIfExists)
} else {
writeErr = c.targetFiler.Write(ctx, targetPath, r)
modes = append(modes, filer.OverwriteIfExists)
}
writeErr := c.targetFiler.Write(ctx, targetPath, r, modes...)
closeProgress()

// skip if file already exists
Expand Down
96 changes: 96 additions & 0 deletions cmd/fs/cp_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,13 +3,20 @@ package fs
import (
"context"
"errors"
"fmt"
"io"
"io/fs"
"path"
"slices"
"strings"
"sync"
"testing"
"time"

"github.com/databricks/cli/libs/cmdio"
"github.com/databricks/cli/libs/filer"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)

// mockFiler mocks filer.Filer.
Expand Down Expand Up @@ -195,3 +202,92 @@ func TestCp_cpDirToDir_contextCancellation(t *testing.T) {
})
}
}

func TestCp_writeModes(t *testing.T) {
// src/
// ├── file1.txt
// └── subdir/
// └── file2.txt
sourceFiler := &mockFiler{
stat: func(ctx context.Context, path string) (fs.FileInfo, error) {
isDir := path == "src" || path == "src/subdir"
return mockFileInfo{name: path, isDir: isDir}, nil
},
readDir: func(ctx context.Context, path string) ([]fs.DirEntry, error) {
switch path {
case "src":
return []fs.DirEntry{
mockDirEntry{name: "file1.txt"},
mockDirEntry{name: "subdir", isDir: true},
}, nil
case "src/subdir":
return []fs.DirEntry{mockDirEntry{name: "file2.txt"}}, nil
}
return nil, nil
},
read: func(ctx context.Context, path string) (io.ReadCloser, error) {
return io.NopCloser(strings.NewReader("content")), nil
},
}

// The target filer records the modes of each write, and fails a write whose
// directory the walk has not created yet: skipping the check is only safe
// because of that Mkdir.
newTargetFiler := func(got map[string][]filer.WriteMode) *mockFiler {
var mu sync.Mutex
created := map[string]bool{}
return &mockFiler{
mkdir: func(ctx context.Context, dir string) error {
mu.Lock()
defer mu.Unlock()
created[dir] = true
return nil
},
write: func(ctx context.Context, p string, r io.Reader, mode ...filer.WriteMode) error {
mu.Lock()
defer mu.Unlock()
if slices.Contains(mode, filer.CreateParentDirectories) && !created[path.Dir(p)] {
return fmt.Errorf("write to %s before Mkdir of its directory", p)
}
got[p] = mode
return nil
},
}
}

for _, overwrite := range []bool{false, true} {
t.Run(fmt.Sprintf("overwrite=%t", overwrite), func(t *testing.T) {
want := []filer.WriteMode{filer.CreateParentDirectories}
if overwrite {
want = append(want, filer.OverwriteIfExists)
}

// A recursive copy creates each target directory during the walk,
// so it skips the per-file parent-directory check.
gotDir := map[string][]filer.WriteMode{}
c := &copy{
recursive: true,
overwrite: overwrite,
concurrency: 2,
sourceFiler: sourceFiler,
targetFiler: newTargetFiler(gotDir),
}
require.NoError(t, c.cpDirToDir(cmdio.MockDiscard(t.Context()), "src", "dst"))
assert.Equal(t, map[string][]filer.WriteMode{
"dst/file1.txt": want,
"dst/subdir/file2.txt": want,
}, gotDir)

// A single-file copy keeps the check: its target directory was not
// created by this command.
gotFile := map[string][]filer.WriteMode{}
c.targetFiler = newTargetFiler(gotFile)
require.NoError(t, c.cpFileToFile(cmdio.MockDiscard(t.Context()), "src/file1.txt", "dst/file1.txt"))
var wantFile []filer.WriteMode
if overwrite {
wantFile = []filer.WriteMode{filer.OverwriteIfExists}
}
assert.Equal(t, map[string][]filer.WriteMode{"dst/file1.txt": wantFile}, gotFile)
})
}
}
Loading