diff --git a/.nextchanges/cli/fs-cp-r-skip-parent-check.md b/.nextchanges/cli/fs-cp-r-skip-parent-check.md new file mode 100644 index 00000000000..ed2980129f8 --- /dev/null +++ b/.nextchanges/cli/fs-cp-r-skip-parent-check.md @@ -0,0 +1 @@ +* `databricks fs cp -r` no longer checks that each file's parent directory exists before writing it, because the copy has already created that directory. This saves one request per file: a copy to a Volume sends one request per small file instead of two. ([#7711](https://github.com/databricks/cli/pull/7711)) diff --git a/acceptance/cmd/fs/cp-r-requests/out.test.toml b/acceptance/cmd/fs/cp-r-requests/out.test.toml new file mode 100644 index 00000000000..18b1a88417e --- /dev/null +++ b/acceptance/cmd/fs/cp-r-requests/out.test.toml @@ -0,0 +1 @@ +Cloud = false diff --git a/acceptance/cmd/fs/cp-r-requests/output.txt b/acceptance/cmd/fs/cp-r-requests/output.txt new file mode 100644 index 00000000000..1982e113bd2 --- /dev/null +++ b/acceptance/cmd/fs/cp-r-requests/output.txt @@ -0,0 +1,33 @@ + +>>> [CLI] fs cp -r localdir dbfs:/Volumes/main/default/fs-cp-r-requests/dst +localdir/a.txt -> dbfs:/Volumes/main/default/fs-cp-r-requests/dst/a.txt +localdir/b.txt -> dbfs:/Volumes/main/default/fs-cp-r-requests/dst/b.txt +localdir/sub/c.txt -> dbfs:/Volumes/main/default/fs-cp-r-requests/dst/sub/c.txt + +=== Recursive copy: a directory PUT per directory, a file PUT per file, no directory HEAD per file + +>>> print_requests.py --sort //api/2.0/fs/ +{"method":"PUT","path":"/api/2.0/fs/directories/Volumes/main/default/fs-cp-r-requests/dst"} +{"method":"PUT","path":"/api/2.0/fs/directories/Volumes/main/default/fs-cp-r-requests/dst/sub"} +{"method":"PUT","path":"/api/2.0/fs/files/Volumes/main/default/fs-cp-r-requests/dst/a.txt"} +{"method":"PUT","path":"/api/2.0/fs/files/Volumes/main/default/fs-cp-r-requests/dst/b.txt"} +{"method":"PUT","path":"/api/2.0/fs/files/Volumes/main/default/fs-cp-r-requests/dst/sub/c.txt"} + +=== Single-file copy still stats the target and checks its parent directory + +>>> [CLI] fs cp localdir/a.txt dbfs:/Volumes/main/default/fs-cp-r-requests/dst/sub/single.txt +localdir/a.txt -> dbfs:/Volumes/main/default/fs-cp-r-requests/dst/sub/single.txt + +>>> print_requests.py --sort //api/2.0/fs/ +{"method":"HEAD","path":"/api/2.0/fs/directories/Volumes/main/default/fs-cp-r-requests/dst/sub"} +{"method":"HEAD","path":"/api/2.0/fs/directories/Volumes/main/default/fs-cp-r-requests/dst/sub/single.txt"} +{"method":"HEAD","path":"/api/2.0/fs/files/Volumes/main/default/fs-cp-r-requests/dst/sub/single.txt"} +{"method":"PUT","path":"/api/2.0/fs/files/Volumes/main/default/fs-cp-r-requests/dst/sub/single.txt"} + +=== Copied files + +>>> [CLI] fs cat dbfs:/Volumes/main/default/fs-cp-r-requests/dst/a.txt +one + +>>> [CLI] fs cat dbfs:/Volumes/main/default/fs-cp-r-requests/dst/sub/c.txt +three diff --git a/acceptance/cmd/fs/cp-r-requests/script b/acceptance/cmd/fs/cp-r-requests/script new file mode 100644 index 00000000000..f36d1c64a2d --- /dev/null +++ b/acceptance/cmd/fs/cp-r-requests/script @@ -0,0 +1,25 @@ +# fs cp -r creates each target directory during the walk, so it should not +# check the parent directory again before writing each file. +mkdir -p localdir/sub +echo one > localdir/a.txt +echo two > localdir/b.txt +echo three > localdir/sub/c.txt + +cleanup() { + rm -rf localdir "$OUT_REQUESTS" +} +trap cleanup EXIT + +TARGET="dbfs:/Volumes/main/default/fs-cp-r-requests/dst" +trace $CLI fs cp -r localdir ${TARGET} 2>&1 | sort + +title "Recursive copy: a directory PUT per directory, a file PUT per file, no directory HEAD per file" +trace print_requests.py --sort //api/2.0/fs/ | jq -c '{method, path}' + +title "Single-file copy still stats the target and checks its parent directory" +trace $CLI fs cp localdir/a.txt ${TARGET}/sub/single.txt +trace print_requests.py --sort //api/2.0/fs/ | jq -c '{method, path}' + +title "Copied files" +trace $CLI fs cat ${TARGET}/a.txt +trace $CLI fs cat ${TARGET}/sub/c.txt diff --git a/acceptance/cmd/fs/cp-r-requests/test.toml b/acceptance/cmd/fs/cp-r-requests/test.toml new file mode 100644 index 00000000000..159efe02696 --- /dev/null +++ b/acceptance/cmd/fs/cp-r-requests/test.toml @@ -0,0 +1 @@ +RecordRequests = true diff --git a/cmd/fs/cp.go b/cmd/fs/cp.go index 734faaeb996..957aaaf2f55 100644 --- a/cmd/fs/cp.go +++ b/cmd/fs/cp.go @@ -8,6 +8,7 @@ import ( "os" "path" "path/filepath" + "slices" "strings" "sync" @@ -102,7 +103,9 @@ func (c *copy) cpDirToDir(ctx context.Context, sourceDir, targetDir string) erro if ctx.Err() != nil { return ctx.Err() } - return c.cpFileToFile(ctx, sourcePath, targetPath) + // The walk already created the target directory, so skip the filer's + // per-file parent-directory check. + return c.cpFileToFile(ctx, sourcePath, targetPath, filer.CreateParentDirectories) }) return nil }) @@ -121,7 +124,9 @@ func (c *copy) cpFileToDir(ctx context.Context, sourcePath, targetDir string) er return c.cpFileToFile(ctx, sourcePath, targetPath) } -func (c *copy) cpFileToFile(ctx context.Context, sourcePath, targetPath string) error { +// cpFileToFile copies one file. extraModes are passed to Write, in addition to +// OverwriteIfExists when --overwrite is set. +func (c *copy) cpFileToFile(ctx context.Context, sourcePath, targetPath string, extraModes ...filer.WriteMode) error { // Get reader for file at source path r, err := c.sourceFiler.Read(ctx, sourcePath) if err != nil { @@ -140,12 +145,11 @@ func (c *copy) cpFileToFile(ctx context.Context, sourcePath, targetPath string) closeProgress = stop } - var writeErr error + modes := slices.Clone(extraModes) if c.overwrite { - writeErr = c.targetFiler.Write(ctx, targetPath, r, filer.OverwriteIfExists) - } else { - writeErr = c.targetFiler.Write(ctx, targetPath, r) + modes = append(modes, filer.OverwriteIfExists) } + writeErr := c.targetFiler.Write(ctx, targetPath, r, modes...) closeProgress() // skip if file already exists diff --git a/cmd/fs/cp_test.go b/cmd/fs/cp_test.go index b50ee0658f9..a25416d6bc5 100644 --- a/cmd/fs/cp_test.go +++ b/cmd/fs/cp_test.go @@ -3,13 +3,20 @@ package fs import ( "context" "errors" + "fmt" "io" "io/fs" + "path" + "slices" "strings" + "sync" "testing" "time" + "github.com/databricks/cli/libs/cmdio" "github.com/databricks/cli/libs/filer" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) // mockFiler mocks filer.Filer. @@ -195,3 +202,92 @@ func TestCp_cpDirToDir_contextCancellation(t *testing.T) { }) } } + +func TestCp_writeModes(t *testing.T) { + // src/ + // ├── file1.txt + // └── subdir/ + // └── file2.txt + sourceFiler := &mockFiler{ + stat: func(ctx context.Context, path string) (fs.FileInfo, error) { + isDir := path == "src" || path == "src/subdir" + return mockFileInfo{name: path, isDir: isDir}, nil + }, + readDir: func(ctx context.Context, path string) ([]fs.DirEntry, error) { + switch path { + case "src": + return []fs.DirEntry{ + mockDirEntry{name: "file1.txt"}, + mockDirEntry{name: "subdir", isDir: true}, + }, nil + case "src/subdir": + return []fs.DirEntry{mockDirEntry{name: "file2.txt"}}, nil + } + return nil, nil + }, + read: func(ctx context.Context, path string) (io.ReadCloser, error) { + return io.NopCloser(strings.NewReader("content")), nil + }, + } + + // The target filer records the modes of each write, and fails a write whose + // directory the walk has not created yet: skipping the check is only safe + // because of that Mkdir. + newTargetFiler := func(got map[string][]filer.WriteMode) *mockFiler { + var mu sync.Mutex + created := map[string]bool{} + return &mockFiler{ + mkdir: func(ctx context.Context, dir string) error { + mu.Lock() + defer mu.Unlock() + created[dir] = true + return nil + }, + write: func(ctx context.Context, p string, r io.Reader, mode ...filer.WriteMode) error { + mu.Lock() + defer mu.Unlock() + if slices.Contains(mode, filer.CreateParentDirectories) && !created[path.Dir(p)] { + return fmt.Errorf("write to %s before Mkdir of its directory", p) + } + got[p] = mode + return nil + }, + } + } + + for _, overwrite := range []bool{false, true} { + t.Run(fmt.Sprintf("overwrite=%t", overwrite), func(t *testing.T) { + want := []filer.WriteMode{filer.CreateParentDirectories} + if overwrite { + want = append(want, filer.OverwriteIfExists) + } + + // A recursive copy creates each target directory during the walk, + // so it skips the per-file parent-directory check. + gotDir := map[string][]filer.WriteMode{} + c := ©{ + recursive: true, + overwrite: overwrite, + concurrency: 2, + sourceFiler: sourceFiler, + targetFiler: newTargetFiler(gotDir), + } + require.NoError(t, c.cpDirToDir(cmdio.MockDiscard(t.Context()), "src", "dst")) + assert.Equal(t, map[string][]filer.WriteMode{ + "dst/file1.txt": want, + "dst/subdir/file2.txt": want, + }, gotDir) + + // A single-file copy keeps the check: its target directory was not + // created by this command. + gotFile := map[string][]filer.WriteMode{} + c.targetFiler = newTargetFiler(gotFile) + require.NoError(t, c.cpFileToFile(cmdio.MockDiscard(t.Context()), "src/file1.txt", "dst/file1.txt")) + var wantFile []filer.WriteMode + if overwrite { + wantFile = []filer.WriteMode{filer.OverwriteIfExists} + } + assert.Equal(t, map[string][]filer.WriteMode{"dst/file1.txt": wantFile}, gotFile) + }) + } +}