From 521797e1f812da4137688ea250dbe87c7cace91a Mon Sep 17 00:00:00 2001 From: Yuanjie Ding Date: Tue, 17 Nov 2026 09:00:12 +0000 Subject: [PATCH 1/3] Skip the per-file parent-directory check in fs cp -r A recursive copy creates every target directory during the walk, before it queues the files in that directory. The filer then checked the parent directory again before each write (a HEAD on the Files API, get-status on DBFS), doubling the request count for copies of many small files. Pass filer.CreateParentDirectories for files written by cpDirToDir so the filer skips that check. Single-file copies are unchanged. Co-authored-by: Isaac --- .nextchanges/cli/fs-cp-r-skip-parent-check.md | 1 + acceptance/cmd/fs/cp-r-requests/out.test.toml | 1 + acceptance/cmd/fs/cp-r-requests/output.txt | 33 ++++++++ acceptance/cmd/fs/cp-r-requests/script | 25 ++++++ acceptance/cmd/fs/cp-r-requests/test.toml | 1 + cmd/fs/cp.go | 16 ++-- cmd/fs/cp_test.go | 81 +++++++++++++++++++ 7 files changed, 152 insertions(+), 6 deletions(-) create mode 100644 .nextchanges/cli/fs-cp-r-skip-parent-check.md create mode 100644 acceptance/cmd/fs/cp-r-requests/out.test.toml create mode 100644 acceptance/cmd/fs/cp-r-requests/output.txt create mode 100644 acceptance/cmd/fs/cp-r-requests/script create mode 100644 acceptance/cmd/fs/cp-r-requests/test.toml diff --git a/.nextchanges/cli/fs-cp-r-skip-parent-check.md b/.nextchanges/cli/fs-cp-r-skip-parent-check.md new file mode 100644 index 00000000000..7a141677beb --- /dev/null +++ b/.nextchanges/cli/fs-cp-r-skip-parent-check.md @@ -0,0 +1 @@ +* `databricks fs cp -r` no longer checks that the parent directory exists before each file it copies, cutting the number of requests for a copy of many small files by about half. diff --git a/acceptance/cmd/fs/cp-r-requests/out.test.toml b/acceptance/cmd/fs/cp-r-requests/out.test.toml new file mode 100644 index 00000000000..18b1a88417e --- /dev/null +++ b/acceptance/cmd/fs/cp-r-requests/out.test.toml @@ -0,0 +1 @@ +Cloud = false diff --git a/acceptance/cmd/fs/cp-r-requests/output.txt b/acceptance/cmd/fs/cp-r-requests/output.txt new file mode 100644 index 00000000000..c08472017b5 --- /dev/null +++ b/acceptance/cmd/fs/cp-r-requests/output.txt @@ -0,0 +1,33 @@ + +>>> [CLI] fs cp -r localdir dbfs:/Volumes/main/default/fs-cp-r-requests/dst +localdir/a.txt -> dbfs:/Volumes/main/default/fs-cp-r-requests/dst/a.txt +localdir/b.txt -> dbfs:/Volumes/main/default/fs-cp-r-requests/dst/b.txt +localdir/sub/c.txt -> dbfs:/Volumes/main/default/fs-cp-r-requests/dst/sub/c.txt + +=== Recursive copy: one mkdir per directory, one upload per file, no per-file directory check + +>>> print_requests.py --sort //api/2.0/fs/ +{"method":"PUT","path":"/api/2.0/fs/directories/Volumes/main/default/fs-cp-r-requests/dst"} +{"method":"PUT","path":"/api/2.0/fs/directories/Volumes/main/default/fs-cp-r-requests/dst/sub"} +{"method":"PUT","path":"/api/2.0/fs/files/Volumes/main/default/fs-cp-r-requests/dst/a.txt"} +{"method":"PUT","path":"/api/2.0/fs/files/Volumes/main/default/fs-cp-r-requests/dst/b.txt"} +{"method":"PUT","path":"/api/2.0/fs/files/Volumes/main/default/fs-cp-r-requests/dst/sub/c.txt"} + +=== Single-file copy still checks the parent directory + +>>> [CLI] fs cp localdir/a.txt dbfs:/Volumes/main/default/fs-cp-r-requests/dst/sub/single.txt +localdir/a.txt -> dbfs:/Volumes/main/default/fs-cp-r-requests/dst/sub/single.txt + +>>> print_requests.py --sort //api/2.0/fs/ +{"method":"HEAD","path":"/api/2.0/fs/directories/Volumes/main/default/fs-cp-r-requests/dst/sub"} +{"method":"HEAD","path":"/api/2.0/fs/directories/Volumes/main/default/fs-cp-r-requests/dst/sub/single.txt"} +{"method":"HEAD","path":"/api/2.0/fs/files/Volumes/main/default/fs-cp-r-requests/dst/sub/single.txt"} +{"method":"PUT","path":"/api/2.0/fs/files/Volumes/main/default/fs-cp-r-requests/dst/sub/single.txt"} + +=== Copied files + +>>> [CLI] fs cat dbfs:/Volumes/main/default/fs-cp-r-requests/dst/a.txt +one + +>>> [CLI] fs cat dbfs:/Volumes/main/default/fs-cp-r-requests/dst/sub/c.txt +three diff --git a/acceptance/cmd/fs/cp-r-requests/script b/acceptance/cmd/fs/cp-r-requests/script new file mode 100644 index 00000000000..fde6e975a73 --- /dev/null +++ b/acceptance/cmd/fs/cp-r-requests/script @@ -0,0 +1,25 @@ +# fs cp -r creates each target directory during the walk, so it should not +# check the parent directory again before writing each file. +mkdir -p localdir/sub +echo one > localdir/a.txt +echo two > localdir/b.txt +echo three > localdir/sub/c.txt + +cleanup() { + rm -rf localdir "$OUT_REQUESTS" +} +trap cleanup EXIT + +TARGET="dbfs:/Volumes/main/default/fs-cp-r-requests/dst" +trace $CLI fs cp -r localdir ${TARGET} 2>&1 | sort + +title "Recursive copy: one mkdir per directory, one upload per file, no per-file directory check" +trace print_requests.py --sort //api/2.0/fs/ | jq -c '{method, path}' + +title "Single-file copy still checks the parent directory" +trace $CLI fs cp localdir/a.txt ${TARGET}/sub/single.txt +trace print_requests.py --sort //api/2.0/fs/ | jq -c '{method, path}' + +title "Copied files" +trace $CLI fs cat ${TARGET}/a.txt +trace $CLI fs cat ${TARGET}/sub/c.txt diff --git a/acceptance/cmd/fs/cp-r-requests/test.toml b/acceptance/cmd/fs/cp-r-requests/test.toml new file mode 100644 index 00000000000..159efe02696 --- /dev/null +++ b/acceptance/cmd/fs/cp-r-requests/test.toml @@ -0,0 +1 @@ +RecordRequests = true diff --git a/cmd/fs/cp.go b/cmd/fs/cp.go index 734faaeb996..d12b18865f8 100644 --- a/cmd/fs/cp.go +++ b/cmd/fs/cp.go @@ -8,6 +8,7 @@ import ( "os" "path" "path/filepath" + "slices" "strings" "sync" @@ -102,7 +103,9 @@ func (c *copy) cpDirToDir(ctx context.Context, sourceDir, targetDir string) erro if ctx.Err() != nil { return ctx.Err() } - return c.cpFileToFile(ctx, sourcePath, targetPath) + // The walk has already created the target directory for this file, + // so tell the filer not to check that it exists before every write. + return c.cpFileToFile(ctx, sourcePath, targetPath, filer.CreateParentDirectories) }) return nil }) @@ -121,7 +124,9 @@ func (c *copy) cpFileToDir(ctx context.Context, sourcePath, targetDir string) er return c.cpFileToFile(ctx, sourcePath, targetPath) } -func (c *copy) cpFileToFile(ctx context.Context, sourcePath, targetPath string) error { +// cpFileToFile copies a single file. Callers pass extraModes to add write modes +// beyond the one implied by --overwrite. +func (c *copy) cpFileToFile(ctx context.Context, sourcePath, targetPath string, extraModes ...filer.WriteMode) error { // Get reader for file at source path r, err := c.sourceFiler.Read(ctx, sourcePath) if err != nil { @@ -140,12 +145,11 @@ func (c *copy) cpFileToFile(ctx context.Context, sourcePath, targetPath string) closeProgress = stop } - var writeErr error + modes := slices.Clone(extraModes) if c.overwrite { - writeErr = c.targetFiler.Write(ctx, targetPath, r, filer.OverwriteIfExists) - } else { - writeErr = c.targetFiler.Write(ctx, targetPath, r) + modes = append(modes, filer.OverwriteIfExists) } + writeErr := c.targetFiler.Write(ctx, targetPath, r, modes...) closeProgress() // skip if file already exists diff --git a/cmd/fs/cp_test.go b/cmd/fs/cp_test.go index b50ee0658f9..37ceaebf8b7 100644 --- a/cmd/fs/cp_test.go +++ b/cmd/fs/cp_test.go @@ -3,13 +3,18 @@ package fs import ( "context" "errors" + "fmt" "io" "io/fs" "strings" + "sync" "testing" "time" + "github.com/databricks/cli/libs/cmdio" "github.com/databricks/cli/libs/filer" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) // mockFiler mocks filer.Filer. @@ -195,3 +200,79 @@ func TestCp_cpDirToDir_contextCancellation(t *testing.T) { }) } } + +func TestCp_writeModes(t *testing.T) { + // src/ + // ├── file1.txt + // └── subdir/ + // └── file2.txt + sourceFiler := &mockFiler{ + stat: func(ctx context.Context, path string) (fs.FileInfo, error) { + isDir := path == "src" || path == "src/subdir" + return mockFileInfo{name: path, isDir: isDir}, nil + }, + readDir: func(ctx context.Context, path string) ([]fs.DirEntry, error) { + switch path { + case "src": + return []fs.DirEntry{ + mockDirEntry{name: "file1.txt"}, + mockDirEntry{name: "subdir", isDir: true}, + }, nil + case "src/subdir": + return []fs.DirEntry{mockDirEntry{name: "file2.txt"}}, nil + } + return nil, nil + }, + read: func(ctx context.Context, path string) (io.ReadCloser, error) { + return io.NopCloser(strings.NewReader("content")), nil + }, + } + + newTargetFiler := func(got map[string][]filer.WriteMode) *mockFiler { + var mu sync.Mutex + return &mockFiler{ + write: func(ctx context.Context, path string, r io.Reader, mode ...filer.WriteMode) error { + mu.Lock() + defer mu.Unlock() + got[path] = mode + return nil + }, + } + } + + for _, overwrite := range []bool{false, true} { + t.Run(fmt.Sprintf("overwrite=%t", overwrite), func(t *testing.T) { + want := []filer.WriteMode{filer.CreateParentDirectories} + if overwrite { + want = append(want, filer.OverwriteIfExists) + } + + // A recursive copy creates each target directory during the walk, + // so it skips the per-file parent-directory check. + gotDir := map[string][]filer.WriteMode{} + c := ©{ + recursive: true, + overwrite: overwrite, + concurrency: 2, + sourceFiler: sourceFiler, + targetFiler: newTargetFiler(gotDir), + } + require.NoError(t, c.cpDirToDir(cmdio.MockDiscard(t.Context()), "src", "dst")) + assert.Equal(t, map[string][]filer.WriteMode{ + "dst/file1.txt": want, + "dst/subdir/file2.txt": want, + }, gotDir) + + // A single-file copy keeps the check: its target directory was not + // created by this command. + gotFile := map[string][]filer.WriteMode{} + c.targetFiler = newTargetFiler(gotFile) + require.NoError(t, c.cpFileToFile(cmdio.MockDiscard(t.Context()), "src/file1.txt", "dst/file1.txt")) + var wantFile []filer.WriteMode + if overwrite { + wantFile = []filer.WriteMode{filer.OverwriteIfExists} + } + assert.Equal(t, map[string][]filer.WriteMode{"dst/file1.txt": wantFile}, gotFile) + }) + } +} From 40a9b2f9c0464a9ae6cc5a2603d976fee6ce141e Mon Sep 17 00:00:00 2001 From: Yuanjie Ding Date: Tue, 17 Nov 2026 09:01:27 +0000 Subject: [PATCH 2/3] Link the release note to #7711 Co-authored-by: Isaac --- .nextchanges/cli/fs-cp-r-skip-parent-check.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.nextchanges/cli/fs-cp-r-skip-parent-check.md b/.nextchanges/cli/fs-cp-r-skip-parent-check.md index 7a141677beb..5beae1f32e0 100644 --- a/.nextchanges/cli/fs-cp-r-skip-parent-check.md +++ b/.nextchanges/cli/fs-cp-r-skip-parent-check.md @@ -1 +1 @@ -* `databricks fs cp -r` no longer checks that the parent directory exists before each file it copies, cutting the number of requests for a copy of many small files by about half. +* `databricks fs cp -r` no longer checks that the parent directory exists before each file it copies, cutting the number of requests for a copy of many small files by about half. ([#7711](https://github.com/databricks/cli/pull/7711)) From 31b84c469a8d0247c5423e41fbe38fe010f6df64 Mon Sep 17 00:00:00 2001 From: Yuanjie Ding Date: Tue, 17 Nov 2026 09:05:54 +0000 Subject: [PATCH 3/3] Address review: tighten comments, test Mkdir ordering, fix claims Co-authored-by: Isaac --- .nextchanges/cli/fs-cp-r-skip-parent-check.md | 2 +- acceptance/cmd/fs/cp-r-requests/output.txt | 4 ++-- acceptance/cmd/fs/cp-r-requests/script | 4 ++-- cmd/fs/cp.go | 8 ++++---- cmd/fs/cp_test.go | 19 +++++++++++++++++-- 5 files changed, 26 insertions(+), 11 deletions(-) diff --git a/.nextchanges/cli/fs-cp-r-skip-parent-check.md b/.nextchanges/cli/fs-cp-r-skip-parent-check.md index 5beae1f32e0..ed2980129f8 100644 --- a/.nextchanges/cli/fs-cp-r-skip-parent-check.md +++ b/.nextchanges/cli/fs-cp-r-skip-parent-check.md @@ -1 +1 @@ -* `databricks fs cp -r` no longer checks that the parent directory exists before each file it copies, cutting the number of requests for a copy of many small files by about half. ([#7711](https://github.com/databricks/cli/pull/7711)) +* `databricks fs cp -r` no longer checks that each file's parent directory exists before writing it, because the copy has already created that directory. This saves one request per file: a copy to a Volume sends one request per small file instead of two. ([#7711](https://github.com/databricks/cli/pull/7711)) diff --git a/acceptance/cmd/fs/cp-r-requests/output.txt b/acceptance/cmd/fs/cp-r-requests/output.txt index c08472017b5..1982e113bd2 100644 --- a/acceptance/cmd/fs/cp-r-requests/output.txt +++ b/acceptance/cmd/fs/cp-r-requests/output.txt @@ -4,7 +4,7 @@ localdir/a.txt -> dbfs:/Volumes/main/default/fs-cp-r-requests/dst/a.txt localdir/b.txt -> dbfs:/Volumes/main/default/fs-cp-r-requests/dst/b.txt localdir/sub/c.txt -> dbfs:/Volumes/main/default/fs-cp-r-requests/dst/sub/c.txt -=== Recursive copy: one mkdir per directory, one upload per file, no per-file directory check +=== Recursive copy: a directory PUT per directory, a file PUT per file, no directory HEAD per file >>> print_requests.py --sort //api/2.0/fs/ {"method":"PUT","path":"/api/2.0/fs/directories/Volumes/main/default/fs-cp-r-requests/dst"} @@ -13,7 +13,7 @@ localdir/sub/c.txt -> dbfs:/Volumes/main/default/fs-cp-r-requests/dst/sub/c.txt {"method":"PUT","path":"/api/2.0/fs/files/Volumes/main/default/fs-cp-r-requests/dst/b.txt"} {"method":"PUT","path":"/api/2.0/fs/files/Volumes/main/default/fs-cp-r-requests/dst/sub/c.txt"} -=== Single-file copy still checks the parent directory +=== Single-file copy still stats the target and checks its parent directory >>> [CLI] fs cp localdir/a.txt dbfs:/Volumes/main/default/fs-cp-r-requests/dst/sub/single.txt localdir/a.txt -> dbfs:/Volumes/main/default/fs-cp-r-requests/dst/sub/single.txt diff --git a/acceptance/cmd/fs/cp-r-requests/script b/acceptance/cmd/fs/cp-r-requests/script index fde6e975a73..f36d1c64a2d 100644 --- a/acceptance/cmd/fs/cp-r-requests/script +++ b/acceptance/cmd/fs/cp-r-requests/script @@ -13,10 +13,10 @@ trap cleanup EXIT TARGET="dbfs:/Volumes/main/default/fs-cp-r-requests/dst" trace $CLI fs cp -r localdir ${TARGET} 2>&1 | sort -title "Recursive copy: one mkdir per directory, one upload per file, no per-file directory check" +title "Recursive copy: a directory PUT per directory, a file PUT per file, no directory HEAD per file" trace print_requests.py --sort //api/2.0/fs/ | jq -c '{method, path}' -title "Single-file copy still checks the parent directory" +title "Single-file copy still stats the target and checks its parent directory" trace $CLI fs cp localdir/a.txt ${TARGET}/sub/single.txt trace print_requests.py --sort //api/2.0/fs/ | jq -c '{method, path}' diff --git a/cmd/fs/cp.go b/cmd/fs/cp.go index d12b18865f8..957aaaf2f55 100644 --- a/cmd/fs/cp.go +++ b/cmd/fs/cp.go @@ -103,8 +103,8 @@ func (c *copy) cpDirToDir(ctx context.Context, sourceDir, targetDir string) erro if ctx.Err() != nil { return ctx.Err() } - // The walk has already created the target directory for this file, - // so tell the filer not to check that it exists before every write. + // The walk already created the target directory, so skip the filer's + // per-file parent-directory check. return c.cpFileToFile(ctx, sourcePath, targetPath, filer.CreateParentDirectories) }) return nil @@ -124,8 +124,8 @@ func (c *copy) cpFileToDir(ctx context.Context, sourcePath, targetDir string) er return c.cpFileToFile(ctx, sourcePath, targetPath) } -// cpFileToFile copies a single file. Callers pass extraModes to add write modes -// beyond the one implied by --overwrite. +// cpFileToFile copies one file. extraModes are passed to Write, in addition to +// OverwriteIfExists when --overwrite is set. func (c *copy) cpFileToFile(ctx context.Context, sourcePath, targetPath string, extraModes ...filer.WriteMode) error { // Get reader for file at source path r, err := c.sourceFiler.Read(ctx, sourcePath) diff --git a/cmd/fs/cp_test.go b/cmd/fs/cp_test.go index 37ceaebf8b7..a25416d6bc5 100644 --- a/cmd/fs/cp_test.go +++ b/cmd/fs/cp_test.go @@ -6,6 +6,8 @@ import ( "fmt" "io" "io/fs" + "path" + "slices" "strings" "sync" "testing" @@ -228,13 +230,26 @@ func TestCp_writeModes(t *testing.T) { }, } + // The target filer records the modes of each write, and fails a write whose + // directory the walk has not created yet: skipping the check is only safe + // because of that Mkdir. newTargetFiler := func(got map[string][]filer.WriteMode) *mockFiler { var mu sync.Mutex + created := map[string]bool{} return &mockFiler{ - write: func(ctx context.Context, path string, r io.Reader, mode ...filer.WriteMode) error { + mkdir: func(ctx context.Context, dir string) error { mu.Lock() defer mu.Unlock() - got[path] = mode + created[dir] = true + return nil + }, + write: func(ctx context.Context, p string, r io.Reader, mode ...filer.WriteMode) error { + mu.Lock() + defer mu.Unlock() + if slices.Contains(mode, filer.CreateParentDirectories) && !created[path.Dir(p)] { + return fmt.Errorf("write to %s before Mkdir of its directory", p) + } + got[p] = mode return nil }, }