Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .nextchanges/cli/fs-cp-r-keep-going.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
* `databricks fs cp -r` keeps going when a file fails. A file that fails with a temporary error (HTTP 408, 429, 5xx, or a network error) is retried once after the rest of the copy, and other failed files are reported while the copy continues. The copy stops early on an authentication or permission error, or on a directory it cannot list or create. A summary goes to stderr, and the command exits with code 1 if any file was not copied. ([#7712](https://github.com/databricks/cli/pull/7712))
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
file 1
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
file 2
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
INJECT_ERROR
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
file 4
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
Cloud = false
20 changes: 20 additions & 0 deletions acceptance/cmd/fs/cp-r-keep-going/persistent-500/output.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@

>>> cp_r localdir dbfs:/Volumes/main/default/keep-going/dst --concurrency 2
Copy incomplete: 3 copied, 0 skipped (already existed), 1 failed, 0 not attempted.
Failed:
localdir/f3.txt -> dbfs:/Volumes/main/default/keep-going/dst/f3.txt: HTTP 500 Internal Server Error
Error: 1 file was not copied
localdir/f1.txt -> dbfs:/Volumes/main/default/keep-going/dst/f1.txt
localdir/f2.txt -> dbfs:/Volumes/main/default/keep-going/dst/f2.txt
localdir/f4.txt -> dbfs:/Volumes/main/default/keep-going/dst/f4.txt

Exit code: 1

=== Four uploads plus one retry of f3.txt

>>> print_requests.py --summary --sort //api/2.0/fs/files/
PUT /api/2.0/fs/files/Volumes/main/default/keep-going/dst/f1.txt overwrite=false
PUT /api/2.0/fs/files/Volumes/main/default/keep-going/dst/f2.txt overwrite=false
PUT /api/2.0/fs/files/Volumes/main/default/keep-going/dst/f3.txt overwrite=false
PUT /api/2.0/fs/files/Volumes/main/default/keep-going/dst/f3.txt overwrite=false
PUT /api/2.0/fs/files/Volumes/main/default/keep-going/dst/f4.txt overwrite=false
7 changes: 7 additions & 0 deletions acceptance/cmd/fs/cp-r-keep-going/persistent-500/script
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
# f3.txt contains INJECT_ERROR, so the testserver answers every upload of it
# with a 500 whose body is not JSON. The file gets its retry after the walk,
# then fails alone; the other files are copied and the command exits 1.
errcode trace cp_r localdir ${TARGET} --concurrency 2

title "Four uploads plus one retry of f3.txt"
trace print_requests.py --summary --sort //api/2.0/fs/files/
8 changes: 8 additions & 0 deletions acceptance/cmd/fs/cp-r-keep-going/script.prepare
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
TARGET="dbfs:/Volumes/main/default/keep-going/dst"

# cp_r runs fs cp -r with the given arguments and prints the per-file lines
# sorted, since files are copied in parallel. It returns the exit code of fs cp.
cp_r() {
$CLI fs cp -r "$@" | sort
return "${PIPESTATUS[0]}"
}
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
file 1
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
file 2
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
file 3
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
file 4
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
file 5
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
file 6
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
Cloud = false
20 changes: 20 additions & 0 deletions acceptance/cmd/fs/cp-r-keep-going/single-500/output.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@

>>> cp_r localdir dbfs:/Volumes/main/default/keep-going/dst --concurrency 2
Copied 6 files.
localdir/f1.txt -> dbfs:/Volumes/main/default/keep-going/dst/f1.txt
localdir/f2.txt -> dbfs:/Volumes/main/default/keep-going/dst/f2.txt
localdir/f3.txt -> dbfs:/Volumes/main/default/keep-going/dst/f3.txt
localdir/f4.txt -> dbfs:/Volumes/main/default/keep-going/dst/f4.txt
localdir/f5.txt -> dbfs:/Volumes/main/default/keep-going/dst/f5.txt
localdir/f6.txt -> dbfs:/Volumes/main/default/keep-going/dst/f6.txt

=== Six uploads plus one retry

>>> print_requests.py --summary --sort //api/2.0/fs/files/
PUT /api/2.0/fs/files/Volumes/main/default/keep-going/dst/f1.txt overwrite=false
PUT /api/2.0/fs/files/Volumes/main/default/keep-going/dst/f2.txt overwrite=false
PUT /api/2.0/fs/files/Volumes/main/default/keep-going/dst/f3.txt overwrite=false
PUT /api/2.0/fs/files/Volumes/main/default/keep-going/dst/f3.txt overwrite=false
PUT /api/2.0/fs/files/Volumes/main/default/keep-going/dst/f4.txt overwrite=false
PUT /api/2.0/fs/files/Volumes/main/default/keep-going/dst/f5.txt overwrite=false
PUT /api/2.0/fs/files/Volumes/main/default/keep-going/dst/f6.txt overwrite=false
9 changes: 9 additions & 0 deletions acceptance/cmd/fs/cp-r-keep-going/single-500/script
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
# A 500 is not retried by the filer. The copy defers the file and retries it
# once after the walk, so the copy still succeeds. The fault matches f3.txt by
# its content, since files are uploaded in parallel.
fault.py --body-contains "file 3" "PUT /api/2.0/fs/files/**" 500 0 1

trace cp_r localdir ${TARGET} --concurrency 2

title "Six uploads plus one retry"
trace print_requests.py --summary --sort //api/2.0/fs/files/
1 change: 1 addition & 0 deletions acceptance/cmd/fs/cp-r-keep-going/test.toml
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
RecordRequests = true
1 change: 1 addition & 0 deletions acceptance/cmd/fs/cp-r-requests/output.txt
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@

>>> [CLI] fs cp -r localdir dbfs:/Volumes/main/default/fs-cp-r-requests/dst
Copied 3 files.
localdir/a.txt -> dbfs:/Volumes/main/default/fs-cp-r-requests/dst/a.txt
localdir/b.txt -> dbfs:/Volumes/main/default/fs-cp-r-requests/dst/b.txt
localdir/sub/c.txt -> dbfs:/Volumes/main/default/fs-cp-r-requests/dst/sub/c.txt
Expand Down
1 change: 1 addition & 0 deletions acceptance/cmd/fs/cp/dir-to-dir/output.txt
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@

>>> [CLI] fs cp -r localdir dbfs:/Volumes/main/default/fs-cp-test-[UNIQUE_NAME]/uploaded-dir
Copied 2 files.
localdir/file1.txt -> dbfs:/Volumes/main/default/fs-cp-test-[UNIQUE_NAME]/uploaded-dir/file1.txt
localdir/file2.txt -> dbfs:/Volumes/main/default/fs-cp-test-[UNIQUE_NAME]/uploaded-dir/file2.txt

Expand Down
129 changes: 43 additions & 86 deletions cmd/fs/cp.go
Original file line number Diff line number Diff line change
Expand Up @@ -15,8 +15,8 @@ import (
"github.com/databricks/cli/cmd/root"
"github.com/databricks/cli/libs/cmdio"
"github.com/databricks/cli/libs/filer"
"github.com/databricks/cli/libs/flags"
"github.com/spf13/cobra"
"golang.org/x/sync/errgroup"
)

// defaultConcurrency is the number of files copied in parallel. Each in-flight
Expand Down Expand Up @@ -44,77 +44,21 @@ type copy struct {
// would each fight for the spinner line).
showProgress bool

// outputJSON is set when the command renders JSON. A recursive copy then
// leaves out its one-line summary of a complete copy.
outputJSON bool

mu sync.Mutex // protect output from concurrent writes
}

// cpDirToDir recursively copies the content of a directory to another
// directory.
//
// There is no guarantee on the order in which the files are copied.
//
// The method does not take care of retrying on error; this is considered to
// be the responsibility of the Filer implementation. If a file copy fails,
// the error is returned and the other copies are cancelled.
// directory. There is no guarantee on the order in which the files are copied.
// It keeps going past failed files; see [recursiveCopy].
func (c *copy) cpDirToDir(ctx context.Context, sourceDir, targetDir string) error {
if !c.recursive {
return fmt.Errorf("source path %s is a directory. Please specify the --recursive flag", sourceDir)
}

// Create a cancellable context purely for the purpose of having a way to
// cancel the goroutines in case of error walking the directory.
ctx, cancel := context.WithCancel(ctx)
defer cancel()

// Pool of workers to process copy operations in parallel. The created
// context is the real context for this operation. It is shared by the
// walking function and the goroutines and can be cancelled manually
// by calling the cancel() function of its parent context.
g, ctx := errgroup.WithContext(ctx)
g.SetLimit(c.concurrency)

// Walk the source directory, queueing file copy operations for processing.
sourceFs := filer.NewFS(ctx, c.sourceFiler)
err := fs.WalkDir(sourceFs, sourceDir, func(sourcePath string, d fs.DirEntry, err error) error {
if err != nil {
return err
}

// Compute path relative to the source directory.
relPath, err := filepath.Rel(sourceDir, sourcePath)
if err != nil {
return err
}
relPath = filepath.ToSlash(relPath)

// Compute target path for the file.
targetPath := path.Join(targetDir, relPath)

// Create the directory synchronously. This must happen before files
// are copied into it, and WalkDir guarantees directories are visited
// before their contents.
if d.IsDir() {
return c.targetFiler.Mkdir(ctx, targetPath)
}

g.Go(func() error {
// Goroutines are queued and may start after the context is already
// cancelled (e.g. a prior copy failed). This check aims to avoid
// starting work that will inevitably fail.
if ctx.Err() != nil {
return ctx.Err()
}
// The walk already created the target directory, so skip the filer's
// per-file parent-directory check.
return c.cpFileToFile(ctx, sourcePath, targetPath, filer.CreateParentDirectories)
})
return nil
})
if err != nil {
cancel() // cancel the goroutines
_ = g.Wait() // wait for the goroutines to finish
return err // return the "real" error that led to cancellation
}
return g.Wait()
return (&recursiveCopy{c: c}).run(ctx, sourceDir, targetDir)
}

func (c *copy) cpFileToDir(ctx context.Context, sourcePath, targetDir string) error {
Expand All @@ -124,13 +68,22 @@ func (c *copy) cpFileToDir(ctx context.Context, sourcePath, targetDir string) er
return c.cpFileToFile(ctx, sourcePath, targetPath)
}

// cpFileToFile copies one file. extraModes are passed to Write, in addition to
// OverwriteIfExists when --overwrite is set.
func (c *copy) cpFileToFile(ctx context.Context, sourcePath, targetPath string, extraModes ...filer.WriteMode) error {
func (c *copy) cpFileToFile(ctx context.Context, sourcePath, targetPath string) error {
skipped, err := c.writeFile(ctx, sourcePath, targetPath)
if err != nil {
return err
}
return c.emitResult(ctx, sourcePath, targetPath, skipped)
}

// writeFile copies one file without printing its line, and reports whether it
// was skipped because the target exists. extraModes are passed to Write, in
// addition to OverwriteIfExists when --overwrite is set.
func (c *copy) writeFile(ctx context.Context, sourcePath, targetPath string, extraModes ...filer.WriteMode) (bool, error) {
// Get reader for file at source path
r, err := c.sourceFiler.Read(ctx, sourcePath)
if err != nil {
return err
return false, err
}
defer r.Close()

Expand All @@ -154,25 +107,35 @@ func (c *copy) cpFileToFile(ctx context.Context, sourcePath, targetPath string,

// skip if file already exists
if !c.overwrite && writeErr != nil && errors.Is(writeErr, fs.ErrExist) {
return c.emitFileSkippedEvent(ctx, sourcePath, targetPath)
return true, nil
}
if writeErr != nil {
return writeErr
return false, writeErr
}

// emitResult prints the line for a copied or skipped file.
func (c *copy) emitResult(ctx context.Context, sourcePath, targetPath string, skipped bool) error {
if skipped {
return c.emitFileSkippedEvent(ctx, sourcePath, targetPath)
}
return c.emitFileCopiedEvent(ctx, sourcePath, targetPath)
}

// TODO: emit these events on stderr
// TODO: add integration tests for these events
func (c *copy) emitFileSkippedEvent(ctx context.Context, sourcePath, targetPath string) error {
fullSourcePath := sourcePath
// fullPaths returns the source and target paths with their scheme, as shown to
// the user.
func (c *copy) fullPaths(sourcePath, targetPath string) (string, string) {
if c.sourceScheme != "" {
fullSourcePath = path.Join(c.sourceScheme+":", sourcePath)
sourcePath = path.Join(c.sourceScheme+":", sourcePath)
}
fullTargetPath := targetPath
if c.targetScheme != "" {
fullTargetPath = path.Join(c.targetScheme+":", targetPath)
targetPath = path.Join(c.targetScheme+":", targetPath)
}
return sourcePath, targetPath
}

// TODO: emit these events on stderr
// TODO: add integration tests for these events
func (c *copy) emitFileSkippedEvent(ctx context.Context, sourcePath, targetPath string) error {
fullSourcePath, fullTargetPath := c.fullPaths(sourcePath, targetPath)

event := newFileSkippedEvent(fullSourcePath, fullTargetPath)
template := "{{.SourcePath}} -> {{.TargetPath}} (skipped; already exists)\n"
Expand All @@ -183,14 +146,7 @@ func (c *copy) emitFileSkippedEvent(ctx context.Context, sourcePath, targetPath
}

func (c *copy) emitFileCopiedEvent(ctx context.Context, sourcePath, targetPath string) error {
fullSourcePath := sourcePath
if c.sourceScheme != "" {
fullSourcePath = path.Join(c.sourceScheme+":", sourcePath)
}
fullTargetPath := targetPath
if c.targetScheme != "" {
fullTargetPath = path.Join(c.targetScheme+":", targetPath)
}
fullSourcePath, fullTargetPath := c.fullPaths(sourcePath, targetPath)

event := newFileCopiedEvent(fullSourcePath, fullTargetPath)
template := "{{.SourcePath}} -> {{.TargetPath}}\n"
Expand Down Expand Up @@ -242,6 +198,7 @@ func newCpCommand() *cobra.Command {

cmd.RunE = func(cmd *cobra.Command, args []string) error {
ctx := cmd.Context()
c.outputJSON = root.OutputType(cmd) == flags.OutputJSON

// Get source filer and source path without scheme
fullSourcePath := args[0]
Expand Down
Loading
Loading