From 7d8cf4b44ae7be85a52cbfc60da991d4aedef9c5 Mon Sep 17 00:00:00 2001 From: Yuanjie Ding Date: Wed, 18 Nov 2026 17:43:00 +0000 Subject: [PATCH] fs cp -r: keep going past failed files fs cp -r used to stop at the first failed file: one transient 500 or 503 in a copy of many small files ended the whole copy. The errgroup is replaced by a copy that runs up to --concurrency files at a time and keeps going: - A retryable failure (408, 429, any 5xx including a non-JSON body, or a network error, after the filer's own retries) defers the file. After the walk, each deferred file gets one more full attempt. - 400, 404 and other errors fail only that file. - 401, 403, cancellation and a failure to print stop the copy. The walk then only counts the files left, reported as not attempted. - A retry without --overwrite that finds the target already there skips it only if the sizes match: a failed DBFS write can leave a truncated file. A different size or a failed Stat fails the file. - The run ends with a summary on stderr and exits 1 if any file was not copied. Co-authored-by: Isaac --- .nextchanges/cli/fs-cp-r-keep-going.md | 1 + .../persistent-500/localdir/f1.txt | 1 + .../persistent-500/localdir/f2.txt | 1 + .../persistent-500/localdir/f3.txt | 1 + .../persistent-500/localdir/f4.txt | 1 + .../persistent-500/out.test.toml | 1 + .../cp-r-keep-going/persistent-500/output.txt | 20 + .../fs/cp-r-keep-going/persistent-500/script | 7 + .../cmd/fs/cp-r-keep-going/script.prepare | 8 + .../single-500/localdir/f1.txt | 1 + .../single-500/localdir/f2.txt | 1 + .../single-500/localdir/f3.txt | 1 + .../single-500/localdir/f4.txt | 1 + .../single-500/localdir/f5.txt | 1 + .../single-500/localdir/f6.txt | 1 + .../cp-r-keep-going/single-500/out.test.toml | 1 + .../fs/cp-r-keep-going/single-500/output.txt | 20 + .../cmd/fs/cp-r-keep-going/single-500/script | 9 + acceptance/cmd/fs/cp-r-keep-going/test.toml | 1 + acceptance/cmd/fs/cp-r-requests/output.txt | 1 + acceptance/cmd/fs/cp/dir-to-dir/output.txt | 1 + cmd/fs/cp.go | 129 ++---- cmd/fs/cp_recursive.go | 414 +++++++++++++++++ cmd/fs/cp_recursive_test.go | 433 ++++++++++++++++++ cmd/fs/cp_test.go | 107 ----- 25 files changed, 970 insertions(+), 193 deletions(-) create mode 100644 .nextchanges/cli/fs-cp-r-keep-going.md create mode 100644 acceptance/cmd/fs/cp-r-keep-going/persistent-500/localdir/f1.txt create mode 100644 acceptance/cmd/fs/cp-r-keep-going/persistent-500/localdir/f2.txt create mode 100644 acceptance/cmd/fs/cp-r-keep-going/persistent-500/localdir/f3.txt create mode 100644 acceptance/cmd/fs/cp-r-keep-going/persistent-500/localdir/f4.txt create mode 100644 acceptance/cmd/fs/cp-r-keep-going/persistent-500/out.test.toml create mode 100644 acceptance/cmd/fs/cp-r-keep-going/persistent-500/output.txt create mode 100644 acceptance/cmd/fs/cp-r-keep-going/persistent-500/script create mode 100644 acceptance/cmd/fs/cp-r-keep-going/script.prepare create mode 100644 acceptance/cmd/fs/cp-r-keep-going/single-500/localdir/f1.txt create mode 100644 acceptance/cmd/fs/cp-r-keep-going/single-500/localdir/f2.txt create mode 100644 acceptance/cmd/fs/cp-r-keep-going/single-500/localdir/f3.txt create mode 100644 acceptance/cmd/fs/cp-r-keep-going/single-500/localdir/f4.txt create mode 100644 acceptance/cmd/fs/cp-r-keep-going/single-500/localdir/f5.txt create mode 100644 acceptance/cmd/fs/cp-r-keep-going/single-500/localdir/f6.txt create mode 100644 acceptance/cmd/fs/cp-r-keep-going/single-500/out.test.toml create mode 100644 acceptance/cmd/fs/cp-r-keep-going/single-500/output.txt create mode 100644 acceptance/cmd/fs/cp-r-keep-going/single-500/script create mode 100644 acceptance/cmd/fs/cp-r-keep-going/test.toml create mode 100644 cmd/fs/cp_recursive.go create mode 100644 cmd/fs/cp_recursive_test.go diff --git a/.nextchanges/cli/fs-cp-r-keep-going.md b/.nextchanges/cli/fs-cp-r-keep-going.md new file mode 100644 index 00000000000..8c648b6ca5c --- /dev/null +++ b/.nextchanges/cli/fs-cp-r-keep-going.md @@ -0,0 +1 @@ +* `databricks fs cp -r` keeps going when a file fails. A file that fails with a temporary error (HTTP 408, 429, 5xx, or a network error) is retried once after the rest of the copy, and other failed files are reported while the copy continues. The copy stops early on an authentication or permission error, or on a directory it cannot list or create. A summary goes to stderr, and the command exits with code 1 if any file was not copied. ([#7712](https://github.com/databricks/cli/pull/7712)) diff --git a/acceptance/cmd/fs/cp-r-keep-going/persistent-500/localdir/f1.txt b/acceptance/cmd/fs/cp-r-keep-going/persistent-500/localdir/f1.txt new file mode 100644 index 00000000000..366f17ff507 --- /dev/null +++ b/acceptance/cmd/fs/cp-r-keep-going/persistent-500/localdir/f1.txt @@ -0,0 +1 @@ +file 1 diff --git a/acceptance/cmd/fs/cp-r-keep-going/persistent-500/localdir/f2.txt b/acceptance/cmd/fs/cp-r-keep-going/persistent-500/localdir/f2.txt new file mode 100644 index 00000000000..fac580e9808 --- /dev/null +++ b/acceptance/cmd/fs/cp-r-keep-going/persistent-500/localdir/f2.txt @@ -0,0 +1 @@ +file 2 diff --git a/acceptance/cmd/fs/cp-r-keep-going/persistent-500/localdir/f3.txt b/acceptance/cmd/fs/cp-r-keep-going/persistent-500/localdir/f3.txt new file mode 100644 index 00000000000..3ca7e65ae83 --- /dev/null +++ b/acceptance/cmd/fs/cp-r-keep-going/persistent-500/localdir/f3.txt @@ -0,0 +1 @@ +INJECT_ERROR diff --git a/acceptance/cmd/fs/cp-r-keep-going/persistent-500/localdir/f4.txt b/acceptance/cmd/fs/cp-r-keep-going/persistent-500/localdir/f4.txt new file mode 100644 index 00000000000..4b24a15d33f --- /dev/null +++ b/acceptance/cmd/fs/cp-r-keep-going/persistent-500/localdir/f4.txt @@ -0,0 +1 @@ +file 4 diff --git a/acceptance/cmd/fs/cp-r-keep-going/persistent-500/out.test.toml b/acceptance/cmd/fs/cp-r-keep-going/persistent-500/out.test.toml new file mode 100644 index 00000000000..18b1a88417e --- /dev/null +++ b/acceptance/cmd/fs/cp-r-keep-going/persistent-500/out.test.toml @@ -0,0 +1 @@ +Cloud = false diff --git a/acceptance/cmd/fs/cp-r-keep-going/persistent-500/output.txt b/acceptance/cmd/fs/cp-r-keep-going/persistent-500/output.txt new file mode 100644 index 00000000000..2d24f08a6dc --- /dev/null +++ b/acceptance/cmd/fs/cp-r-keep-going/persistent-500/output.txt @@ -0,0 +1,20 @@ + +>>> cp_r localdir dbfs:/Volumes/main/default/keep-going/dst --concurrency 2 +Copy incomplete: 3 copied, 0 skipped (already existed), 1 failed, 0 not attempted. +Failed: + localdir/f3.txt -> dbfs:/Volumes/main/default/keep-going/dst/f3.txt: HTTP 500 Internal Server Error +Error: 1 file was not copied +localdir/f1.txt -> dbfs:/Volumes/main/default/keep-going/dst/f1.txt +localdir/f2.txt -> dbfs:/Volumes/main/default/keep-going/dst/f2.txt +localdir/f4.txt -> dbfs:/Volumes/main/default/keep-going/dst/f4.txt + +Exit code: 1 + +=== Four uploads plus one retry of f3.txt + +>>> print_requests.py --summary --sort //api/2.0/fs/files/ +PUT /api/2.0/fs/files/Volumes/main/default/keep-going/dst/f1.txt overwrite=false +PUT /api/2.0/fs/files/Volumes/main/default/keep-going/dst/f2.txt overwrite=false +PUT /api/2.0/fs/files/Volumes/main/default/keep-going/dst/f3.txt overwrite=false +PUT /api/2.0/fs/files/Volumes/main/default/keep-going/dst/f3.txt overwrite=false +PUT /api/2.0/fs/files/Volumes/main/default/keep-going/dst/f4.txt overwrite=false diff --git a/acceptance/cmd/fs/cp-r-keep-going/persistent-500/script b/acceptance/cmd/fs/cp-r-keep-going/persistent-500/script new file mode 100644 index 00000000000..70c12ea5245 --- /dev/null +++ b/acceptance/cmd/fs/cp-r-keep-going/persistent-500/script @@ -0,0 +1,7 @@ +# f3.txt contains INJECT_ERROR, so the testserver answers every upload of it +# with a 500 whose body is not JSON. The file gets its retry after the walk, +# then fails alone; the other files are copied and the command exits 1. +errcode trace cp_r localdir ${TARGET} --concurrency 2 + +title "Four uploads plus one retry of f3.txt" +trace print_requests.py --summary --sort //api/2.0/fs/files/ diff --git a/acceptance/cmd/fs/cp-r-keep-going/script.prepare b/acceptance/cmd/fs/cp-r-keep-going/script.prepare new file mode 100644 index 00000000000..95e9f7c2f46 --- /dev/null +++ b/acceptance/cmd/fs/cp-r-keep-going/script.prepare @@ -0,0 +1,8 @@ +TARGET="dbfs:/Volumes/main/default/keep-going/dst" + +# cp_r runs fs cp -r with the given arguments and prints the per-file lines +# sorted, since files are copied in parallel. It returns the exit code of fs cp. +cp_r() { + $CLI fs cp -r "$@" | sort + return "${PIPESTATUS[0]}" +} diff --git a/acceptance/cmd/fs/cp-r-keep-going/single-500/localdir/f1.txt b/acceptance/cmd/fs/cp-r-keep-going/single-500/localdir/f1.txt new file mode 100644 index 00000000000..366f17ff507 --- /dev/null +++ b/acceptance/cmd/fs/cp-r-keep-going/single-500/localdir/f1.txt @@ -0,0 +1 @@ +file 1 diff --git a/acceptance/cmd/fs/cp-r-keep-going/single-500/localdir/f2.txt b/acceptance/cmd/fs/cp-r-keep-going/single-500/localdir/f2.txt new file mode 100644 index 00000000000..fac580e9808 --- /dev/null +++ b/acceptance/cmd/fs/cp-r-keep-going/single-500/localdir/f2.txt @@ -0,0 +1 @@ +file 2 diff --git a/acceptance/cmd/fs/cp-r-keep-going/single-500/localdir/f3.txt b/acceptance/cmd/fs/cp-r-keep-going/single-500/localdir/f3.txt new file mode 100644 index 00000000000..dc43ed8669d --- /dev/null +++ b/acceptance/cmd/fs/cp-r-keep-going/single-500/localdir/f3.txt @@ -0,0 +1 @@ +file 3 diff --git a/acceptance/cmd/fs/cp-r-keep-going/single-500/localdir/f4.txt b/acceptance/cmd/fs/cp-r-keep-going/single-500/localdir/f4.txt new file mode 100644 index 00000000000..4b24a15d33f --- /dev/null +++ b/acceptance/cmd/fs/cp-r-keep-going/single-500/localdir/f4.txt @@ -0,0 +1 @@ +file 4 diff --git a/acceptance/cmd/fs/cp-r-keep-going/single-500/localdir/f5.txt b/acceptance/cmd/fs/cp-r-keep-going/single-500/localdir/f5.txt new file mode 100644 index 00000000000..d2d8a5a10fa --- /dev/null +++ b/acceptance/cmd/fs/cp-r-keep-going/single-500/localdir/f5.txt @@ -0,0 +1 @@ +file 5 diff --git a/acceptance/cmd/fs/cp-r-keep-going/single-500/localdir/f6.txt b/acceptance/cmd/fs/cp-r-keep-going/single-500/localdir/f6.txt new file mode 100644 index 00000000000..9454c471bbe --- /dev/null +++ b/acceptance/cmd/fs/cp-r-keep-going/single-500/localdir/f6.txt @@ -0,0 +1 @@ +file 6 diff --git a/acceptance/cmd/fs/cp-r-keep-going/single-500/out.test.toml b/acceptance/cmd/fs/cp-r-keep-going/single-500/out.test.toml new file mode 100644 index 00000000000..18b1a88417e --- /dev/null +++ b/acceptance/cmd/fs/cp-r-keep-going/single-500/out.test.toml @@ -0,0 +1 @@ +Cloud = false diff --git a/acceptance/cmd/fs/cp-r-keep-going/single-500/output.txt b/acceptance/cmd/fs/cp-r-keep-going/single-500/output.txt new file mode 100644 index 00000000000..e8af6cac556 --- /dev/null +++ b/acceptance/cmd/fs/cp-r-keep-going/single-500/output.txt @@ -0,0 +1,20 @@ + +>>> cp_r localdir dbfs:/Volumes/main/default/keep-going/dst --concurrency 2 +Copied 6 files. +localdir/f1.txt -> dbfs:/Volumes/main/default/keep-going/dst/f1.txt +localdir/f2.txt -> dbfs:/Volumes/main/default/keep-going/dst/f2.txt +localdir/f3.txt -> dbfs:/Volumes/main/default/keep-going/dst/f3.txt +localdir/f4.txt -> dbfs:/Volumes/main/default/keep-going/dst/f4.txt +localdir/f5.txt -> dbfs:/Volumes/main/default/keep-going/dst/f5.txt +localdir/f6.txt -> dbfs:/Volumes/main/default/keep-going/dst/f6.txt + +=== Six uploads plus one retry + +>>> print_requests.py --summary --sort //api/2.0/fs/files/ +PUT /api/2.0/fs/files/Volumes/main/default/keep-going/dst/f1.txt overwrite=false +PUT /api/2.0/fs/files/Volumes/main/default/keep-going/dst/f2.txt overwrite=false +PUT /api/2.0/fs/files/Volumes/main/default/keep-going/dst/f3.txt overwrite=false +PUT /api/2.0/fs/files/Volumes/main/default/keep-going/dst/f3.txt overwrite=false +PUT /api/2.0/fs/files/Volumes/main/default/keep-going/dst/f4.txt overwrite=false +PUT /api/2.0/fs/files/Volumes/main/default/keep-going/dst/f5.txt overwrite=false +PUT /api/2.0/fs/files/Volumes/main/default/keep-going/dst/f6.txt overwrite=false diff --git a/acceptance/cmd/fs/cp-r-keep-going/single-500/script b/acceptance/cmd/fs/cp-r-keep-going/single-500/script new file mode 100644 index 00000000000..6d447fc8063 --- /dev/null +++ b/acceptance/cmd/fs/cp-r-keep-going/single-500/script @@ -0,0 +1,9 @@ +# A 500 is not retried by the filer. The copy defers the file and retries it +# once after the walk, so the copy still succeeds. The fault matches f3.txt by +# its content, since files are uploaded in parallel. +fault.py --body-contains "file 3" "PUT /api/2.0/fs/files/**" 500 0 1 + +trace cp_r localdir ${TARGET} --concurrency 2 + +title "Six uploads plus one retry" +trace print_requests.py --summary --sort //api/2.0/fs/files/ diff --git a/acceptance/cmd/fs/cp-r-keep-going/test.toml b/acceptance/cmd/fs/cp-r-keep-going/test.toml new file mode 100644 index 00000000000..159efe02696 --- /dev/null +++ b/acceptance/cmd/fs/cp-r-keep-going/test.toml @@ -0,0 +1 @@ +RecordRequests = true diff --git a/acceptance/cmd/fs/cp-r-requests/output.txt b/acceptance/cmd/fs/cp-r-requests/output.txt index 1982e113bd2..a4e827fc540 100644 --- a/acceptance/cmd/fs/cp-r-requests/output.txt +++ b/acceptance/cmd/fs/cp-r-requests/output.txt @@ -1,5 +1,6 @@ >>> [CLI] fs cp -r localdir dbfs:/Volumes/main/default/fs-cp-r-requests/dst +Copied 3 files. localdir/a.txt -> dbfs:/Volumes/main/default/fs-cp-r-requests/dst/a.txt localdir/b.txt -> dbfs:/Volumes/main/default/fs-cp-r-requests/dst/b.txt localdir/sub/c.txt -> dbfs:/Volumes/main/default/fs-cp-r-requests/dst/sub/c.txt diff --git a/acceptance/cmd/fs/cp/dir-to-dir/output.txt b/acceptance/cmd/fs/cp/dir-to-dir/output.txt index 7987c56b138..862aa812cf4 100644 --- a/acceptance/cmd/fs/cp/dir-to-dir/output.txt +++ b/acceptance/cmd/fs/cp/dir-to-dir/output.txt @@ -1,5 +1,6 @@ >>> [CLI] fs cp -r localdir dbfs:/Volumes/main/default/fs-cp-test-[UNIQUE_NAME]/uploaded-dir +Copied 2 files. localdir/file1.txt -> dbfs:/Volumes/main/default/fs-cp-test-[UNIQUE_NAME]/uploaded-dir/file1.txt localdir/file2.txt -> dbfs:/Volumes/main/default/fs-cp-test-[UNIQUE_NAME]/uploaded-dir/file2.txt diff --git a/cmd/fs/cp.go b/cmd/fs/cp.go index 957aaaf2f55..d7bc6db7e71 100644 --- a/cmd/fs/cp.go +++ b/cmd/fs/cp.go @@ -15,8 +15,8 @@ import ( "github.com/databricks/cli/cmd/root" "github.com/databricks/cli/libs/cmdio" "github.com/databricks/cli/libs/filer" + "github.com/databricks/cli/libs/flags" "github.com/spf13/cobra" - "golang.org/x/sync/errgroup" ) // defaultConcurrency is the number of files copied in parallel. Each in-flight @@ -44,77 +44,21 @@ type copy struct { // would each fight for the spinner line). showProgress bool + // outputJSON is set when the command renders JSON. A recursive copy then + // leaves out its one-line summary of a complete copy. + outputJSON bool + mu sync.Mutex // protect output from concurrent writes } // cpDirToDir recursively copies the content of a directory to another -// directory. -// -// There is no guarantee on the order in which the files are copied. -// -// The method does not take care of retrying on error; this is considered to -// be the responsibility of the Filer implementation. If a file copy fails, -// the error is returned and the other copies are cancelled. +// directory. There is no guarantee on the order in which the files are copied. +// It keeps going past failed files; see [recursiveCopy]. func (c *copy) cpDirToDir(ctx context.Context, sourceDir, targetDir string) error { if !c.recursive { return fmt.Errorf("source path %s is a directory. Please specify the --recursive flag", sourceDir) } - - // Create a cancellable context purely for the purpose of having a way to - // cancel the goroutines in case of error walking the directory. - ctx, cancel := context.WithCancel(ctx) - defer cancel() - - // Pool of workers to process copy operations in parallel. The created - // context is the real context for this operation. It is shared by the - // walking function and the goroutines and can be cancelled manually - // by calling the cancel() function of its parent context. - g, ctx := errgroup.WithContext(ctx) - g.SetLimit(c.concurrency) - - // Walk the source directory, queueing file copy operations for processing. - sourceFs := filer.NewFS(ctx, c.sourceFiler) - err := fs.WalkDir(sourceFs, sourceDir, func(sourcePath string, d fs.DirEntry, err error) error { - if err != nil { - return err - } - - // Compute path relative to the source directory. - relPath, err := filepath.Rel(sourceDir, sourcePath) - if err != nil { - return err - } - relPath = filepath.ToSlash(relPath) - - // Compute target path for the file. - targetPath := path.Join(targetDir, relPath) - - // Create the directory synchronously. This must happen before files - // are copied into it, and WalkDir guarantees directories are visited - // before their contents. - if d.IsDir() { - return c.targetFiler.Mkdir(ctx, targetPath) - } - - g.Go(func() error { - // Goroutines are queued and may start after the context is already - // cancelled (e.g. a prior copy failed). This check aims to avoid - // starting work that will inevitably fail. - if ctx.Err() != nil { - return ctx.Err() - } - // The walk already created the target directory, so skip the filer's - // per-file parent-directory check. - return c.cpFileToFile(ctx, sourcePath, targetPath, filer.CreateParentDirectories) - }) - return nil - }) - if err != nil { - cancel() // cancel the goroutines - _ = g.Wait() // wait for the goroutines to finish - return err // return the "real" error that led to cancellation - } - return g.Wait() + return (&recursiveCopy{c: c}).run(ctx, sourceDir, targetDir) } func (c *copy) cpFileToDir(ctx context.Context, sourcePath, targetDir string) error { @@ -124,13 +68,22 @@ func (c *copy) cpFileToDir(ctx context.Context, sourcePath, targetDir string) er return c.cpFileToFile(ctx, sourcePath, targetPath) } -// cpFileToFile copies one file. extraModes are passed to Write, in addition to -// OverwriteIfExists when --overwrite is set. -func (c *copy) cpFileToFile(ctx context.Context, sourcePath, targetPath string, extraModes ...filer.WriteMode) error { +func (c *copy) cpFileToFile(ctx context.Context, sourcePath, targetPath string) error { + skipped, err := c.writeFile(ctx, sourcePath, targetPath) + if err != nil { + return err + } + return c.emitResult(ctx, sourcePath, targetPath, skipped) +} + +// writeFile copies one file without printing its line, and reports whether it +// was skipped because the target exists. extraModes are passed to Write, in +// addition to OverwriteIfExists when --overwrite is set. +func (c *copy) writeFile(ctx context.Context, sourcePath, targetPath string, extraModes ...filer.WriteMode) (bool, error) { // Get reader for file at source path r, err := c.sourceFiler.Read(ctx, sourcePath) if err != nil { - return err + return false, err } defer r.Close() @@ -154,25 +107,35 @@ func (c *copy) cpFileToFile(ctx context.Context, sourcePath, targetPath string, // skip if file already exists if !c.overwrite && writeErr != nil && errors.Is(writeErr, fs.ErrExist) { - return c.emitFileSkippedEvent(ctx, sourcePath, targetPath) + return true, nil } - if writeErr != nil { - return writeErr + return false, writeErr +} + +// emitResult prints the line for a copied or skipped file. +func (c *copy) emitResult(ctx context.Context, sourcePath, targetPath string, skipped bool) error { + if skipped { + return c.emitFileSkippedEvent(ctx, sourcePath, targetPath) } return c.emitFileCopiedEvent(ctx, sourcePath, targetPath) } -// TODO: emit these events on stderr -// TODO: add integration tests for these events -func (c *copy) emitFileSkippedEvent(ctx context.Context, sourcePath, targetPath string) error { - fullSourcePath := sourcePath +// fullPaths returns the source and target paths with their scheme, as shown to +// the user. +func (c *copy) fullPaths(sourcePath, targetPath string) (string, string) { if c.sourceScheme != "" { - fullSourcePath = path.Join(c.sourceScheme+":", sourcePath) + sourcePath = path.Join(c.sourceScheme+":", sourcePath) } - fullTargetPath := targetPath if c.targetScheme != "" { - fullTargetPath = path.Join(c.targetScheme+":", targetPath) + targetPath = path.Join(c.targetScheme+":", targetPath) } + return sourcePath, targetPath +} + +// TODO: emit these events on stderr +// TODO: add integration tests for these events +func (c *copy) emitFileSkippedEvent(ctx context.Context, sourcePath, targetPath string) error { + fullSourcePath, fullTargetPath := c.fullPaths(sourcePath, targetPath) event := newFileSkippedEvent(fullSourcePath, fullTargetPath) template := "{{.SourcePath}} -> {{.TargetPath}} (skipped; already exists)\n" @@ -183,14 +146,7 @@ func (c *copy) emitFileSkippedEvent(ctx context.Context, sourcePath, targetPath } func (c *copy) emitFileCopiedEvent(ctx context.Context, sourcePath, targetPath string) error { - fullSourcePath := sourcePath - if c.sourceScheme != "" { - fullSourcePath = path.Join(c.sourceScheme+":", sourcePath) - } - fullTargetPath := targetPath - if c.targetScheme != "" { - fullTargetPath = path.Join(c.targetScheme+":", targetPath) - } + fullSourcePath, fullTargetPath := c.fullPaths(sourcePath, targetPath) event := newFileCopiedEvent(fullSourcePath, fullTargetPath) template := "{{.SourcePath}} -> {{.TargetPath}}\n" @@ -242,6 +198,7 @@ func newCpCommand() *cobra.Command { cmd.RunE = func(cmd *cobra.Command, args []string) error { ctx := cmd.Context() + c.outputJSON = root.OutputType(cmd) == flags.OutputJSON // Get source filer and source path without scheme fullSourcePath := args[0] diff --git a/cmd/fs/cp_recursive.go b/cmd/fs/cp_recursive.go new file mode 100644 index 00000000000..c44e200cfde --- /dev/null +++ b/cmd/fs/cp_recursive.go @@ -0,0 +1,414 @@ +package fs + +import ( + "context" + "errors" + "fmt" + "io" + "io/fs" + "net" + "net/http" + "net/url" + "path" + "path/filepath" + "strconv" + "strings" + "sync" + + "github.com/databricks/cli/libs/cmdio" + "github.com/databricks/cli/libs/filer" + "github.com/databricks/cli/libs/log" + sdkapierr "github.com/databricks/databricks-sdk-go/apierr" + "github.com/databricks/sdk-go/core/apierr" + "github.com/databricks/sdk-go/core/apiretry" +) + +// maxListedFailures is how many failed files the end-of-run summary lists. +const maxListedFailures = 10 + +// errorClass says what a recursive copy does after an error. +type errorClass int + +const ( + // classFile fails only the file it happened on. + classFile errorClass = iota + // classRetryable defers the file, to be tried once more after the walk. + classRetryable + // classFatal stops the copy. + classFatal +) + +// classifyError decides what a recursive copy does after err. It runs after the +// filer's own retries, so a retryable error here is one they did not absorb. +// +// - 401 and 403 stop the copy: every other file would fail the same way. +// - 408, 429 and 5xx (including a 5xx with an empty or non-JSON body, which the +// SDKs still turn into an API error carrying the status) are retryable. +// - Other API errors (400, 404, ...) fail only the file. +// - Transport errors (connection reset or refused, timeouts, an unexpected +// EOF, any other *url.Error from the HTTP client) are retryable. +// - Cancellation of ctx stops the copy. +// - Anything else, e.g. an error reading a local file, fails only the file. +func classifyError(ctx context.Context, err error) errorClass { + if ctx.Err() != nil { + return classFatal + } + if status, _, ok := httpError(err); ok { + switch { + case status == http.StatusUnauthorized || status == http.StatusForbidden: + return classFatal + case status == http.StatusRequestTimeout || status == http.StatusTooManyRequests || status >= 500: + return classRetryable + default: + return classFile + } + } + // ctx is alive, so a context error comes from a timeout on one request. + if errors.Is(err, context.DeadlineExceeded) || errors.Is(err, context.Canceled) || + apiretry.IsTransientNetworkError(err) || errors.Is(err, io.ErrUnexpectedEOF) { + return classRetryable + } + if _, ok := errors.AsType[*url.Error](err); ok { + return classRetryable + } + if _, ok := errors.AsType[net.Error](err); ok { + return classRetryable + } + return classFile +} + +// httpError returns the HTTP status and server message of an API error from +// either SDK: the Files API (Volumes) uses sdk-go, DBFS uses databricks-sdk-go. +func httpError(err error) (int, string, bool) { + if aerr, ok := errors.AsType[*apierr.APIError](err); ok { + return aerr.HTTPStatusCode(), aerr.Message(), true + } + if aerr, ok := errors.AsType[*sdkapierr.APIError](err); ok { + return aerr.StatusCode, aerr.Message, true + } + return 0, "", false +} + +// describeError adds the HTTP status to an API error. A 5xx from a proxy often +// has no JSON body, and then the error has no message of its own. +func describeError(err error) string { + status, message, ok := httpError(err) + if !ok { + return err.Error() + } + statusText := fmt.Sprintf("HTTP %d %s", status, http.StatusText(status)) + if message == "" { + return statusText + } + return fmt.Sprintf("%s (%s)", err, statusText) +} + +// copyJob is one file to copy. +type copyJob struct { + sourcePath string + targetPath string + + // final is set on the deferred pass: a retryable failure then fails the + // file instead of deferring it again. + final bool + + // err is the error of the job's last attempt: the one that deferred it, + // or the one it failed with. + err error +} + +// recursiveCopy runs one `fs cp -r`. It copies up to --concurrency files at a +// time and keeps going past failures: +// +// - A retryable failure defers the file. After the walk, every deferred file +// gets one more full attempt, with its source reopened. +// - A failure that is not retryable fails only that file, except that 401, +// 403, cancellation and a failure to print stop the copy. +// - Once the copy stops, in-flight files are cancelled and reported as +// failed. The walk goes on only to count the files left, which are +// reported as not attempted. +type recursiveCopy struct { + c *copy + + // cancel stops in-flight copies after a fatal error. + cancel context.CancelFunc + // slots holds one token per running copy, so at most --concurrency files + // are copied at once. + slots chan struct{} + inflight sync.WaitGroup + + mu sync.Mutex + copied int + skipped int + notAttempted int + failures []copyJob + deferred []copyJob + + // fatalErr is the error that stopped the copy, if any. + fatalErr error + // walkCutShort is set when the walk ended on an error, so notAttempted + // leaves out files it never reached. + walkCutShort bool +} + +func (r *recursiveCopy) run(ctx context.Context, sourceDir, targetDir string) error { + ctx, cancel := context.WithCancel(ctx) + defer cancel() + r.cancel = cancel + r.slots = make(chan struct{}, r.c.concurrency) + + sourceFs := filer.NewFS(ctx, r.c.sourceFiler) + walkErr := fs.WalkDir(sourceFs, sourceDir, func(sourcePath string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + relPath, err := filepath.Rel(sourceDir, sourcePath) + if err != nil { + return err + } + targetPath := path.Join(targetDir, filepath.ToSlash(relPath)) + if !d.IsDir() { + r.start(ctx, copyJob{sourcePath: sourcePath, targetPath: targetPath}) + return nil + } + // Once the copy is stopping, the walk only counts the files left. + if r.stopping(ctx) { + return nil + } + // Create the directory before the files in it. WalkDir visits a + // directory before its contents. + return r.c.targetFiler.Mkdir(ctx, targetPath) + }) + if walkErr != nil { + r.walkFailed(ctx, walkErr) + } + r.inflight.Wait() + + // Give every deferred file one more attempt. + for job, ok := r.popDeferred(); ok; job, ok = r.popDeferred() { + job.final = true + r.start(ctx, job) + } + r.inflight.Wait() + return r.finish(ctx) +} + +// start copies job in the background once a slot is free, or records it as not +// started if the copy is stopping. +func (r *recursiveCopy) start(ctx context.Context, job copyJob) { + // Check again after waiting for a slot: the copy may have stopped. + if r.stopping(ctx) { + r.notStarted(job) + return + } + r.slots <- struct{}{} + if r.stopping(ctx) { + <-r.slots + r.notStarted(job) + return + } + r.inflight.Go(func() { + defer func() { <-r.slots }() + r.copyOne(ctx, job) + }) +} + +// stopping reports whether the copy should start no new files. +func (r *recursiveCopy) stopping(ctx context.Context) bool { + // A fatal error cancels ctx. + return ctx.Err() != nil +} + +// walkFailed records an error that ended the walk. It stops the copy, unless +// the copy was already stopping and the walk was only counting files. +func (r *recursiveCopy) walkFailed(ctx context.Context, err error) { + r.mu.Lock() + defer r.mu.Unlock() + r.walkCutShort = true + if ctx.Err() == nil && r.fatalErr == nil { + r.fatalErr = err + r.cancel() + } +} + +func (r *recursiveCopy) popDeferred() (copyJob, bool) { + r.mu.Lock() + defer r.mu.Unlock() + if len(r.deferred) == 0 { + return copyJob{}, false + } + job := r.deferred[0] + r.deferred = r.deferred[1:] + return job, true +} + +// notStarted records a job that was not started because the copy is stopping. +// A deferred job already failed once, so it counts as failed with that error. +func (r *recursiveCopy) notStarted(job copyJob) { + r.mu.Lock() + defer r.mu.Unlock() + if job.err != nil { + r.failures = append(r.failures, job) + return + } + r.notAttempted++ +} + +// errCancelledMidCopy fails a file whose copy was cancelled because the copy +// stopped. On DBFS it may have left a partial target. +var errCancelledMidCopy = errors.New("cancelled: the copy stopped before this file finished") + +// errPartialTarget fails a retried file whose target exists with another size. +var errPartialTarget = errors.New("target exists with a different size; an earlier attempt may have left a partial file. Rerun with --overwrite to replace it") + +// copyOne copies one file and records the outcome. +func (r *recursiveCopy) copyOne(ctx context.Context, job copyJob) { + // The walk already created the target directory, so skip the filer's + // per-file parent-directory check. + skipped, err := r.c.writeFile(ctx, job.sourcePath, job.targetPath, filer.CreateParentDirectories) + + class := classFile + switch { + case err != nil: + class = classifyError(ctx, err) + case skipped && job.err != nil: + // Without --overwrite, a retry finds whatever an earlier attempt of + // ours left behind. On Volumes a single-shot upload is one atomic PUT, + // so that is the complete file. A DBFS write streams blocks into an + // open handle, so a failed attempt can leave a truncated prefix, and + // a size check catches it. A partial of the same size is not + // detected, but a truncated prefix is shorter unless the failure + // came after the last block. Any error here fails the file rather + // than guessing that it was copied, though a 401, 403 or cancellation + // still stops the copy. + if err = r.checkExistingTarget(ctx, job); err != nil { + class = classifyError(ctx, err) + if class == classRetryable { + class = classFile + } + } + } + if err == nil { + // Failing to print means the output is gone (e.g. a closed pipe); + // every other file would fail the same way. + if err = r.c.emitResult(ctx, job.sourcePath, job.targetPath, skipped); err != nil { + class = classFatal + } + } + if class == classFatal && ctx.Err() != nil { + // Cancelled because the copy is stopping; not this file's fault. A + // deferred file keeps its earlier error. + if job.err == nil { + job.err = errCancelledMidCopy + } + r.notStarted(job) + return + } + + r.mu.Lock() + defer r.mu.Unlock() + switch { + case err == nil && skipped: + r.skipped++ + case err == nil: + r.copied++ + case class == classRetryable && !job.final: + log.Debugf(ctx, "fs cp: deferring %s: %s", job.sourcePath, describeError(err)) + job.err = err + r.deferred = append(r.deferred, job) + default: + if class == classRetryable { + log.Debugf(ctx, "fs cp: %s failed on its retry: %s", job.sourcePath, describeError(err)) + } + job.err = err + r.failures = append(r.failures, job) + if class == classFatal && r.fatalErr == nil { + r.fatalErr = err + r.cancel() + } + } +} + +// checkExistingTarget returns errPartialTarget if the target's size differs from +// the source's, or the error from either Stat. +func (r *recursiveCopy) checkExistingTarget(ctx context.Context, job copyJob) error { + source, err := r.c.sourceFiler.Stat(ctx, job.sourcePath) + if err != nil { + return err + } + target, err := r.c.targetFiler.Stat(ctx, job.targetPath) + if err != nil { + return err + } + if source.Size() != target.Size() { + return errPartialTarget + } + return nil +} + +// finish prints the end-of-run summary to stderr and returns the command's +// error. A complete copy gets one line, which JSON output omits. An incomplete +// copy gets the counts, the reason it stopped early, if it did, and the first +// failures. +func (r *recursiveCopy) finish(ctx context.Context) error { + r.mu.Lock() + defer r.mu.Unlock() + + // The copy cancels its own context only after a fatal error, so any other + // cancellation came from the caller. + cancelled := ctx.Err() != nil && r.fatalErr == nil + notCopied := len(r.failures) + r.notAttempted + if notCopied == 0 && r.fatalErr == nil && !cancelled { + if !r.c.outputJSON { + line := "Copied " + plural(r.copied, "file") + if r.skipped > 0 { + line += fmt.Sprintf(", skipped %d that already existed", r.skipped) + } + cmdio.LogString(ctx, line+".") + } + return nil + } + + notAttempted := strconv.Itoa(r.notAttempted) + if r.walkCutShort { + notAttempted = "at least " + notAttempted + } + var b strings.Builder + fmt.Fprintf(&b, "Copy incomplete: %d copied, %d skipped (already existed), %d failed, %s not attempted.", + r.copied, r.skipped, len(r.failures), notAttempted) + switch { + case cancelled: + b.WriteString("\nStopped: the copy was cancelled.") + case r.fatalErr != nil: + b.WriteString("\nStopped early because of the error shown below.") + } + if len(r.failures) > 0 { + b.WriteString("\nFailed:") + for _, job := range r.failures[:min(len(r.failures), maxListedFailures)] { + source, target := r.c.fullPaths(job.sourcePath, job.targetPath) + fmt.Fprintf(&b, "\n %s -> %s: %s", source, target, describeError(job.err)) + } + if n := len(r.failures) - maxListedFailures; n > 0 { + fmt.Fprintf(&b, "\n ... and %d more", n) + } + } + cmdio.LogString(ctx, b.String()) + + switch { + case r.fatalErr != nil: + return r.fatalErr + case cancelled: + return fmt.Errorf("copy cancelled: %w", ctx.Err()) + } + if notCopied == 1 { + return errors.New("1 file was not copied") + } + return fmt.Errorf("%d files were not copied", notCopied) +} + +func plural(n int, unit string) string { + if n == 1 { + return "1 " + unit + } + return fmt.Sprintf("%d %ss", n, unit) +} diff --git a/cmd/fs/cp_recursive_test.go b/cmd/fs/cp_recursive_test.go new file mode 100644 index 00000000000..214ca5e7a05 --- /dev/null +++ b/cmd/fs/cp_recursive_test.go @@ -0,0 +1,433 @@ +package fs + +import ( + "bytes" + "context" + "errors" + "fmt" + "io" + "io/fs" + "log/slog" + "net/http" + "net/url" + "path" + "strings" + "sync" + "syscall" + "testing" + "time" + + "github.com/databricks/cli/libs/cmdio" + "github.com/databricks/cli/libs/fakefs" + "github.com/databricks/cli/libs/filer" + "github.com/databricks/cli/libs/flags" + "github.com/databricks/cli/libs/log" + sdkapierr "github.com/databricks/databricks-sdk-go/apierr" + "github.com/databricks/sdk-go/core/apierr" + "github.com/stretchr/testify/assert" +) + +func apiError(status int, header http.Header, body string) error { + return apierr.FromHTTPError(status, header, []byte(body)) +} + +func TestClassifyError(t *testing.T) { + cancelled, cancel := context.WithCancel(t.Context()) + cancel() + + jsonBody := `{"error_code": "INJECTED", "message": "boom"}` + tests := []struct { + name string + ctx context.Context + err error + want errorClass + }{ + {"500 with empty body", t.Context(), apiError(500, nil, ""), classRetryable}, + {"500 with non-JSON body", t.Context(), apiError(500, nil, "oops"), classRetryable}, + {"503 wrapped", t.Context(), fmt.Errorf("write: %w", apiError(503, nil, jsonBody)), classRetryable}, + {"429", t.Context(), apiError(429, nil, jsonBody), classRetryable}, + {"408", t.Context(), apiError(408, nil, jsonBody), classRetryable}, + {"DBFS 500", t.Context(), &sdkapierr.APIError{StatusCode: 500}, classRetryable}, + {"400", t.Context(), apiError(400, nil, jsonBody), classFile}, + {"404", t.Context(), apiError(404, nil, jsonBody), classFile}, + {"DBFS 404", t.Context(), &sdkapierr.APIError{StatusCode: 404}, classFile}, + {"401", t.Context(), apiError(401, nil, jsonBody), classFatal}, + {"403", t.Context(), apiError(403, nil, jsonBody), classFatal}, + {"DBFS 403", t.Context(), &sdkapierr.APIError{StatusCode: 403}, classFatal}, + {"connection reset", t.Context(), fmt.Errorf("put: %w", syscall.ECONNRESET), classRetryable}, + {"unexpected EOF", t.Context(), io.ErrUnexpectedEOF, classRetryable}, + {"HTTP/2 stream refused", t.Context(), &url.Error{Op: "Put", URL: "u", Err: errors.New("stream error: REFUSED_STREAM")}, classRetryable}, + {"request timeout", t.Context(), fmt.Errorf("put: %w", context.DeadlineExceeded), classRetryable}, + {"copy cancelled", cancelled, apiError(503, nil, jsonBody), classFatal}, + {"local read error", t.Context(), errors.New("read src/a: input/output error"), classFile}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equal(t, tt.want, classifyError(tt.ctx, tt.err)) + }) + } +} + +func TestDescribeError(t *testing.T) { + assert.Equal(t, "HTTP 500 Internal Server Error", describeError(apiError(500, nil, ""))) + assert.Equal(t, "HTTP 502 Bad Gateway", describeError(apiError(502, nil, "bad gateway"))) + assert.Equal(t, "databricks-api error: boom (HTTP 503 Service Unavailable)", describeError(apiError(503, nil, `{"message": "boom"}`))) + assert.Equal(t, "HTTP 500 Internal Server Error", describeError(&sdkapierr.APIError{StatusCode: 500})) + assert.Equal(t, "read failed", describeError(errors.New("read failed"))) +} + +// names returns n file names: f00.txt, f01.txt, ... +func names(n int) []string { + var out []string + for i := range n { + out = append(out, fmt.Sprintf("f%02d.txt", i)) + } + return out +} + +// testSource returns a source filer with the given files under "src". Each +// file holds 3 bytes. +func testSource(files ...string) *filer.FakeFiler { + entries := map[string]fakefs.FileInfo{"src": {FakeDir: true}} + for _, f := range files { + p := path.Join("src", f) + entries[p] = fakefs.FileInfo{FakeSize: 3} + for d := path.Dir(p); d != "src"; d = path.Dir(d) { + entries[d] = fakefs.FileInfo{FakeDir: true} + } + } + return filer.NewFakeFiler(entries) +} + +// errCancel makes a scripted write cancel the copy's context. +var errCancel = errors.New("cancel") + +// scriptedTarget is a target filer whose writes return what write returns for +// the path and the attempt on it, counting from 1. A Stat fails the test unless +// the caller sets one. +type scriptedTarget struct { + mockFiler + + mu sync.Mutex + attempts map[string]int + order []string +} + +func newScriptedTarget(t *testing.T, cancel context.CancelFunc, write func(ctx context.Context, p string, attempt int) error) *scriptedTarget { + st := &scriptedTarget{attempts: map[string]int{}} + st.stat = func(ctx context.Context, p string) (fs.FileInfo, error) { + t.Errorf("unexpected Stat of %s", p) + return nil, fs.ErrNotExist + } + st.write = func(ctx context.Context, p string, r io.Reader, mode ...filer.WriteMode) error { + st.mu.Lock() + st.attempts[p]++ + attempt := st.attempts[p] + st.order = append(st.order, p) + st.mu.Unlock() + if write == nil { + return nil + } + err := write(ctx, p, attempt) + if errors.Is(err, errCancel) { + cancel() + return context.Canceled + } + return err + } + return st +} + +type copyResult struct { + r *recursiveCopy + err error + stdout string + stderr string + log string +} + +// runCopy copies "src" to "dst", capturing stdout, stderr and the debug log. +func runCopy(t *testing.T, ctx context.Context, c *copy, stdout io.Writer) copyResult { + var out, stderr, logs bytes.Buffer + if stdout == nil { + stdout = &out + } + format := flags.OutputText + if c.outputJSON { + format = flags.OutputJSON + } + ctx = cmdio.InContext(ctx, cmdio.NewIO(ctx, format, strings.NewReader(""), stdout, &stderr, "", "")) + ctx = log.NewContext(ctx, slog.New(slog.NewTextHandler(&logs, &slog.HandlerOptions{Level: slog.LevelDebug}))) + r := &recursiveCopy{c: c} + + done := make(chan error, 1) + go func() { done <- r.run(ctx, "src", "dst") }() + select { + case err := <-done: + return copyResult{r: r, err: err, stdout: out.String(), stderr: stderr.String(), log: logs.String()} + case <-time.After(10 * time.Second): + t.Fatal("recursive copy did not finish") + return copyResult{} + } +} + +// brokenPipe is stdout after the reader went away. +type brokenPipe struct{} + +var errBrokenPipe = errors.New("write |1: broken pipe") + +func (brokenPipe) Write([]byte) (int, error) { return 0, errBrokenPipe } + +func TestRecursiveCopy(t *testing.T) { + notFound := apiError(404, nil, `{"error_code": "NOT_FOUND", "message": "gone"}`) + sized := func(size int64) func(ctx context.Context, p string) (fs.FileInfo, error) { + return func(ctx context.Context, p string) (fs.FileInfo, error) { + return fakefs.FileInfo{FakeName: path.Base(p), FakeSize: size}, nil + } + } + // failsOnceThen makes f01 fail with a 500 on its first attempt, which + // defers it, and return err on its retry. + failsOnceThen := func(err error) func(ctx context.Context, p string, attempt int) error { + return func(ctx context.Context, p string, attempt int) error { + switch { + case p != "dst/f01.txt": + return nil + case attempt == 1: + return apiError(500, nil, "") + } + return err + } + } + only := func(file string, err error) func(ctx context.Context, p string, attempt int) error { + return func(ctx context.Context, p string, attempt int) error { + if p == file { + return err + } + return nil + } + } + + var failedTwelve strings.Builder + failedTwelve.WriteString("Copy incomplete: 0 copied, 0 skipped (already existed), 12 failed, 0 not attempted.\nFailed:\n") + for _, f := range names(10) { + fmt.Fprintf(&failedTwelve, " src/%s -> dst/%s: databricks-api error: gone (HTTP 404 Not Found)\n", f, f) + } + failedTwelve.WriteString(" ... and 2 more\n") + + tests := []struct { + name string + files []string + concurrency int + json bool + scheme string + brokenPipe bool + write func(ctx context.Context, p string, attempt int) error + targetStat func(ctx context.Context, p string) (fs.FileInfo, error) + mkdir func(ctx context.Context, p string) error + + wantErr string + wantStderr string + wantAttempts map[string]int + wantLog string + }{ + { + name: "all copied", + files: names(5), + wantStderr: "Copied 5 files.\n", + }, + { + name: "one file", + files: names(1), + wantStderr: "Copied 1 file.\n", + }, + { + name: "JSON output leaves out the line for a complete copy", + files: names(3), + json: true, + wantStderr: "", + }, + { + // A file that existed before the copy is skipped without a Stat. + name: "existing file is skipped", + files: names(3), + write: only("dst/f00.txt", fs.ErrExist), + wantStderr: "Copied 2 files, skipped 1 that already existed.\n", + }, + { + name: "a 500 is retried after the walk", + files: names(4), + write: failsOnceThen(nil), + wantStderr: "Copied 4 files.\n", + wantAttempts: map[string]int{"dst/f01.txt": 2}, + wantLog: "fs cp: deferring src/f01.txt: HTTP 500 Internal Server Error", + }, + { + name: "a file still failing on its retry fails alone", + files: names(4), + write: failsOnceThen(apiError(503, nil, "")), + wantErr: "1 file was not copied", + wantStderr: "Copy incomplete: 3 copied, 0 skipped (already existed), 1 failed, 0 not attempted.\nFailed:\n src/f01.txt -> dst/f01.txt: HTTP 503 Service Unavailable\n", + wantAttempts: map[string]int{"dst/f01.txt": 2}, + wantLog: "fs cp: src/f01.txt failed on its retry: HTTP 503 Service Unavailable", + }, + { + name: "a 404 fails only that file, without a retry", + files: names(4), + write: only("dst/f01.txt", notFound), + wantErr: "1 file was not copied", + wantStderr: "Copy incomplete: 3 copied, 0 skipped (already existed), 1 failed, 0 not attempted.\nFailed:\n src/f01.txt -> dst/f01.txt: databricks-api error: gone (HTTP 404 Not Found)\n", + wantAttempts: map[string]int{"dst/f01.txt": 1}, + }, + { + // The walk goes on to count f02 to f04. + name: "a 401 stops the copy", + files: names(5), + concurrency: 1, + write: only("dst/f01.txt", apiError(401, nil, `{"message": "token expired"}`)), + wantErr: "databricks-api error: token expired", + wantStderr: "Copy incomplete: 1 copied, 0 skipped (already existed), 1 failed, 3 not attempted.\nStopped early because of the error shown below.\nFailed:\n src/f01.txt -> dst/f01.txt: databricks-api error: token expired (HTTP 401 Unauthorized)\n", + }, + { + name: "lists the first 10 failures", + files: names(12), + concurrency: 1, + write: func(context.Context, string, int) error { return notFound }, + wantErr: "12 files were not copied", + wantStderr: failedTwelve.String(), + }, + { + name: "shows schemes", + files: names(1), + scheme: "dbfs", + write: only("dst/f00.txt", notFound), + wantErr: "1 file was not copied", + wantStderr: "Copy incomplete: 0 copied, 0 skipped (already existed), 1 failed, 0 not attempted.\nFailed:\n src/f00.txt -> dbfs:/dst/f00.txt: databricks-api error: gone (HTTP 404 Not Found)\n", + }, + { + // The first attempt landed even though it returned an error. + name: "a retry finding a target of the same size skips it", + files: names(2), + write: failsOnceThen(fs.ErrExist), + targetStat: sized(3), + wantStderr: "Copied 1 file, skipped 1 that already existed.\n", + }, + { + name: "a retry finding a target of another size fails it", + files: names(2), + write: failsOnceThen(fs.ErrExist), + targetStat: sized(1), + wantErr: "1 file was not copied", + wantStderr: "Copy incomplete: 1 copied, 0 skipped (already existed), 1 failed, 0 not attempted.\nFailed:\n src/f01.txt -> dst/f01.txt: target exists with a different size; an earlier attempt may have left a partial file. Rerun with --overwrite to replace it\n", + }, + { + name: "a retry failing to Stat the target fails it", + files: names(2), + write: failsOnceThen(fs.ErrExist), + targetStat: func(context.Context, string) (fs.FileInfo, error) { return nil, apiError(503, nil, "") }, + wantErr: "1 file was not copied", + wantStderr: "Copy incomplete: 1 copied, 0 skipped (already existed), 1 failed, 0 not attempted.\nFailed:\n src/f01.txt -> dst/f01.txt: HTTP 503 Service Unavailable\n", + }, + { + name: "a 403 from that Stat stops the copy", + files: names(2), + write: failsOnceThen(fs.ErrExist), + targetStat: func(context.Context, string) (fs.FileInfo, error) { + return nil, apiError(403, nil, `{"message": "denied"}`) + }, + wantErr: "databricks-api error: denied", + wantStderr: "Copy incomplete: 1 copied, 0 skipped (already existed), 1 failed, 0 not attempted.\nStopped early because of the error shown below.\nFailed:\n src/f01.txt -> dst/f01.txt: databricks-api error: denied (HTTP 403 Forbidden)\n", + }, + { + // f01 is cancelled mid-copy; f02 and f03 are counted but not + // started. + name: "cancelling the copy stops it", + files: names(4), + concurrency: 1, + write: only("dst/f01.txt", errCancel), + wantErr: "copy cancelled: context canceled", + wantStderr: "Copy incomplete: 1 copied, 0 skipped (already existed), 1 failed, 2 not attempted.\nStopped: the copy was cancelled.\nFailed:\n src/f01.txt -> dst/f01.txt: cancelled: the copy stopped before this file finished\n", + }, + { + name: "a deferred file cancelled on its retry keeps its error", + files: names(3), + write: failsOnceThen(errCancel), + wantErr: "copy cancelled: context canceled", + wantStderr: "Copy incomplete: 2 copied, 0 skipped (already existed), 1 failed, 0 not attempted.\nStopped: the copy was cancelled.\nFailed:\n src/f01.txt -> dst/f01.txt: HTTP 500 Internal Server Error\n", + }, + { + name: "failing to print stops the copy", + files: names(4), + concurrency: 1, + brokenPipe: true, + wantErr: errBrokenPipe.Error(), + wantStderr: "Copy incomplete: 0 copied, 0 skipped (already existed), 1 failed, 3 not attempted.\nStopped early because of the error shown below.\nFailed:\n src/f00.txt -> dst/f00.txt: write |1: broken pipe\n", + }, + { + // The copy of a.txt waits until it is cancelled. The walk ends at + // the error, so files it never reached are not counted. + name: "a failed Mkdir stops the copy and cancels in-flight files", + files: []string{"a.txt", "sub/b.txt"}, + write: func(ctx context.Context, p string, attempt int) error { + <-ctx.Done() + return ctx.Err() + }, + mkdir: func(ctx context.Context, p string) error { + if p == "dst/sub" { + return apiError(400, nil, `{"message": "bad path"}`) + } + return nil + }, + wantErr: "databricks-api error: bad path", + wantStderr: "Copy incomplete: 0 copied, 0 skipped (already existed), 1 failed, at least 0 not attempted.\nStopped early because of the error shown below.\nFailed:\n src/a.txt -> dst/a.txt: cancelled: the copy stopped before this file finished\n", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + target := newScriptedTarget(t, cancel, tt.write) + if tt.targetStat != nil { + target.stat = tt.targetStat + } + target.mkdir = tt.mkdir + c := ©{ + recursive: true, + concurrency: orDefault(tt.concurrency, 2), + outputJSON: tt.json, + targetScheme: tt.scheme, + sourceFiler: testSource(tt.files...), + targetFiler: target, + } + var stdout io.Writer + if tt.brokenPipe { + stdout = brokenPipe{} + } + res := runCopy(t, ctx, c, stdout) + + if tt.wantErr == "" { + assert.NoError(t, res.err) + } else { + assert.EqualError(t, res.err, tt.wantErr) + } + assert.Equal(t, tt.wantStderr, res.stderr) + if !tt.json && !tt.brokenPipe { + assert.Equal(t, res.r.copied+res.r.skipped, strings.Count(res.stdout, "\n"), "one stdout line per copied or skipped file") + } + for p, n := range tt.wantAttempts { + assert.Equal(t, n, target.attempts[p], "attempts on %s", p) + } + if tt.wantAttempts["dst/f01.txt"] == 2 { + assert.Equal(t, "dst/f01.txt", target.order[len(target.order)-1], "the retry comes after the walk") + } + assert.Contains(t, res.log, tt.wantLog) + }) + } +} + +// orDefault returns v, or def if v is zero. +func orDefault(v, def int) int { + if v == 0 { + return def + } + return v +} diff --git a/cmd/fs/cp_test.go b/cmd/fs/cp_test.go index a25416d6bc5..29ee43810bf 100644 --- a/cmd/fs/cp_test.go +++ b/cmd/fs/cp_test.go @@ -2,7 +2,6 @@ package fs import ( "context" - "errors" "fmt" "io" "io/fs" @@ -97,112 +96,6 @@ func (m mockDirEntry) Info() (fs.FileInfo, error) { return mockFileInfo(m), nil } -func TestCp_cpDirToDir_contextCancellation(t *testing.T) { - testError := errors.New("test error") - - // Mock the stats and readDir methods for a Filer over a file system that - // has the following directory structure: - // - // src/ - // ├── subdir/ - // ├── file1.txt - // ├── file2.txt - // └── file3.txt - // - mockSourceStat := func(ctx context.Context, path string) (fs.FileInfo, error) { - isDir := path == "src" || path == "src/subdir" - return mockFileInfo{name: path, isDir: isDir}, nil - } - mockSourceReadDir := func(ctx context.Context, path string) ([]fs.DirEntry, error) { - if path == "src" { - return []fs.DirEntry{ - mockDirEntry{name: "subdir", isDir: true}, - mockDirEntry{name: "file1.txt", isDir: false}, - mockDirEntry{name: "file2.txt", isDir: false}, - mockDirEntry{name: "file3.txt", isDir: false}, - }, nil - } - return nil, nil - } - - testCases := []struct { - desc string - c *copy - wantErr error - }{ - { - // The source filer's Read method blocks until context is cancelled, - // simulating a slow file copy operation. The target filer's Mkdir - // method returns an error which should cancel the walk and all file - // copy goroutines. - desc: "cancel go routines on walk error", - c: ©{ - recursive: true, - concurrency: 5, - sourceFiler: &mockFiler{ - stat: mockSourceStat, - readDir: mockSourceReadDir, - read: func(ctx context.Context, path string) (io.ReadCloser, error) { - <-ctx.Done() // block until context is cancelled - return nil, ctx.Err() - }, - }, - targetFiler: &mockFiler{ - mkdir: func(ctx context.Context, path string) error { - return testError - }, - }, - }, - wantErr: testError, - }, - { - // The target filer's Write method returns an error when writing the - // file1.txt file. This error is expected to be returned by the file copy - // goroutine and all other file copy goroutines should be cancelled. - desc: "cancel go routines on file copy error", - c: ©{ - recursive: true, - concurrency: 5, - sourceFiler: &mockFiler{ - stat: mockSourceStat, - readDir: mockSourceReadDir, - read: func(ctx context.Context, path string) (io.ReadCloser, error) { - return io.NopCloser(strings.NewReader("content")), nil - }, - }, - targetFiler: &mockFiler{ - write: func(ctx context.Context, path string, r io.Reader, mode ...filer.WriteMode) error { - if path == "dst/file1.txt" { - return testError - } - <-ctx.Done() // block until context is cancelled - return ctx.Err() - }, - }, - }, - wantErr: testError, - }, - } - - for _, tc := range testCases { - t.Run(tc.desc, func(t *testing.T) { - done := make(chan error, 1) - go func() { - done <- tc.c.cpDirToDir(t.Context(), "src", "dst") - }() - - select { - case gotErr := <-done: - if !errors.Is(gotErr, tc.wantErr) { - t.Errorf("want error %v, got %v", tc.wantErr, gotErr) - } - case <-time.After(3 * time.Second): // do not wait too long in case of test issues - t.Fatal("cpDirToDir blocked instead of returning error immediately") - } - }) - } -} - func TestCp_writeModes(t *testing.T) { // src/ // ├── file1.txt