From 9b1f622d82c5fdf4ed7047af3d5418f0ce989fad Mon Sep 17 00:00:00 2001 From: Yuanjie Ding Date: Wed, 18 Nov 2026 18:06:39 +0000 Subject: [PATCH] fs cp -r: retry transient directory errors A 500 on one Mkdir or source listing used to end the whole copy. A retryable error on a target Mkdir or a source directory listing during the walk is now retried in place, one call at a time, with the breaker's 2s to 30s backoff and Retry-After, until the call has failed for the stall timeout. A non-retryable error, or one that outlasts the timeout, still stops the copy. Co-authored-by: Isaac --- .nextchanges/cli/fs-cp-r-directory-retry.md | 1 + .../cp-r-keep-going/mkdir-500/localdir/f1.txt | 1 + .../cp-r-keep-going/mkdir-500/localdir/f2.txt | 1 + .../mkdir-500/localdir/sub/f.txt | 1 + .../cp-r-keep-going/mkdir-500/out.test.toml | 1 + .../fs/cp-r-keep-going/mkdir-500/output.txt | 13 ++ .../cmd/fs/cp-r-keep-going/mkdir-500/script | 8 ++ .../fs/cp-r-keep-going/mkdir-500/test.toml | 2 + cmd/fs/cp_recursive.go | 79 +++++++++- cmd/fs/cp_recursive_test.go | 135 ++++++++++++++++++ 10 files changed, 238 insertions(+), 4 deletions(-) create mode 100644 .nextchanges/cli/fs-cp-r-directory-retry.md create mode 100644 acceptance/cmd/fs/cp-r-keep-going/mkdir-500/localdir/f1.txt create mode 100644 acceptance/cmd/fs/cp-r-keep-going/mkdir-500/localdir/f2.txt create mode 100644 acceptance/cmd/fs/cp-r-keep-going/mkdir-500/localdir/sub/f.txt create mode 100644 acceptance/cmd/fs/cp-r-keep-going/mkdir-500/out.test.toml create mode 100644 acceptance/cmd/fs/cp-r-keep-going/mkdir-500/output.txt create mode 100644 acceptance/cmd/fs/cp-r-keep-going/mkdir-500/script create mode 100644 acceptance/cmd/fs/cp-r-keep-going/mkdir-500/test.toml diff --git a/.nextchanges/cli/fs-cp-r-directory-retry.md b/.nextchanges/cli/fs-cp-r-directory-retry.md new file mode 100644 index 00000000000..398d99d8ee4 --- /dev/null +++ b/.nextchanges/cli/fs-cp-r-directory-retry.md @@ -0,0 +1 @@ +* `databricks fs cp -r` retries a temporary error (HTTP 408, 429, 5xx, or a network error) when listing a source directory or creating a target one, for up to about 5 minutes, instead of stopping the copy. ([#7787](https://github.com/databricks/cli/pull/7787)) diff --git a/acceptance/cmd/fs/cp-r-keep-going/mkdir-500/localdir/f1.txt b/acceptance/cmd/fs/cp-r-keep-going/mkdir-500/localdir/f1.txt new file mode 100644 index 00000000000..366f17ff507 --- /dev/null +++ b/acceptance/cmd/fs/cp-r-keep-going/mkdir-500/localdir/f1.txt @@ -0,0 +1 @@ +file 1 diff --git a/acceptance/cmd/fs/cp-r-keep-going/mkdir-500/localdir/f2.txt b/acceptance/cmd/fs/cp-r-keep-going/mkdir-500/localdir/f2.txt new file mode 100644 index 00000000000..fac580e9808 --- /dev/null +++ b/acceptance/cmd/fs/cp-r-keep-going/mkdir-500/localdir/f2.txt @@ -0,0 +1 @@ +file 2 diff --git a/acceptance/cmd/fs/cp-r-keep-going/mkdir-500/localdir/sub/f.txt b/acceptance/cmd/fs/cp-r-keep-going/mkdir-500/localdir/sub/f.txt new file mode 100644 index 00000000000..b147bbd546c --- /dev/null +++ b/acceptance/cmd/fs/cp-r-keep-going/mkdir-500/localdir/sub/f.txt @@ -0,0 +1 @@ +file in sub diff --git a/acceptance/cmd/fs/cp-r-keep-going/mkdir-500/out.test.toml b/acceptance/cmd/fs/cp-r-keep-going/mkdir-500/out.test.toml new file mode 100644 index 00000000000..18b1a88417e --- /dev/null +++ b/acceptance/cmd/fs/cp-r-keep-going/mkdir-500/out.test.toml @@ -0,0 +1 @@ +Cloud = false diff --git a/acceptance/cmd/fs/cp-r-keep-going/mkdir-500/output.txt b/acceptance/cmd/fs/cp-r-keep-going/mkdir-500/output.txt new file mode 100644 index 00000000000..a14e4f9c12a --- /dev/null +++ b/acceptance/cmd/fs/cp-r-keep-going/mkdir-500/output.txt @@ -0,0 +1,13 @@ + +>>> cp_r localdir dbfs:/Volumes/main/default/keep-going/dst --concurrency 2 +Copied 3 files. +localdir/f1.txt -> dbfs:/Volumes/main/default/keep-going/dst/f1.txt +localdir/f2.txt -> dbfs:/Volumes/main/default/keep-going/dst/f2.txt +localdir/sub/f.txt -> dbfs:/Volumes/main/default/keep-going/dst/sub/f.txt + +=== The PUT of dst/sub is sent twice + +>>> print_requests.py --summary --sort //api/2.0/fs/directories/ +PUT /api/2.0/fs/directories/Volumes/main/default/keep-going/dst +PUT /api/2.0/fs/directories/Volumes/main/default/keep-going/dst/sub +PUT /api/2.0/fs/directories/Volumes/main/default/keep-going/dst/sub diff --git a/acceptance/cmd/fs/cp-r-keep-going/mkdir-500/script b/acceptance/cmd/fs/cp-r-keep-going/mkdir-500/script new file mode 100644 index 00000000000..f34753f2f04 --- /dev/null +++ b/acceptance/cmd/fs/cp-r-keep-going/mkdir-500/script @@ -0,0 +1,8 @@ +# A 500 on creating a target directory is retried in place, and the copy +# succeeds. +fault.py "PUT /api/2.0/fs/directories/**" 500 1 1 + +trace cp_r localdir ${TARGET} --concurrency 2 + +title "The PUT of dst/sub is sent twice" +trace print_requests.py --summary --sort //api/2.0/fs/directories/ diff --git a/acceptance/cmd/fs/cp-r-keep-going/mkdir-500/test.toml b/acceptance/cmd/fs/cp-r-keep-going/mkdir-500/test.toml new file mode 100644 index 00000000000..d5aa55b02ce --- /dev/null +++ b/acceptance/cmd/fs/cp-r-keep-going/mkdir-500/test.toml @@ -0,0 +1,2 @@ +# Test-only override: cap the retry wait at 200ms. +Env.DATABRICKS_FS_CP_PROBE_BACKOFF_MAX_MS = "200" diff --git a/cmd/fs/cp_recursive.go b/cmd/fs/cp_recursive.go index 98030475ab7..c040f29acf3 100644 --- a/cmd/fs/cp_recursive.go +++ b/cmd/fs/cp_recursive.go @@ -34,8 +34,9 @@ const maxListedFailures = 10 const stallTimeout = 5 * time.Minute // probeBackoffMin and probeBackoffMax bound the pause before each attempt made -// while the circuit breaker is open. A Retry-After hint from the server is -// honored, capped at probeBackoffMax. +// while the circuit breaker is open, and before each retry of a directory +// listing or Mkdir. A Retry-After hint from the server is honored, capped at +// probeBackoffMax. const ( probeBackoffMin = 2 * time.Second probeBackoffMax = 30 * time.Second @@ -156,6 +157,8 @@ type copyJob struct { // failures never stalls. // - When a fatal error or a cancellation stops the copy, in-flight files are // cancelled and reported as failed. +// - A retryable error listing a source directory or creating a target one is +// retried in place, for up to the stall timeout. // - Once the copy stops, the walk goes on only to count the files left, which // are reported as not attempted. type recursiveCopy struct { @@ -245,7 +248,7 @@ func (r *recursiveCopy) run(ctx context.Context, sourceDir, targetDir string) er r.lastSuccess = r.now() r.slots = make(chan struct{}, r.c.concurrency) - sourceFs := filer.NewFS(ctx, r.c.sourceFiler) + sourceFs := filer.NewFS(ctx, &walkRetryFiler{Filer: r.c.sourceFiler, r: r}) walkErr := fs.WalkDir(sourceFs, sourceDir, func(sourcePath string, d fs.DirEntry, err error) error { if err != nil { return err @@ -265,7 +268,9 @@ func (r *recursiveCopy) run(ctx context.Context, sourceDir, targetDir string) er } // Create the directory before the files in it. WalkDir visits a // directory before its contents. - if err := r.c.targetFiler.Mkdir(ctx, targetPath); err != nil { + if err := r.retryWalk(ctx, "create directory "+targetPath, func() error { + return r.c.targetFiler.Mkdir(ctx, targetPath) + }); err != nil { return err } // A created directory is progress too, so a tree of slow directories @@ -558,6 +563,72 @@ func (r *recursiveCopy) checkExistingTarget(ctx context.Context, job copyJob) er return nil } +// retryWalk runs op, a directory listing or Mkdir, retrying a retryable error +// in place with the probe backoff for up to the stall timeout from the first +// attempt. The walk is sequential, so this makes one call at a time. +func (r *recursiveCopy) retryWalk(ctx context.Context, what string, op func() error) error { + start := r.now() + backoff := r.probeBackoffMin + for retried := false; ; retried = true { + err := op() + if err == nil { + // A recovered call is progress, so its retries do not use up the + // stall timeout of the files that follow. + if retried { + r.progress() + } + return nil + } + // No retries while a stalled copy only counts the files left. + if classifyError(ctx, err) != classRetryable || r.isStalled() { + return err + } + remaining := r.stallTimeout - r.now().Sub(start) + if remaining <= 0 { + return err + } + delay := min(max(backoff, retryAfter(err)), r.probeBackoffMax, remaining) + backoff = min(backoff*2, r.probeBackoffMax) + log.Debugf(ctx, "fs cp: retrying %s in %s after: %s", what, delay, describeError(err)) + if r.sleep(ctx, delay) != nil { + return err + } + } +} + +func (r *recursiveCopy) isStalled() bool { + r.mu.Lock() + defer r.mu.Unlock() + return r.stalled +} + +// walkRetryFiler retries the calls the walk makes on the source: a Stat of +// the root and a ReadDir of every directory. +type walkRetryFiler struct { + filer.Filer + r *recursiveCopy +} + +func (w *walkRetryFiler) ReadDir(ctx context.Context, name string) ([]fs.DirEntry, error) { + var entries []fs.DirEntry + err := w.r.retryWalk(ctx, "list "+name, func() error { + var err error + entries, err = w.Filer.ReadDir(ctx, name) + return err + }) + return entries, err +} + +func (w *walkRetryFiler) Stat(ctx context.Context, name string) (fs.FileInfo, error) { + var info fs.FileInfo + err := w.r.retryWalk(ctx, "stat "+name, func() error { + var err error + info, err = w.Filer.Stat(ctx, name) + return err + }) + return info, err +} + // finish prints the end-of-run summary to stderr and returns the command's // error. A complete copy gets one line, which JSON output omits. An incomplete // copy gets the counts, the reason it stopped early, if it did, and the first diff --git a/cmd/fs/cp_recursive_test.go b/cmd/fs/cp_recursive_test.go index 6e6b15689ed..222c97266d1 100644 --- a/cmd/fs/cp_recursive_test.go +++ b/cmd/fs/cp_recursive_test.go @@ -844,3 +844,138 @@ func TestProbeBackoffFromEnv(t *testing.T) { assert.Equal(t, 100*time.Millisecond, r.probeBackoffMax) assert.Equal(t, 100*time.Millisecond, r.probeBackoffMin, "a lower maximum lowers the minimum") } + +func TestRecursiveCopyWalkRetry(t *testing.T) { + unavailable := apiError(503, nil, "") + sec := func(n ...int) []time.Duration { + var out []time.Duration + for _, v := range n { + out = append(out, time.Duration(v)*time.Second) + } + return out + } + tests := []struct { + name string + files []string + listDir string + readDir func(attempt int) error + mkdir func(p string, attempt int) error + write func(clock *fakeClock, p string, attempt int) error + wantErr error + wantStderr string + wantSleeps []time.Duration + wantLog string + }{ + { + name: "a Mkdir 500 is retried in place", + mkdir: func(p string, attempt int) error { + if p == "dst/sub" && attempt <= 2 { + return apiError(500, nil, "") + } + return nil + }, + wantStderr: "Copied 2 files.\n", + wantSleeps: sec(2, 4), + wantLog: "fs cp: retrying create directory dst/sub in 2s after: HTTP 500 Internal Server Error", + }, + { + // The listing recovers after 4m30s of retries, then b.txt takes a + // minute and fails once. Its retry still runs, because the + // recovered listing counts as progress. + name: "a listing recovering after a long retry leaves the files their stall timeout", + files: []string{"sub/b.txt"}, + readDir: func(attempt int) error { + if attempt <= 12 { + return unavailable + } + return nil + }, + write: func(clock *fakeClock, p string, attempt int) error { + if p != "dst/sub/b.txt" { + return nil + } + clock.Advance(time.Minute) + if attempt == 1 { + return unavailable + } + return nil + }, + wantStderr: "Copied 1 file.\n", + wantSleeps: sec(2, 4, 8, 16, 30, 30, 30, 30, 30, 30, 30, 30, 2), + wantLog: "fs cp: retrying list src/sub in 2s after: HTTP 503 Service Unavailable", + }, + { + // The backoff is capped at 30s, and the retries add up to the + // stall timeout. + name: "a listing failing for the stall timeout stops the copy", + listDir: "src", + readDir: func(int) error { return unavailable }, + wantErr: unavailable, + wantStderr: "Copy incomplete: 0 copied, 0 skipped (already existed), 0 failed, at least 0 not attempted.\nStopped early because of the error shown below.\n", + wantSleeps: sec(2, 4, 8, 16, 30, 30, 30, 30, 30, 30, 30, 30, 30), + wantLog: "fs cp: retrying list src in 2s after: HTTP 503 Service Unavailable", + }, + { + // a.txt fails after 6 minutes, which opens the breaker, and so does + // its probe 2s later, so b.txt finds the copy stalled. The walk then + // only counts files, and does not retry the failed listing of sub: + // the only sleep is the breaker's. + name: "a stalled copy does not retry listings", + files: []string{"a.txt", "b.txt", "sub/c.txt"}, + readDir: func(int) error { return unavailable }, + write: func(clock *fakeClock, p string, attempt int) error { + clock.Advance(6 * time.Minute) + return unavailable + }, + wantErr: errors.New("copy stopped: files kept failing and none was copied for 5 minutes"), + wantStderr: "Copy incomplete: 0 copied, 0 skipped (already existed), 1 failed, at least 1 not attempted.\nStopped: files kept failing and none was copied for 5 minutes, so no new files were started.\nFailed:\n src/a.txt -> dst/a.txt: HTTP 503 Service Unavailable\n", + wantSleeps: sec(2), + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + clock := &fakeClock{} + files := tt.files + if files == nil { + files = []string{"a.txt", "sub/b.txt"} + } + source := &flakySource{FakeFiler: testSource(files...), dir: "src/sub", readDir: func(int) error { return nil }} + if tt.listDir != "" { + source.dir = tt.listDir + } + if tt.readDir != nil { + source.readDir = tt.readDir + } + target := newScriptedTarget(t, cancel, func(ctx context.Context, p string, attempt int) error { + if tt.write == nil { + return nil + } + return tt.write(clock, p, attempt) + }) + mkdirs := map[string]int{} + target.mkdir = func(ctx context.Context, p string) error { + mkdirs[p]++ + if tt.mkdir == nil { + return nil + } + return tt.mkdir(p, mkdirs[p]) + } + c := ©{recursive: true, concurrency: 1, sourceFiler: source, targetFiler: target} + res := runCopy(t, ctx, c, nil, clock.install) + + switch { + case tt.wantErr == nil: + assert.NoError(t, res.err) + case errors.Is(tt.wantErr, unavailable): + assert.ErrorIs(t, res.err, unavailable) + default: + assert.EqualError(t, res.err, tt.wantErr.Error()) + } + assert.Equal(t, tt.wantStderr, res.stderr) + assert.Equal(t, tt.wantSleeps, clock.sleeps) + assert.Contains(t, res.log, tt.wantLog) + }) + } +}