Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .nextchanges/cli/fs-cp-r-directory-retry.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
* `databricks fs cp -r` retries a temporary error (HTTP 408, 429, 5xx, or a network error) when listing a source directory or creating a target one, for up to about 5 minutes, instead of stopping the copy. ([#7787](https://github.com/databricks/cli/pull/7787))
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
file 1
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
file 2
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
file in sub
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
Cloud = false
13 changes: 13 additions & 0 deletions acceptance/cmd/fs/cp-r-keep-going/mkdir-500/output.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@

>>> cp_r localdir dbfs:/Volumes/main/default/keep-going/dst --concurrency 2
Copied 3 files.
localdir/f1.txt -> dbfs:/Volumes/main/default/keep-going/dst/f1.txt
localdir/f2.txt -> dbfs:/Volumes/main/default/keep-going/dst/f2.txt
localdir/sub/f.txt -> dbfs:/Volumes/main/default/keep-going/dst/sub/f.txt

=== The PUT of dst/sub is sent twice

>>> print_requests.py --summary --sort //api/2.0/fs/directories/
PUT /api/2.0/fs/directories/Volumes/main/default/keep-going/dst
PUT /api/2.0/fs/directories/Volumes/main/default/keep-going/dst/sub
PUT /api/2.0/fs/directories/Volumes/main/default/keep-going/dst/sub
8 changes: 8 additions & 0 deletions acceptance/cmd/fs/cp-r-keep-going/mkdir-500/script
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
# A 500 on creating a target directory is retried in place, and the copy
# succeeds.
fault.py "PUT /api/2.0/fs/directories/**" 500 1 1

trace cp_r localdir ${TARGET} --concurrency 2

title "The PUT of dst/sub is sent twice"
trace print_requests.py --summary --sort //api/2.0/fs/directories/
2 changes: 2 additions & 0 deletions acceptance/cmd/fs/cp-r-keep-going/mkdir-500/test.toml
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
# Test-only override: cap the retry wait at 200ms.
Env.DATABRICKS_FS_CP_PROBE_BACKOFF_MAX_MS = "200"
79 changes: 75 additions & 4 deletions cmd/fs/cp_recursive.go
Original file line number Diff line number Diff line change
Expand Up @@ -34,8 +34,9 @@ const maxListedFailures = 10
const stallTimeout = 5 * time.Minute

// probeBackoffMin and probeBackoffMax bound the pause before each attempt made
// while the circuit breaker is open. A Retry-After hint from the server is
// honored, capped at probeBackoffMax.
// while the circuit breaker is open, and before each retry of a directory
// listing or Mkdir. A Retry-After hint from the server is honored, capped at
// probeBackoffMax.
const (
probeBackoffMin = 2 * time.Second
probeBackoffMax = 30 * time.Second
Expand Down Expand Up @@ -156,6 +157,8 @@ type copyJob struct {
// failures never stalls.
// - When a fatal error or a cancellation stops the copy, in-flight files are
// cancelled and reported as failed.
// - A retryable error listing a source directory or creating a target one is
// retried in place, for up to the stall timeout.
// - Once the copy stops, the walk goes on only to count the files left, which
// are reported as not attempted.
type recursiveCopy struct {
Expand Down Expand Up @@ -245,7 +248,7 @@ func (r *recursiveCopy) run(ctx context.Context, sourceDir, targetDir string) er
r.lastSuccess = r.now()
r.slots = make(chan struct{}, r.c.concurrency)

sourceFs := filer.NewFS(ctx, r.c.sourceFiler)
sourceFs := filer.NewFS(ctx, &walkRetryFiler{Filer: r.c.sourceFiler, r: r})
walkErr := fs.WalkDir(sourceFs, sourceDir, func(sourcePath string, d fs.DirEntry, err error) error {
if err != nil {
return err
Expand All @@ -265,7 +268,9 @@ func (r *recursiveCopy) run(ctx context.Context, sourceDir, targetDir string) er
}
// Create the directory before the files in it. WalkDir visits a
// directory before its contents.
if err := r.c.targetFiler.Mkdir(ctx, targetPath); err != nil {
if err := r.retryWalk(ctx, "create directory "+targetPath, func() error {
return r.c.targetFiler.Mkdir(ctx, targetPath)
}); err != nil {
return err
}
// A created directory is progress too, so a tree of slow directories
Expand Down Expand Up @@ -558,6 +563,72 @@ func (r *recursiveCopy) checkExistingTarget(ctx context.Context, job copyJob) er
return nil
}

// retryWalk runs op, a directory listing or Mkdir, retrying a retryable error
// in place with the probe backoff for up to the stall timeout from the first
// attempt. The walk is sequential, so this makes one call at a time.
func (r *recursiveCopy) retryWalk(ctx context.Context, what string, op func() error) error {
start := r.now()
backoff := r.probeBackoffMin
for retried := false; ; retried = true {
err := op()
if err == nil {
// A recovered call is progress, so its retries do not use up the
// stall timeout of the files that follow.
if retried {
r.progress()
}
return nil
}
// No retries while a stalled copy only counts the files left.
if classifyError(ctx, err) != classRetryable || r.isStalled() {
return err
}
remaining := r.stallTimeout - r.now().Sub(start)
if remaining <= 0 {
return err
}
delay := min(max(backoff, retryAfter(err)), r.probeBackoffMax, remaining)
backoff = min(backoff*2, r.probeBackoffMax)
log.Debugf(ctx, "fs cp: retrying %s in %s after: %s", what, delay, describeError(err))
if r.sleep(ctx, delay) != nil {
return err
}
}
}

func (r *recursiveCopy) isStalled() bool {
r.mu.Lock()
defer r.mu.Unlock()
return r.stalled
}

// walkRetryFiler retries the calls the walk makes on the source: a Stat of
// the root and a ReadDir of every directory.
type walkRetryFiler struct {
filer.Filer
r *recursiveCopy
}

func (w *walkRetryFiler) ReadDir(ctx context.Context, name string) ([]fs.DirEntry, error) {
var entries []fs.DirEntry
err := w.r.retryWalk(ctx, "list "+name, func() error {
var err error
entries, err = w.Filer.ReadDir(ctx, name)
return err
})
return entries, err
}

func (w *walkRetryFiler) Stat(ctx context.Context, name string) (fs.FileInfo, error) {
var info fs.FileInfo
err := w.r.retryWalk(ctx, "stat "+name, func() error {
var err error
info, err = w.Filer.Stat(ctx, name)
return err
})
return info, err
}

// finish prints the end-of-run summary to stderr and returns the command's
// error. A complete copy gets one line, which JSON output omits. An incomplete
// copy gets the counts, the reason it stopped early, if it did, and the first
Expand Down
135 changes: 135 additions & 0 deletions cmd/fs/cp_recursive_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -844,3 +844,138 @@ func TestProbeBackoffFromEnv(t *testing.T) {
assert.Equal(t, 100*time.Millisecond, r.probeBackoffMax)
assert.Equal(t, 100*time.Millisecond, r.probeBackoffMin, "a lower maximum lowers the minimum")
}

func TestRecursiveCopyWalkRetry(t *testing.T) {
unavailable := apiError(503, nil, "")
sec := func(n ...int) []time.Duration {
var out []time.Duration
for _, v := range n {
out = append(out, time.Duration(v)*time.Second)
}
return out
}
tests := []struct {
name string
files []string
listDir string
readDir func(attempt int) error
mkdir func(p string, attempt int) error
write func(clock *fakeClock, p string, attempt int) error
wantErr error
wantStderr string
wantSleeps []time.Duration
wantLog string
}{
{
name: "a Mkdir 500 is retried in place",
mkdir: func(p string, attempt int) error {
if p == "dst/sub" && attempt <= 2 {
return apiError(500, nil, "")
}
return nil
},
wantStderr: "Copied 2 files.\n",
wantSleeps: sec(2, 4),
wantLog: "fs cp: retrying create directory dst/sub in 2s after: HTTP 500 Internal Server Error",
},
{
// The listing recovers after 4m30s of retries, then b.txt takes a
// minute and fails once. Its retry still runs, because the
// recovered listing counts as progress.
name: "a listing recovering after a long retry leaves the files their stall timeout",
files: []string{"sub/b.txt"},
readDir: func(attempt int) error {
if attempt <= 12 {
return unavailable
}
return nil
},
write: func(clock *fakeClock, p string, attempt int) error {
if p != "dst/sub/b.txt" {
return nil
}
clock.Advance(time.Minute)
if attempt == 1 {
return unavailable
}
return nil
},
wantStderr: "Copied 1 file.\n",
wantSleeps: sec(2, 4, 8, 16, 30, 30, 30, 30, 30, 30, 30, 30, 2),
wantLog: "fs cp: retrying list src/sub in 2s after: HTTP 503 Service Unavailable",
},
{
// The backoff is capped at 30s, and the retries add up to the
// stall timeout.
name: "a listing failing for the stall timeout stops the copy",
listDir: "src",
readDir: func(int) error { return unavailable },
wantErr: unavailable,
wantStderr: "Copy incomplete: 0 copied, 0 skipped (already existed), 0 failed, at least 0 not attempted.\nStopped early because of the error shown below.\n",
wantSleeps: sec(2, 4, 8, 16, 30, 30, 30, 30, 30, 30, 30, 30, 30),
wantLog: "fs cp: retrying list src in 2s after: HTTP 503 Service Unavailable",
},
{
// a.txt fails after 6 minutes, which opens the breaker, and so does
// its probe 2s later, so b.txt finds the copy stalled. The walk then
// only counts files, and does not retry the failed listing of sub:
// the only sleep is the breaker's.
name: "a stalled copy does not retry listings",
files: []string{"a.txt", "b.txt", "sub/c.txt"},
readDir: func(int) error { return unavailable },
write: func(clock *fakeClock, p string, attempt int) error {
clock.Advance(6 * time.Minute)
return unavailable
},
wantErr: errors.New("copy stopped: files kept failing and none was copied for 5 minutes"),
wantStderr: "Copy incomplete: 0 copied, 0 skipped (already existed), 1 failed, at least 1 not attempted.\nStopped: files kept failing and none was copied for 5 minutes, so no new files were started.\nFailed:\n src/a.txt -> dst/a.txt: HTTP 503 Service Unavailable\n",
wantSleeps: sec(2),
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
ctx, cancel := context.WithCancel(t.Context())
defer cancel()
clock := &fakeClock{}
files := tt.files
if files == nil {
files = []string{"a.txt", "sub/b.txt"}
}
source := &flakySource{FakeFiler: testSource(files...), dir: "src/sub", readDir: func(int) error { return nil }}
if tt.listDir != "" {
source.dir = tt.listDir
}
if tt.readDir != nil {
source.readDir = tt.readDir
}
target := newScriptedTarget(t, cancel, func(ctx context.Context, p string, attempt int) error {
if tt.write == nil {
return nil
}
return tt.write(clock, p, attempt)
})
mkdirs := map[string]int{}
target.mkdir = func(ctx context.Context, p string) error {
mkdirs[p]++
if tt.mkdir == nil {
return nil
}
return tt.mkdir(p, mkdirs[p])
}
c := &copy{recursive: true, concurrency: 1, sourceFiler: source, targetFiler: target}
res := runCopy(t, ctx, c, nil, clock.install)

switch {
case tt.wantErr == nil:
assert.NoError(t, res.err)
case errors.Is(tt.wantErr, unavailable):
assert.ErrorIs(t, res.err, unavailable)
default:
assert.EqualError(t, res.err, tt.wantErr.Error())
}
assert.Equal(t, tt.wantStderr, res.stderr)
assert.Equal(t, tt.wantSleeps, clock.sleeps)
assert.Contains(t, res.log, tt.wantLog)
})
}
}
Loading