diff --git a/.nextchanges/cli/fs-cp-r-backoff.md b/.nextchanges/cli/fs-cp-r-backoff.md new file mode 100644 index 00000000000..6fefb0cb675 --- /dev/null +++ b/.nextchanges/cli/fs-cp-r-backoff.md @@ -0,0 +1 @@ +* `databricks fs cp -r` backs off when files keep failing. After as many failures in a row as `--concurrency`, it starts no new files and retries one failed file at a time, waiting 2 to 30 seconds between tries (or the server's Retry-After, up to 30 seconds), until one succeeds. ([#7786](https://github.com/databricks/cli/pull/7786)) diff --git a/acceptance/cmd/fs/cp-r-keep-going/burst-retry-after/localdir/f1.txt b/acceptance/cmd/fs/cp-r-keep-going/burst-retry-after/localdir/f1.txt new file mode 100644 index 00000000000..366f17ff507 --- /dev/null +++ b/acceptance/cmd/fs/cp-r-keep-going/burst-retry-after/localdir/f1.txt @@ -0,0 +1 @@ +file 1 diff --git a/acceptance/cmd/fs/cp-r-keep-going/burst-retry-after/localdir/f2.txt b/acceptance/cmd/fs/cp-r-keep-going/burst-retry-after/localdir/f2.txt new file mode 100644 index 00000000000..fac580e9808 --- /dev/null +++ b/acceptance/cmd/fs/cp-r-keep-going/burst-retry-after/localdir/f2.txt @@ -0,0 +1 @@ +file 2 diff --git a/acceptance/cmd/fs/cp-r-keep-going/burst-retry-after/localdir/f3.txt b/acceptance/cmd/fs/cp-r-keep-going/burst-retry-after/localdir/f3.txt new file mode 100644 index 00000000000..dc43ed8669d --- /dev/null +++ b/acceptance/cmd/fs/cp-r-keep-going/burst-retry-after/localdir/f3.txt @@ -0,0 +1 @@ +file 3 diff --git a/acceptance/cmd/fs/cp-r-keep-going/burst-retry-after/localdir/f4.txt b/acceptance/cmd/fs/cp-r-keep-going/burst-retry-after/localdir/f4.txt new file mode 100644 index 00000000000..4b24a15d33f --- /dev/null +++ b/acceptance/cmd/fs/cp-r-keep-going/burst-retry-after/localdir/f4.txt @@ -0,0 +1 @@ +file 4 diff --git a/acceptance/cmd/fs/cp-r-keep-going/burst-retry-after/localdir/f5.txt b/acceptance/cmd/fs/cp-r-keep-going/burst-retry-after/localdir/f5.txt new file mode 100644 index 00000000000..d2d8a5a10fa --- /dev/null +++ b/acceptance/cmd/fs/cp-r-keep-going/burst-retry-after/localdir/f5.txt @@ -0,0 +1 @@ +file 5 diff --git a/acceptance/cmd/fs/cp-r-keep-going/burst-retry-after/localdir/f6.txt b/acceptance/cmd/fs/cp-r-keep-going/burst-retry-after/localdir/f6.txt new file mode 100644 index 00000000000..9454c471bbe --- /dev/null +++ b/acceptance/cmd/fs/cp-r-keep-going/burst-retry-after/localdir/f6.txt @@ -0,0 +1 @@ +file 6 diff --git a/acceptance/cmd/fs/cp-r-keep-going/burst-retry-after/out.test.toml b/acceptance/cmd/fs/cp-r-keep-going/burst-retry-after/out.test.toml new file mode 100644 index 00000000000..18b1a88417e --- /dev/null +++ b/acceptance/cmd/fs/cp-r-keep-going/burst-retry-after/out.test.toml @@ -0,0 +1 @@ +Cloud = false diff --git a/acceptance/cmd/fs/cp-r-keep-going/burst-retry-after/output.txt b/acceptance/cmd/fs/cp-r-keep-going/burst-retry-after/output.txt new file mode 100644 index 00000000000..47a2634c362 --- /dev/null +++ b/acceptance/cmd/fs/cp-r-keep-going/burst-retry-after/output.txt @@ -0,0 +1,14 @@ + +>>> cp_r localdir dbfs:/Volumes/main/default/keep-going/dst --concurrency 2 +Copied 6 files. +localdir/f1.txt -> dbfs:/Volumes/main/default/keep-going/dst/f1.txt +localdir/f2.txt -> dbfs:/Volumes/main/default/keep-going/dst/f2.txt +localdir/f3.txt -> dbfs:/Volumes/main/default/keep-going/dst/f3.txt +localdir/f4.txt -> dbfs:/Volumes/main/default/keep-going/dst/f4.txt +localdir/f5.txt -> dbfs:/Volumes/main/default/keep-going/dst/f5.txt +localdir/f6.txt -> dbfs:/Volumes/main/default/keep-going/dst/f6.txt + +=== Six uploads plus the six that hit the burst + +>>> print_requests.py --summary //api/2.0/fs/files/ +12 diff --git a/acceptance/cmd/fs/cp-r-keep-going/burst-retry-after/script b/acceptance/cmd/fs/cp-r-keep-going/burst-retry-after/script new file mode 100644 index 00000000000..397ec4cf50b --- /dev/null +++ b/acceptance/cmd/fs/cp-r-keep-going/burst-retry-after/script @@ -0,0 +1,11 @@ +# A burst of six 500s with Retry-After. The burst is a 500 and not a 503: the +# filer retries a 503 itself, ten times over about 75 seconds, before the copy +# sees it. Two failures in a row open the breaker (--concurrency 2), so the +# copy probes one deferred file at a time, waiting for Retry-After (capped by +# the test override at 1s), until the burst ends. Every file is then copied. +fault.py --header Retry-After:1 "PUT /api/2.0/fs/files/**" 500 0 6 + +trace cp_r localdir ${TARGET} --concurrency 2 + +title "Six uploads plus the six that hit the burst" +trace print_requests.py --summary //api/2.0/fs/files/ | wc -l | tr -d ' ' diff --git a/acceptance/cmd/fs/cp-r-keep-going/burst-retry-after/test.toml b/acceptance/cmd/fs/cp-r-keep-going/burst-retry-after/test.toml new file mode 100644 index 00000000000..cd005aac71e --- /dev/null +++ b/acceptance/cmd/fs/cp-r-keep-going/burst-retry-after/test.toml @@ -0,0 +1,2 @@ +# Test-only override: cap the breaker's wait at 1s. +Env.DATABRICKS_FS_CP_PROBE_BACKOFF_MAX_MS = "1000" diff --git a/acceptance/cmd/fs/cp-r-keep-going/outage-past-stall/localdir/f1.txt b/acceptance/cmd/fs/cp-r-keep-going/outage-past-stall/localdir/f1.txt new file mode 100644 index 00000000000..366f17ff507 --- /dev/null +++ b/acceptance/cmd/fs/cp-r-keep-going/outage-past-stall/localdir/f1.txt @@ -0,0 +1 @@ +file 1 diff --git a/acceptance/cmd/fs/cp-r-keep-going/outage-past-stall/localdir/f2.txt b/acceptance/cmd/fs/cp-r-keep-going/outage-past-stall/localdir/f2.txt new file mode 100644 index 00000000000..fac580e9808 --- /dev/null +++ b/acceptance/cmd/fs/cp-r-keep-going/outage-past-stall/localdir/f2.txt @@ -0,0 +1 @@ +file 2 diff --git a/acceptance/cmd/fs/cp-r-keep-going/outage-past-stall/localdir/f3.txt b/acceptance/cmd/fs/cp-r-keep-going/outage-past-stall/localdir/f3.txt new file mode 100644 index 00000000000..dc43ed8669d --- /dev/null +++ b/acceptance/cmd/fs/cp-r-keep-going/outage-past-stall/localdir/f3.txt @@ -0,0 +1 @@ +file 3 diff --git a/acceptance/cmd/fs/cp-r-keep-going/outage-past-stall/localdir/f4.txt b/acceptance/cmd/fs/cp-r-keep-going/outage-past-stall/localdir/f4.txt new file mode 100644 index 00000000000..4b24a15d33f --- /dev/null +++ b/acceptance/cmd/fs/cp-r-keep-going/outage-past-stall/localdir/f4.txt @@ -0,0 +1 @@ +file 4 diff --git a/acceptance/cmd/fs/cp-r-keep-going/outage-past-stall/out.test.toml b/acceptance/cmd/fs/cp-r-keep-going/outage-past-stall/out.test.toml new file mode 100644 index 00000000000..18b1a88417e --- /dev/null +++ b/acceptance/cmd/fs/cp-r-keep-going/outage-past-stall/out.test.toml @@ -0,0 +1 @@ +Cloud = false diff --git a/acceptance/cmd/fs/cp-r-keep-going/outage-past-stall/output.txt b/acceptance/cmd/fs/cp-r-keep-going/outage-past-stall/output.txt new file mode 100644 index 00000000000..d42cdce2521 --- /dev/null +++ b/acceptance/cmd/fs/cp-r-keep-going/outage-past-stall/output.txt @@ -0,0 +1,9 @@ + +>>> cp_r localdir dbfs:/Volumes/main/default/keep-going/dst --concurrency 1 +Copy incomplete: 0 copied, 0 skipped (already existed), 1 failed, 3 not attempted. +Stopped: files kept failing and none was copied for 2 seconds, so no new files were started. +Failed: + localdir/f1.txt -> dbfs:/Volumes/main/default/keep-going/dst/f1.txt: databricks-api error: Fault injected by test. (HTTP 500 Internal Server Error) +Error: copy stopped: files kept failing and none was copied for 2 seconds + +Exit code: 1 diff --git a/acceptance/cmd/fs/cp-r-keep-going/outage-past-stall/script b/acceptance/cmd/fs/cp-r-keep-going/outage-past-stall/script new file mode 100644 index 00000000000..014fccd25b1 --- /dev/null +++ b/acceptance/cmd/fs/cp-r-keep-going/outage-past-stall/script @@ -0,0 +1,9 @@ +# Every upload fails for longer than the stall timeout. The first file fails +# and is deferred, which opens the breaker. Its probes keep failing until no +# file has been copied for the stall timeout. The copy then stops: the +# deferred file is reported with its error, the other files as not attempted, +# and the command exits 1. +fault.py "PUT /api/2.0/fs/files/**" 500 0 1000 + +errcode trace cp_r localdir ${TARGET} --concurrency 1 +print_requests.py //api/ > /dev/null diff --git a/acceptance/cmd/fs/cp-r-keep-going/outage-past-stall/test.toml b/acceptance/cmd/fs/cp-r-keep-going/outage-past-stall/test.toml new file mode 100644 index 00000000000..503ee6f1b00 --- /dev/null +++ b/acceptance/cmd/fs/cp-r-keep-going/outage-past-stall/test.toml @@ -0,0 +1,3 @@ +# Test-only overrides: a 2s stall timeout and breaker waits of 200ms. +Env.DATABRICKS_FS_CP_STALL_TIMEOUT_MS = "2000" +Env.DATABRICKS_FS_CP_PROBE_BACKOFF_MAX_MS = "200" diff --git a/cmd/fs/cp_recursive.go b/cmd/fs/cp_recursive.go index 8fde8bfa4c0..98030475ab7 100644 --- a/cmd/fs/cp_recursive.go +++ b/cmd/fs/cp_recursive.go @@ -33,9 +33,20 @@ const maxListedFailures = 10 // finish. It is internal, not a flag. const stallTimeout = 5 * time.Minute -// stallTimeoutEnvVar is an undocumented, test-only override of stallTimeout in -// milliseconds. -const stallTimeoutEnvVar = "DATABRICKS_FS_CP_STALL_TIMEOUT_MS" +// probeBackoffMin and probeBackoffMax bound the pause before each attempt made +// while the circuit breaker is open. A Retry-After hint from the server is +// honored, capped at probeBackoffMax. +const ( + probeBackoffMin = 2 * time.Second + probeBackoffMax = 30 * time.Second +) + +// Undocumented, test-only overrides of stallTimeout and probeBackoffMax in +// milliseconds. A probeBackoffMax below probeBackoffMin lowers the minimum too. +const ( + stallTimeoutEnvVar = "DATABRICKS_FS_CP_STALL_TIMEOUT_MS" + probeBackoffMaxEnvVar = "DATABRICKS_FS_CP_PROBE_BACKOFF_MAX_MS" +) // errorClass says what a recursive copy does after an error. type errorClass int @@ -135,6 +146,11 @@ type copyJob struct { // gets one more full attempt, with its source reopened. // - A failure that is not retryable fails only that file, except that 401, // 403, cancellation and a failure to print stop the copy. +// - After --concurrency consecutive retryable failures, a circuit breaker +// opens. While it is open no new files are started. Instead, one deferred +// file at a time is tried, with a backoff between attempts. The first +// success closes it. A failed probe defers its file again, so during an +// outage a file can be tried more than twice, until the stall timeout. // - If files keep failing and none has been copied for the stall timeout, // no new files are started and in-flight ones finish. A slow copy with no // failures never stalls. @@ -145,9 +161,12 @@ type copyJob struct { type recursiveCopy struct { c *copy - // now is the clock, injectable for tests. - now func() time.Time - stallTimeout time.Duration + // now and sleep are the clock, injectable for tests. + now func() time.Time + sleep func(ctx context.Context, d time.Duration) error + stallTimeout time.Duration + probeBackoffMin time.Duration + probeBackoffMax time.Duration // cancel stops in-flight copies after a fatal error. cancel context.CancelFunc @@ -170,6 +189,13 @@ type recursiveCopy struct { // lastSuccess. A permanent error such as a 404 is not a sign of an outage. failingSince time.Time + // consecutiveRetryable counts retryable failures since the last success. + consecutiveRetryable int + breakerOpen bool + probeBackoff time.Duration + // lastRetryAfter is the largest Retry-After hint since the last success. + lastRetryAfter time.Duration + // fatalErr is the error that stopped the copy, if any. fatalErr error // stalled is set once no file succeeded for the stall timeout. @@ -180,11 +206,15 @@ type recursiveCopy struct { } func newRecursiveCopy(ctx context.Context, c *copy) *recursiveCopy { - return &recursiveCopy{ - c: c, - now: time.Now, - stallTimeout: durationFromEnv(ctx, stallTimeoutEnvVar, stallTimeout), - } + r := &recursiveCopy{ + c: c, + now: time.Now, + sleep: sleepContext, + stallTimeout: durationFromEnv(ctx, stallTimeoutEnvVar, stallTimeout), + probeBackoffMax: durationFromEnv(ctx, probeBackoffMaxEnvVar, probeBackoffMax), + } + r.probeBackoffMin = min(probeBackoffMin, r.probeBackoffMax) + return r } // durationFromEnv reads a duration in milliseconds from the environment @@ -197,6 +227,17 @@ func durationFromEnv(ctx context.Context, key string, def time.Duration) time.Du return time.Duration(ms) * time.Millisecond } +func sleepContext(ctx context.Context, d time.Duration) error { + t := time.NewTimer(d) + defer t.Stop() + select { + case <-ctx.Done(): + return ctx.Err() + case <-t.C: + return nil + } +} + func (r *recursiveCopy) run(ctx context.Context, sourceDir, targetDir string) error { ctx, cancel := context.WithCancel(ctx) defer cancel() @@ -246,26 +287,94 @@ func (r *recursiveCopy) run(ctx context.Context, sourceDir, targetDir string) er return r.finish(ctx) } -// start copies job in the background once a slot is free, or records it as not -// started if the copy is stopping. +// start copies job in the background once a slot is free and the breaker +// admits it, or records it as not started if the copy is stopping. func (r *recursiveCopy) start(ctx context.Context, job copyJob) { - // Check again after waiting for a slot: the copy may have stopped. if r.stopping(ctx) { r.notStarted(job) return } + // Wait for a slot before checking the breaker, so the outcome of the + // file before is known. r.slots <- struct{}{} - if r.stopping(ctx) { + if !r.admit(ctx, job) { <-r.slots - r.notStarted(job) return } r.inflight.Go(func() { defer func() { <-r.slots }() - r.copyOne(ctx, job) + _ = r.copyOne(ctx, job) }) } +// admit waits until job may start, and reports whether it should. It runs on +// the walker goroutine, so probing while the breaker is open is serial by +// construction. While the +// breaker is open, it probes one deferred file at a time, after a backoff, +// until one succeeds or the copy stops. On the deferred pass, or when nothing +// is deferred, job itself is the probe, and then admit has copied it. If the +// copy stops, admit records job as not started. +func (r *recursiveCopy) admit(ctx context.Context, job copyJob) bool { + for { + if r.stopping(ctx) { + r.notStarted(job) + return false + } + delay, open := r.breakerDelay() + if !open { + return true + } + if r.sleep(ctx, delay) != nil || r.stopping(ctx) { + r.notStarted(job) + return false + } + probe, deferred := job, false + if !job.final { + if p, ok := r.popDeferred(); ok { + probe, deferred = p, true + } + } + log.Debugf(ctx, "fs cp: breaker open, probing with %s after waiting %s", probe.sourcePath, delay) + if err := r.copyOne(ctx, probe); err != nil { + log.Debugf(ctx, "fs cp: probe with %s failed: %s", probe.sourcePath, describeError(err)) + } else { + log.Debugf(ctx, "fs cp: probe with %s succeeded", probe.sourcePath) + } + if !deferred { + return false + } + } +} + +// breakerDelay reports whether the breaker is open and, if it is, how long to +// wait before the next probe. Each call doubles the backoff for the next one. +func (r *recursiveCopy) breakerDelay() (time.Duration, bool) { + r.mu.Lock() + defer r.mu.Unlock() + if !r.breakerOpen { + return 0, false + } + delay := r.probeBackoff + r.probeBackoff = min(r.probeBackoff*2, r.probeBackoffMax) + return min(max(delay, r.lastRetryAfter), r.probeBackoffMax), true +} + +// retryAfter returns the server's Retry-After hint for err, or 0. +func retryAfter(err error) time.Duration { + if d := apiretry.RetryDurationHint(err); d > 0 { + return d + } + aerr, ok := errors.AsType[*sdkapierr.APIError](err) + if !ok || aerr.ResponseWrapper == nil || aerr.ResponseWrapper.Response == nil { + return 0 + } + seconds, err := strconv.Atoi(aerr.ResponseWrapper.Response.Header.Get("Retry-After")) + if err != nil || seconds <= 0 { + return 0 + } + return time.Duration(seconds) * time.Second +} + // stopping reports whether the copy should start no new files: after a fatal // error, which cancels ctx, or a stall. func (r *recursiveCopy) stopping(ctx context.Context) bool { @@ -341,8 +450,8 @@ var errCancelledMidCopy = errors.New("cancelled: the copy stopped before this fi // errPartialTarget fails a retried file whose target exists with another size. var errPartialTarget = errors.New("target exists with a different size; an earlier attempt may have left a partial file. Rerun with --overwrite to replace it") -// copyOne copies one file and records the outcome. -func (r *recursiveCopy) copyOne(ctx context.Context, job copyJob) { +// copyOne copies one file, records the outcome and returns its error. +func (r *recursiveCopy) copyOne(ctx context.Context, job copyJob) error { // The walk already created the target directory, so skip the filer's // per-file parent-directory check. skipped, err := r.c.writeFile(ctx, job.sourcePath, job.targetPath, filer.CreateParentDirectories) @@ -382,7 +491,7 @@ func (r *recursiveCopy) copyOne(ctx context.Context, job copyJob) { job.err = errCancelledMidCopy } r.notStarted(job) - return + return err } r.mu.Lock() @@ -391,9 +500,24 @@ func (r *recursiveCopy) copyOne(ctx context.Context, job copyJob) { case err == nil: r.lastSuccess = r.now() r.failingSince = time.Time{} + r.consecutiveRetryable = 0 + r.lastRetryAfter = 0 + if r.breakerOpen { + r.breakerOpen = false + log.Infof(ctx, "fs cp: breaker closed: %s was copied; starting new files again", job.sourcePath) + } case class == classRetryable && r.failingSince.IsZero(): r.failingSince = r.now() } + if class == classRetryable { + r.consecutiveRetryable++ + r.lastRetryAfter = max(r.lastRetryAfter, retryAfter(err)) + if !r.breakerOpen && r.consecutiveRetryable >= r.c.concurrency { + r.breakerOpen = true + r.probeBackoff = r.probeBackoffMin + log.Infof(ctx, "fs cp: breaker opened after %d retryable failures in a row (last: %s); trying one file at a time", r.consecutiveRetryable, describeError(err)) + } + } switch { case err == nil && skipped: r.skipped++ @@ -414,6 +538,7 @@ func (r *recursiveCopy) copyOne(ctx context.Context, job copyJob) { r.cancel() } } + return err } // checkExistingTarget returns errPartialTarget if the target's size differs from diff --git a/cmd/fs/cp_recursive_test.go b/cmd/fs/cp_recursive_test.go index d79a714034f..6e6b15689ed 100644 --- a/cmd/fs/cp_recursive_test.go +++ b/cmd/fs/cp_recursive_test.go @@ -24,8 +24,10 @@ import ( "github.com/databricks/cli/libs/flags" "github.com/databricks/cli/libs/log" sdkapierr "github.com/databricks/databricks-sdk-go/apierr" + "github.com/databricks/databricks-sdk-go/common" "github.com/databricks/sdk-go/core/apierr" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func apiError(status int, header http.Header, body string) error { @@ -162,6 +164,7 @@ func runCopy(t *testing.T, ctx context.Context, c *copy, stdout io.Writer, opts ctx = log.NewContext(ctx, slog.New(slog.NewTextHandler(&logs, &slog.HandlerOptions{Level: slog.LevelDebug}))) // Ignore any override in the environment: tests assume the defaults. ctx = env.Set(ctx, stallTimeoutEnvVar, "") + ctx = env.Set(ctx, probeBackoffMaxEnvVar, "") r := newRecursiveCopy(ctx, c) for _, opt := range opts { opt(r) @@ -458,10 +461,28 @@ func (f *flakySource) ReadDir(ctx context.Context, name string) ([]fs.DirEntry, return f.FakeFiler.ReadDir(ctx, name) } -// fakeClock is a clock that only moves when the test advances it. +// fakeClock is a clock that only moves when the test advances it or the code +// under test sleeps. type fakeClock struct { - mu sync.Mutex - t time.Time + mu sync.Mutex + t time.Time + sleeps []time.Duration +} + +func (c *fakeClock) install(r *recursiveCopy) { + r.now = c.Now + r.sleep = c.Sleep +} + +func (c *fakeClock) Sleep(ctx context.Context, d time.Duration) error { + if err := ctx.Err(); err != nil { + return err + } + c.mu.Lock() + defer c.mu.Unlock() + c.t = c.t.Add(d) + c.sleeps = append(c.sleeps, d) + return nil } func (c *fakeClock) Now() time.Time { @@ -478,14 +499,10 @@ func (c *fakeClock) Advance(d time.Duration) { func TestRecursiveCopyStall(t *testing.T) { unavailable := apiError(503, nil, "") - stalledSix := func(notAttempted string) string { - var b strings.Builder - fmt.Fprintf(&b, "Copy incomplete: 0 copied, 0 skipped (already existed), 6 failed, %s not attempted.\n"+ - "Stopped: files kept failing and none was copied for 5 minutes, so no new files were started.\nFailed:\n", notAttempted) - for _, f := range names(6) { - fmt.Fprintf(&b, " src/%s -> dst/%s: HTTP 503 Service Unavailable\n", f, f) - } - return b.String() + stalledOne := func(notAttempted string) string { + return "Copy incomplete: 0 copied, 0 skipped (already existed), 1 failed, " + notAttempted + " not attempted.\n" + + "Stopped: files kept failing and none was copied for 5 minutes, so no new files were started.\nFailed:\n" + + " src/f00.txt -> dst/f00.txt: HTTP 503 Service Unavailable\n" } tests := []struct { @@ -500,16 +517,15 @@ func TestRecursiveCopyStall(t *testing.T) { wantStderr string }{ { - // Each write fails after a minute. f00 fails at 1 minute, so the - // timeout runs to 6 minutes: f00 to f05 fail and are deferred, - // then no new files start, and the deferred files keep their - // errors. + // Each write fails after a minute. f00 fails and opens the + // breaker, which probes it until files have kept failing for 5 + // minutes. Then no new files start, and f00 keeps its error. name: "an outage stops the copy", files: names(20), writeTime: time.Minute, fail: func(string, int) bool { return true }, wantErr: "copy stopped: files kept failing and none was copied for 5 minutes", - wantStderr: stalledSix("14"), + wantStderr: stalledOne("19"), }, { // While the stalled copy only counts the files left, a failed @@ -520,7 +536,7 @@ func TestRecursiveCopyStall(t *testing.T) { writeTime: time.Minute, fail: func(string, int) bool { return true }, wantErr: "copy stopped: files kept failing and none was copied for 5 minutes", - wantStderr: stalledSix("at least 14"), + wantStderr: stalledOne("at least 19"), }, { // Writes take 2 minutes and every other one fails on its first @@ -589,7 +605,7 @@ func TestRecursiveCopyStall(t *testing.T) { }} } c := ©{recursive: true, concurrency: 1, sourceFiler: source, targetFiler: target} - res := runCopy(t, ctx, c, nil, func(r *recursiveCopy) { r.now = clock.Now }) + res := runCopy(t, ctx, c, nil, clock.install) if tt.wantErr == "" { assert.NoError(t, res.err) @@ -681,3 +697,150 @@ func TestFormatDuration(t *testing.T) { assert.Equal(t, "90 seconds", formatDuration(90*time.Second)) assert.Equal(t, "1.5s", formatDuration(1500*time.Millisecond)) } + +func TestRecursiveCopyBreakerBackoff(t *testing.T) { + tests := []struct { + name string + retryAfter string + wantSleeps []time.Duration + }{ + { + name: "exponential from 2s", + wantSleeps: []time.Duration{2 * time.Second, 4 * time.Second, 8 * time.Second, 16 * time.Second}, + }, + { + name: "Retry-After above the backoff is honored", + retryAfter: "5", + wantSleeps: []time.Duration{5 * time.Second, 5 * time.Second, 8 * time.Second, 16 * time.Second}, + }, + { + name: "Retry-After is capped at 30s", + retryAfter: "120", + wantSleeps: []time.Duration{30 * time.Second, 30 * time.Second, 30 * time.Second, 30 * time.Second}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + header := http.Header{} + if tt.retryAfter != "" { + header.Set("Retry-After", tt.retryAfter) + } + // The first 4 writes fail with 503. With one worker the breaker + // opens after the first, and each probe retries the deferred f00 + // until one succeeds. + var mu sync.Mutex + failuresLeft := 4 + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + target := newScriptedTarget(t, cancel, func(context.Context, string, int) error { + mu.Lock() + defer mu.Unlock() + if failuresLeft > 0 { + failuresLeft-- + return apiError(503, header, "") + } + return nil + }) + c := ©{recursive: true, concurrency: 1, sourceFiler: testSource(names(3)...), targetFiler: target} + clock := &fakeClock{} + res := runCopy(t, ctx, c, nil, clock.install) + + require.NoError(t, res.err) + assert.Equal(t, "Copied 3 files.\n", res.stderr) + assert.Equal(t, tt.wantSleeps, clock.sleeps) + assert.Equal(t, []string{"dst/f00.txt", "dst/f00.txt", "dst/f00.txt", "dst/f00.txt", "dst/f00.txt", "dst/f01.txt", "dst/f02.txt"}, target.order) + for _, line := range []string{ + `level=INFO msg="fs cp: breaker opened after 1 retryable failures in a row (last: HTTP 503 Service Unavailable); trying one file at a time"`, + "fs cp: probe with src/f00.txt failed: HTTP 503 Service Unavailable", + "fs cp: probe with src/f00.txt succeeded", + `level=INFO msg="fs cp: breaker closed: src/f00.txt was copied; starting new files again"`, + } { + assert.Contains(t, res.log, line) + } + }) + } +} + +func TestRecursiveCopyBreakerOnDeferredPass(t *testing.T) { + // f00 and f01 fail on their first attempt, which opens the breaker + // (--concurrency 2). On the deferred pass, the first retry is the probe: + // it runs after a backoff, succeeds and closes the breaker, and the second + // retry starts normally. + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + target := newScriptedTarget(t, cancel, func(ctx context.Context, p string, attempt int) error { + if attempt == 1 { + return apiError(503, nil, "") + } + return nil + }) + c := ©{recursive: true, concurrency: 2, sourceFiler: testSource(names(2)...), targetFiler: target} + clock := &fakeClock{} + res := runCopy(t, ctx, c, nil, clock.install) + + require.NoError(t, res.err) + assert.Equal(t, "Copied 2 files.\n", res.stderr) + assert.Equal(t, []time.Duration{2 * time.Second}, clock.sleeps) + assert.Equal(t, map[string]int{"dst/f00.txt": 2, "dst/f01.txt": 2}, target.attempts) + assert.Contains(t, res.log, "fs cp: breaker open, probing with src/f0") + assert.Contains(t, res.log, "fs cp: breaker closed") +} + +func TestBreakerOpensAndCloses(t *testing.T) { + ctx := cmdio.MockDiscard(t.Context()) + hint := http.Header{"Retry-After": []string{"9"}} + steps := []struct { + file string + err error + wantOpen bool + }{ + // With --concurrency 2, a success between failures keeps it closed. + {"f00.txt", apiError(503, nil, ""), false}, + {"f01.txt", nil, false}, + {"f02.txt", apiError(500, nil, ""), false}, + // Two in a row open it. A later failure without a hint keeps the + // Retry-After of the one before. + {"f03.txt", apiError(503, hint, ""), true}, + {"f04.txt", apiError(500, nil, ""), true}, + {"f05.txt", nil, false}, + } + errs := map[string]error{} + var files []string + for _, step := range steps { + errs["dst/"+step.file] = step.err + files = append(files, step.file) + } + c := ©{concurrency: 2, sourceFiler: testSource(files...)} + c.targetFiler = newScriptedTarget(t, nil, func(ctx context.Context, p string, attempt int) error { return errs[p] }) + r := newRecursiveCopy(ctx, c) + r.cancel = func() {} + + for _, step := range steps { + _ = r.copyOne(ctx, copyJob{sourcePath: "src/" + step.file, targetPath: "dst/" + step.file}) + assert.Equal(t, step.wantOpen, r.breakerOpen, "after %s", step.file) + if step.file == "f04.txt" { + delay, open := r.breakerDelay() + assert.True(t, open) + assert.Equal(t, 9*time.Second, delay) + } + } + assert.Len(t, r.deferred, 4) +} + +func TestRetryAfter(t *testing.T) { + header := http.Header{"Retry-After": []string{"7"}} + assert.Equal(t, 7*time.Second, retryAfter(apiError(503, header, ""))) + assert.Equal(t, time.Duration(0), retryAfter(apiError(503, nil, ""))) + dbfs := &sdkapierr.APIError{StatusCode: 503, ResponseWrapper: &common.ResponseWrapper{Response: &http.Response{Header: header}}} + assert.Equal(t, 7*time.Second, retryAfter(dbfs)) +} + +func TestProbeBackoffFromEnv(t *testing.T) { + r := newRecursiveCopy(t.Context(), ©{}) + assert.Equal(t, probeBackoffMin, r.probeBackoffMin) + assert.Equal(t, probeBackoffMax, r.probeBackoffMax) + + r = newRecursiveCopy(env.Set(t.Context(), probeBackoffMaxEnvVar, "100"), ©{}) + assert.Equal(t, 100*time.Millisecond, r.probeBackoffMax) + assert.Equal(t, 100*time.Millisecond, r.probeBackoffMin, "a lower maximum lowers the minimum") +}