diff --git a/.nextchanges/cli/fs-cp-r-stall-timeout.md b/.nextchanges/cli/fs-cp-r-stall-timeout.md new file mode 100644 index 00000000000..f907781383d --- /dev/null +++ b/.nextchanges/cli/fs-cp-r-stall-timeout.md @@ -0,0 +1 @@ +* `databricks fs cp -r` stops starting new files when files keep failing and none has been copied for 5 minutes, for example during an outage. It lets the files in progress finish and reports the rest as not attempted. A slow copy with no failures is not affected. ([#7785](https://github.com/databricks/cli/pull/7785)) diff --git a/cmd/fs/cp.go b/cmd/fs/cp.go index d7bc6db7e71..f57c90a5f22 100644 --- a/cmd/fs/cp.go +++ b/cmd/fs/cp.go @@ -58,7 +58,7 @@ func (c *copy) cpDirToDir(ctx context.Context, sourceDir, targetDir string) erro if !c.recursive { return fmt.Errorf("source path %s is a directory. Please specify the --recursive flag", sourceDir) } - return (&recursiveCopy{c: c}).run(ctx, sourceDir, targetDir) + return newRecursiveCopy(ctx, c).run(ctx, sourceDir, targetDir) } func (c *copy) cpFileToDir(ctx context.Context, sourcePath, targetDir string) error { diff --git a/cmd/fs/cp_recursive.go b/cmd/fs/cp_recursive.go index c44e200cfde..8fde8bfa4c0 100644 --- a/cmd/fs/cp_recursive.go +++ b/cmd/fs/cp_recursive.go @@ -14,8 +14,10 @@ import ( "strconv" "strings" "sync" + "time" "github.com/databricks/cli/libs/cmdio" + "github.com/databricks/cli/libs/env" "github.com/databricks/cli/libs/filer" "github.com/databricks/cli/libs/log" sdkapierr "github.com/databricks/databricks-sdk-go/apierr" @@ -26,6 +28,15 @@ import ( // maxListedFailures is how many failed files the end-of-run summary lists. const maxListedFailures = 10 +// stallTimeout is how long a recursive copy keeps going while files fail and +// none succeeds. After that it starts no new files and lets in-flight ones +// finish. It is internal, not a flag. +const stallTimeout = 5 * time.Minute + +// stallTimeoutEnvVar is an undocumented, test-only override of stallTimeout in +// milliseconds. +const stallTimeoutEnvVar = "DATABRICKS_FS_CP_STALL_TIMEOUT_MS" + // errorClass says what a recursive copy does after an error. type errorClass int @@ -124,12 +135,20 @@ type copyJob struct { // gets one more full attempt, with its source reopened. // - A failure that is not retryable fails only that file, except that 401, // 403, cancellation and a failure to print stop the copy. -// - Once the copy stops, in-flight files are cancelled and reported as -// failed. The walk goes on only to count the files left, which are -// reported as not attempted. +// - If files keep failing and none has been copied for the stall timeout, +// no new files are started and in-flight ones finish. A slow copy with no +// failures never stalls. +// - When a fatal error or a cancellation stops the copy, in-flight files are +// cancelled and reported as failed. +// - Once the copy stops, the walk goes on only to count the files left, which +// are reported as not attempted. type recursiveCopy struct { c *copy + // now is the clock, injectable for tests. + now func() time.Time + stallTimeout time.Duration + // cancel stops in-flight copies after a fatal error. cancel context.CancelFunc // slots holds one token per running copy, so at most --concurrency files @@ -144,17 +163,45 @@ type recursiveCopy struct { failures []copyJob deferred []copyJob + lastSuccess time.Time + // failingSince is when a file first failed with a retryable error since + // the last file success, or zero if none has. Only a copy failing like + // that can stall, and the stall timeout runs from the later of this and + // lastSuccess. A permanent error such as a 404 is not a sign of an outage. + failingSince time.Time + // fatalErr is the error that stopped the copy, if any. fatalErr error + // stalled is set once no file succeeded for the stall timeout. + stalled bool // walkCutShort is set when the walk ended on an error, so notAttempted // leaves out files it never reached. walkCutShort bool } +func newRecursiveCopy(ctx context.Context, c *copy) *recursiveCopy { + return &recursiveCopy{ + c: c, + now: time.Now, + stallTimeout: durationFromEnv(ctx, stallTimeoutEnvVar, stallTimeout), + } +} + +// durationFromEnv reads a duration in milliseconds from the environment +// variable key, falling back to def if it is unset or invalid. +func durationFromEnv(ctx context.Context, key string, def time.Duration) time.Duration { + ms, err := strconv.Atoi(env.Get(ctx, key)) + if err != nil || ms <= 0 { + return def + } + return time.Duration(ms) * time.Millisecond +} + func (r *recursiveCopy) run(ctx context.Context, sourceDir, targetDir string) error { ctx, cancel := context.WithCancel(ctx) defer cancel() r.cancel = cancel + r.lastSuccess = r.now() r.slots = make(chan struct{}, r.c.concurrency) sourceFs := filer.NewFS(ctx, r.c.sourceFiler) @@ -177,7 +224,13 @@ func (r *recursiveCopy) run(ctx context.Context, sourceDir, targetDir string) er } // Create the directory before the files in it. WalkDir visits a // directory before its contents. - return r.c.targetFiler.Mkdir(ctx, targetPath) + if err := r.c.targetFiler.Mkdir(ctx, targetPath); err != nil { + return err + } + // A created directory is progress too, so a tree of slow directories + // does not stall. + r.progress() + return nil }) if walkErr != nil { r.walkFailed(ctx, walkErr) @@ -213,10 +266,27 @@ func (r *recursiveCopy) start(ctx context.Context, job copyJob) { }) } -// stopping reports whether the copy should start no new files. +// stopping reports whether the copy should start no new files: after a fatal +// error, which cancels ctx, or a stall. func (r *recursiveCopy) stopping(ctx context.Context) bool { - // A fatal error cancels ctx. - return ctx.Err() != nil + if ctx.Err() != nil { + return true + } + r.mu.Lock() + since := r.failingSince + if r.lastSuccess.After(since) { + since = r.lastSuccess + } + stalledNow := !r.stalled && !r.failingSince.IsZero() && r.now().Sub(since) >= r.stallTimeout + if stalledNow { + r.stalled = true + } + stalled := r.stalled + r.mu.Unlock() + if stalledNow { + log.Infof(ctx, "fs cp: files kept failing and none was copied for %s; starting no new files", formatDuration(r.stallTimeout)) + } + return stalled } // walkFailed records an error that ended the walk. It stops the copy, unless @@ -225,10 +295,20 @@ func (r *recursiveCopy) walkFailed(ctx context.Context, err error) { r.mu.Lock() defer r.mu.Unlock() r.walkCutShort = true - if ctx.Err() == nil && r.fatalErr == nil { + if ctx.Err() == nil && r.fatalErr == nil && !r.stalled { r.fatalErr = err r.cancel() + return } + log.Debugf(ctx, "fs cp: the walk stopped counting files: %s", describeError(err)) +} + +// progress records that the copy created a directory, which resets the stall +// timer. Only a file success clears failingSince. +func (r *recursiveCopy) progress() { + r.mu.Lock() + defer r.mu.Unlock() + r.lastSuccess = r.now() } func (r *recursiveCopy) popDeferred() (copyJob, bool) { @@ -308,6 +388,13 @@ func (r *recursiveCopy) copyOne(ctx context.Context, job copyJob) { r.mu.Lock() defer r.mu.Unlock() switch { + case err == nil: + r.lastSuccess = r.now() + r.failingSince = time.Time{} + case class == classRetryable && r.failingSince.IsZero(): + r.failingSince = r.now() + } + switch { case err == nil && skipped: r.skipped++ case err == nil: @@ -358,7 +445,7 @@ func (r *recursiveCopy) finish(ctx context.Context) error { // cancellation came from the caller. cancelled := ctx.Err() != nil && r.fatalErr == nil notCopied := len(r.failures) + r.notAttempted - if notCopied == 0 && r.fatalErr == nil && !cancelled { + if notCopied == 0 && r.fatalErr == nil && !cancelled && !r.stalled { if !r.c.outputJSON { line := "Copied " + plural(r.copied, "file") if r.skipped > 0 { @@ -376,11 +463,14 @@ func (r *recursiveCopy) finish(ctx context.Context) error { var b strings.Builder fmt.Fprintf(&b, "Copy incomplete: %d copied, %d skipped (already existed), %d failed, %s not attempted.", r.copied, r.skipped, len(r.failures), notAttempted) + // The same order as the returned error, so the reason matches it. switch { - case cancelled: - b.WriteString("\nStopped: the copy was cancelled.") case r.fatalErr != nil: b.WriteString("\nStopped early because of the error shown below.") + case cancelled: + b.WriteString("\nStopped: the copy was cancelled.") + case r.stalled: + fmt.Fprintf(&b, "\nStopped: files kept failing and none was copied for %s, so no new files were started.", formatDuration(r.stallTimeout)) } if len(r.failures) > 0 { b.WriteString("\nFailed:") @@ -399,6 +489,8 @@ func (r *recursiveCopy) finish(ctx context.Context) error { return r.fatalErr case cancelled: return fmt.Errorf("copy cancelled: %w", ctx.Err()) + case r.stalled: + return fmt.Errorf("copy stopped: files kept failing and none was copied for %s", formatDuration(r.stallTimeout)) } if notCopied == 1 { return errors.New("1 file was not copied") @@ -406,6 +498,18 @@ func (r *recursiveCopy) finish(ctx context.Context) error { return fmt.Errorf("%d files were not copied", notCopied) } +// formatDuration writes d as "5 minutes" or "2 seconds" when it is a whole +// number of them. +func formatDuration(d time.Duration) string { + switch { + case d >= time.Minute && d%time.Minute == 0: + return plural(int(d/time.Minute), "minute") + case d >= time.Second && d%time.Second == 0: + return plural(int(d/time.Second), "second") + } + return d.String() +} + func plural(n int, unit string) string { if n == 1 { return "1 " + unit diff --git a/cmd/fs/cp_recursive_test.go b/cmd/fs/cp_recursive_test.go index 214ca5e7a05..d79a714034f 100644 --- a/cmd/fs/cp_recursive_test.go +++ b/cmd/fs/cp_recursive_test.go @@ -18,6 +18,7 @@ import ( "time" "github.com/databricks/cli/libs/cmdio" + "github.com/databricks/cli/libs/env" "github.com/databricks/cli/libs/fakefs" "github.com/databricks/cli/libs/filer" "github.com/databricks/cli/libs/flags" @@ -147,7 +148,8 @@ type copyResult struct { } // runCopy copies "src" to "dst", capturing stdout, stderr and the debug log. -func runCopy(t *testing.T, ctx context.Context, c *copy, stdout io.Writer) copyResult { +// opts adjust the copy before it runs. +func runCopy(t *testing.T, ctx context.Context, c *copy, stdout io.Writer, opts ...func(*recursiveCopy)) copyResult { var out, stderr, logs bytes.Buffer if stdout == nil { stdout = &out @@ -158,7 +160,12 @@ func runCopy(t *testing.T, ctx context.Context, c *copy, stdout io.Writer) copyR } ctx = cmdio.InContext(ctx, cmdio.NewIO(ctx, format, strings.NewReader(""), stdout, &stderr, "", "")) ctx = log.NewContext(ctx, slog.New(slog.NewTextHandler(&logs, &slog.HandlerOptions{Level: slog.LevelDebug}))) - r := &recursiveCopy{c: c} + // Ignore any override in the environment: tests assume the defaults. + ctx = env.Set(ctx, stallTimeoutEnvVar, "") + r := newRecursiveCopy(ctx, c) + for _, opt := range opts { + opt(r) + } done := make(chan error, 1) go func() { done <- r.run(ctx, "src", "dst") }() @@ -431,3 +438,246 @@ func orDefault(v, def int) int { } return v } + +// flakySource is a source filer whose ReadDir of dir returns what readDir +// returns for the attempt, counting from 1. +type flakySource struct { + *filer.FakeFiler + dir string + readDir func(attempt int) error + attempts int +} + +func (f *flakySource) ReadDir(ctx context.Context, name string) ([]fs.DirEntry, error) { + if name == f.dir { + f.attempts++ + if err := f.readDir(f.attempts); err != nil { + return nil, err + } + } + return f.FakeFiler.ReadDir(ctx, name) +} + +// fakeClock is a clock that only moves when the test advances it. +type fakeClock struct { + mu sync.Mutex + t time.Time +} + +func (c *fakeClock) Now() time.Time { + c.mu.Lock() + defer c.mu.Unlock() + return c.t +} + +func (c *fakeClock) Advance(d time.Duration) { + c.mu.Lock() + defer c.mu.Unlock() + c.t = c.t.Add(d) +} + +func TestRecursiveCopyStall(t *testing.T) { + unavailable := apiError(503, nil, "") + stalledSix := func(notAttempted string) string { + var b strings.Builder + fmt.Fprintf(&b, "Copy incomplete: 0 copied, 0 skipped (already existed), 6 failed, %s not attempted.\n"+ + "Stopped: files kept failing and none was copied for 5 minutes, so no new files were started.\nFailed:\n", notAttempted) + for _, f := range names(6) { + fmt.Fprintf(&b, " src/%s -> dst/%s: HTTP 503 Service Unavailable\n", f, f) + } + return b.String() + } + + tests := []struct { + name string + files []string + failListing string + listTime time.Duration + mkdirTime time.Duration + writeTime time.Duration + fail func(p string, attempt int) bool + wantErr string + wantStderr string + }{ + { + // Each write fails after a minute. f00 fails at 1 minute, so the + // timeout runs to 6 minutes: f00 to f05 fail and are deferred, + // then no new files start, and the deferred files keep their + // errors. + name: "an outage stops the copy", + files: names(20), + writeTime: time.Minute, + fail: func(string, int) bool { return true }, + wantErr: "copy stopped: files kept failing and none was copied for 5 minutes", + wantStderr: stalledSix("14"), + }, + { + // While the stalled copy only counts the files left, a failed + // listing ends the count but is not why the copy stopped. + name: "a listing error while counting keeps the stall as the reason", + files: append(names(20), "z/x.txt"), + failListing: "src/z", + writeTime: time.Minute, + fail: func(string, int) bool { return true }, + wantErr: "copy stopped: files kept failing and none was copied for 5 minutes", + wantStderr: stalledSix("at least 14"), + }, + { + // Writes take 2 minutes and every other one fails on its first + // attempt. The copy takes 12 minutes, but the gap between + // successes stays under 5. + name: "a success resets the timer", + files: names(4), + writeTime: 2 * time.Minute, + fail: func(p string, attempt int) bool { + return attempt == 1 && (p == "dst/f01.txt" || p == "dst/f03.txt") + }, + wantStderr: "Copied 4 files.\n", + }, + { + // Each Mkdir takes 6 minutes. a.txt and b.txt fail on their first + // attempt (with one worker, b.txt starts only after a.txt failed), + // so d1/c.txt starts 6 minutes after the first failure. Creating + // d1 in between counts as progress. + name: "slow directories count as progress", + files: []string{"a.txt", "b.txt", "d1/c.txt"}, + mkdirTime: 6 * time.Minute, + fail: func(p string, attempt int) bool { + return attempt == 1 && (p == "dst/a.txt" || p == "dst/b.txt") + }, + wantStderr: "Copied 3 files.\n", + }, + { + // Listing sub and each write take 6 minutes, with no failures. + name: "a slow copy with no failures never stalls", + files: []string{"sub/a.txt", "sub/b.txt"}, + listTime: 6 * time.Minute, + writeTime: 6 * time.Minute, + wantStderr: "Copied 2 files.\n", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + clock := &fakeClock{} + var startedLate []string + target := newScriptedTarget(t, cancel, func(ctx context.Context, p string, attempt int) error { + // In the stalling rows the first write fails after writeTime, + // and the timeout runs from that failure. + if clock.Now().Sub(time.Time{}) >= tt.writeTime+stallTimeout { + startedLate = append(startedLate, p) + } + clock.Advance(tt.writeTime) + if tt.fail != nil && tt.fail(p, attempt) { + return unavailable + } + return nil + }) + target.mkdir = func(context.Context, string) error { + clock.Advance(tt.mkdirTime) + return nil + } + var source filer.Filer = testSource(tt.files...) + switch { + case tt.failListing != "": + source = &flakySource{FakeFiler: testSource(tt.files...), dir: tt.failListing, readDir: func(int) error { return unavailable }} + case tt.listTime > 0: + source = &flakySource{FakeFiler: testSource(tt.files...), dir: "src/sub", readDir: func(int) error { + clock.Advance(tt.listTime) + return nil + }} + } + c := ©{recursive: true, concurrency: 1, sourceFiler: source, targetFiler: target} + res := runCopy(t, ctx, c, nil, func(r *recursiveCopy) { r.now = clock.Now }) + + if tt.wantErr == "" { + assert.NoError(t, res.err) + assert.NotContains(t, res.log, "starting no new files") + } else { + assert.EqualError(t, res.err, tt.wantErr) + assert.Contains(t, res.log, "level=INFO msg=\"fs cp: files kept failing and none was copied for 5 minutes; starting no new files\"") + assert.Empty(t, startedLate, "no file starts after the stall timeout") + } + assert.Equal(t, tt.wantStderr, res.stderr) + }) + } +} + +func TestPermanentErrorsDoNotStall(t *testing.T) { + // f00.txt fails with a 404, then listing sub takes 6 minutes with nothing + // completing. A permanent error is not an outage, so sub/a.txt still runs. + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + // A real clock is never at the zero time, which failingSince uses to mean + // "not failing". + clock := &fakeClock{t: time.Time{}.Add(time.Hour)} + target := newScriptedTarget(t, cancel, func(ctx context.Context, p string, attempt int) error { + if p == "dst/f00.txt" { + return apiError(404, nil, `{"message": "gone"}`) + } + return nil + }) + var r *recursiveCopy + source := &flakySource{FakeFiler: testSource("f00.txt", "sub/a.txt"), dir: "src/sub", readDir: func(int) error { + // The 404 is recorded on another goroutine; wait for it, so it comes + // before the slow listing. + for { + r.mu.Lock() + n := len(r.failures) + r.mu.Unlock() + if n > 0 { + break + } + time.Sleep(time.Millisecond) + } + clock.Advance(6 * time.Minute) + return nil + }} + c := ©{recursive: true, concurrency: 1, sourceFiler: source, targetFiler: target} + res := runCopy(t, ctx, c, nil, func(rc *recursiveCopy) { + rc.now = clock.Now + r = rc + }) + + assert.EqualError(t, res.err, "1 file was not copied") + assert.Equal(t, 1, res.r.copied) + assert.NotContains(t, res.log, "starting no new files") +} + +func TestStallTimesTheFailingPeriod(t *testing.T) { + // The last success was at 0, and a long healthy stretch with nothing + // completing ends in a failure at 6 minutes. That alone must not stall the + // copy: the timeout runs from the failure. + clock := &fakeClock{} + r := newRecursiveCopy(t.Context(), ©{}) + r.now = clock.Now + r.lastSuccess = clock.Now() + assert.False(t, r.stopping(t.Context()), "no failure, no stall") + + clock.Advance(6 * time.Minute) + r.failingSince = clock.Now() + assert.False(t, r.stopping(t.Context()), "a failure just now") + + clock.Advance(5 * time.Minute) + assert.True(t, r.stopping(t.Context()), "failing for the stall timeout") +} + +func TestStallTimeoutFromEnv(t *testing.T) { + for value, want := range map[string]time.Duration{ + "": stallTimeout, + "1500": 1500 * time.Millisecond, + "0": stallTimeout, + "5m": stallTimeout, + } { + ctx := env.Set(t.Context(), stallTimeoutEnvVar, value) + assert.Equal(t, want, newRecursiveCopy(ctx, ©{}).stallTimeout, "value %q", value) + } +} + +func TestFormatDuration(t *testing.T) { + assert.Equal(t, "5 minutes", formatDuration(5*time.Minute)) + assert.Equal(t, "1 minute", formatDuration(time.Minute)) + assert.Equal(t, "90 seconds", formatDuration(90*time.Second)) + assert.Equal(t, "1.5s", formatDuration(1500*time.Millisecond)) +}