From 53cc5561f447970190dc7184c04f0b3a17db4573 Mon Sep 17 00:00:00 2001 From: "denis.bilenko" Date: Wed, 18 Nov 2026 08:22:35 +0000 Subject: [PATCH 1/2] Replace a stale deploy lock left by a timed-out lock write When a lock write fails without a definite answer (timeout, 5xx, cut connection), the server may apply it after the command released the lock. Record such lock IDs locally and, when the next Lock finds a lock with one of them, delete it and create the lock again. aws: TestAccept/bundle/resources/secrets/basic TestAccept/bundle/select/basic Co-authored-by: Isaac --- .../bundles/stale-lock-after-timeout.md | 1 + bundle/deploy/lock/acquire.go | 2 + libs/locker/locker.go | 132 ++++++++++++++++-- libs/locker/locker_test.go | 98 +++++++++++++ 4 files changed, 221 insertions(+), 12 deletions(-) create mode 100644 .nextchanges/bundles/stale-lock-after-timeout.md diff --git a/.nextchanges/bundles/stale-lock-after-timeout.md b/.nextchanges/bundles/stale-lock-after-timeout.md new file mode 100644 index 00000000000..d877a9baf7e --- /dev/null +++ b/.nextchanges/bundles/stale-lock-after-timeout.md @@ -0,0 +1 @@ +* Replace a stale deploy lock that a timed-out lock write left behind after this checkout released the lock, instead of failing the next deploy with "deploy lock acquired by ...". diff --git a/bundle/deploy/lock/acquire.go b/bundle/deploy/lock/acquire.go index fab822ac2e4..627ffd84e20 100644 --- a/bundle/deploy/lock/acquire.go +++ b/bundle/deploy/lock/acquire.go @@ -4,6 +4,7 @@ import ( "context" "errors" "io/fs" + "path/filepath" "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/config" @@ -39,6 +40,7 @@ func (m *acquire) init(ctx context.Context, b *bundle.Bundle) error { return err } + l.ReleasedLocksPath = filepath.Join(b.GetLocalStateDir(ctx), "released-deploy-locks.json") b.Locker = l return nil } diff --git a/libs/locker/locker.go b/libs/locker/locker.go index 95b9f86a233..72ff1ddc766 100644 --- a/libs/locker/locker.go +++ b/libs/locker/locker.go @@ -8,11 +8,14 @@ import ( "fmt" "io" "io/fs" + "os" + "path/filepath" "slices" "time" "uuid" "github.com/databricks/cli/libs/filer" + "github.com/databricks/cli/libs/log" "github.com/databricks/cli/libs/retry" "github.com/databricks/databricks-sdk-go" ) @@ -51,6 +54,20 @@ const LockFileName = "deploy.lock" type Locker struct { filer filer.Filer + // lockFiler writes the lock file. Unlike filer it does not retry writes: Lock retries them + // itself, so that it knows when an attempt failed in a way that may still be applied later. + lockFiler filer.Filer + + // createUncertain is set when a lock write failed without a definite answer from the server + // (a timeout, a 5xx, a cut connection, a cancelled request). The server may still apply such a + // write after we release the lock, leaving behind a lock with our ID. + createUncertain bool + + // ReleasedLocksPath is a local file listing the IDs of locks released after an uncertain + // write (see createUncertain). A lock that carries one of these IDs is stale: its owner has + // already released it. Empty disables the list. + ReleasedLocksPath string + // scope of the locker TargetDir string // Active == true implies exclusive access to TargetDir for the client. @@ -141,6 +158,15 @@ func (locker *Locker) Read(ctx context.Context, path string) (io.ReadCloser, err } func (locker *Locker) Lock(ctx context.Context, isForced bool) error { + err := locker.lock(ctx, isForced) + if err != nil && locker.createUncertain { + // We do not hold the lock, but a write that timed out may still create it later. + locker.recordReleased(ctx) + } + return err +} + +func (locker *Locker) lock(ctx context.Context, isForced bool) error { newLockerState := LockState{ ID: locker.State.ID, AcquisitionTime: time.Now(), @@ -152,19 +178,29 @@ func (locker *Locker) Lock(ctx context.Context, isForced bool) error { return err } + // Repeating a create that does not overwrite is safe because the lock file carries our unique + // ID: if an attempt that failed with a transient error did create the file, the retry fails + // with fs.ErrExist and assertLockHeld below tells whether the lock is ours. That holds for a + // connection cut mid-response ("unexpected EOF") too, so this uses the wider check meant for + // reads. Forced writes overwrite and are idempotent. + mode := []filer.WriteMode{filer.CreateParentDirectories} if isForced { - // The retrying filer already retries overwrite writes, on the errors [retry.IsTransient] - // covers; an "unexpected EOF" is not retried there, only read back below. - err = locker.filer.Write(ctx, LockFileName, bytes.NewReader(buf), filer.CreateParentDirectories, filer.OverwriteIfExists) - } else { - // The retrying filer does not repeat writes that do not overwrite. Repeating this one is - // safe because the lock file carries our unique ID: if an attempt that failed with a - // transient error did create the file, the retry fails with fs.ErrExist and - // assertLockHeld below tells whether the lock is ours. That holds for a connection cut - // mid-response ("unexpected EOF") too, so this uses the wider check meant for reads. - _, err = retry.With(ctx, retry.IsTransientRead, func() (struct{}, error) { - return struct{}{}, locker.filer.Write(ctx, LockFileName, bytes.NewReader(buf), filer.CreateParentDirectories) + mode = append(mode, filer.OverwriteIfExists) + } + write := func() error { + _, err := retry.With(ctx, retry.IsTransientRead, func() (struct{}, error) { + err := locker.lockFiler.Write(ctx, LockFileName, bytes.NewReader(buf), mode...) + if err != nil && !errors.Is(err, fs.ErrExist) && !errors.Is(err, fs.ErrPermission) && !errors.Is(err, fs.ErrNotExist) { + locker.createUncertain = true + } + return struct{}{}, err }) + return err + } + + err = write() + if errors.Is(err, fs.ErrExist) && locker.removeStaleLock(ctx) { + err = write() } switch { @@ -203,6 +239,10 @@ func (locker *Locker) Unlock(ctx context.Context, opts ...UnlockOption) error { if !locker.Active { return errors.New("unlock called when lock is not held") } + if locker.createUncertain { + // Recorded even if the unlock fails: a lock with our ID left behind is stale either way. + defer locker.recordReleased(ctx) + } // if allowLockFileNotExist is set, do not throw an error if the lock file does // not exist. This is helpful when destroying a bundle in which case the lock @@ -224,6 +264,73 @@ func (locker *Locker) Unlock(ctx context.Context, opts ...UnlockOption) error { return nil } +// maxReleasedLocks bounds the list in ReleasedLocksPath. A late write lands within minutes, so +// only the most recent entries matter. +const maxReleasedLocks = 20 + +func (locker *Locker) readReleased(ctx context.Context) []uuid.UUID { + if locker.ReleasedLocksPath == "" { + return nil + } + data, err := os.ReadFile(locker.ReleasedLocksPath) + var ids []uuid.UUID + if err == nil { + err = json.Unmarshal(data, &ids) + } + if err != nil && !errors.Is(err, fs.ErrNotExist) { + log.Warnf(ctx, "Failed to read released deploy locks: %v", err) + } + return ids +} + +// recordReleased adds our ID to ReleasedLocksPath. Failing to record it only means that a late +// write, if one lands, needs --force-lock to clear, so errors are logged and not returned. +func (locker *Locker) recordReleased(ctx context.Context) { + if locker.ReleasedLocksPath == "" { + return + } + ids := append(locker.readReleased(ctx), locker.State.ID) + if len(ids) > maxReleasedLocks { + ids = ids[len(ids)-maxReleasedLocks:] + } + data, err := json.Marshal(ids) + if err == nil { + err = os.MkdirAll(filepath.Dir(locker.ReleasedLocksPath), 0o700) + } + if err == nil { + // Write to a temporary file and rename it, so that a concurrent reader never sees a + // partial list. + tmp := locker.ReleasedLocksPath + "." + uuid.New().String() + ".tmp" + err = os.WriteFile(tmp, data, 0o600) + if err == nil { + err = os.Rename(tmp, locker.ReleasedLocksPath) + } + if err != nil { + _ = os.Remove(tmp) + } + } + if err != nil { + log.Warnf(ctx, "Failed to record released deploy lock: %v", err) + } +} + +// removeStaleLock deletes the remote lock if it carries an ID from ReleasedLocksPath: a write +// that timed out in an earlier command was applied after that command released the lock. It +// reports whether it deleted the lock. A --force-lock from another client between the read and +// the delete loses its lock here; forced locks already give up exclusive access. +func (locker *Locker) removeStaleLock(ctx context.Context) bool { + released := locker.readReleased(ctx) + if len(released) == 0 { + return false + } + activeLockState, err := locker.GetActiveLockState(ctx) + if err != nil || !slices.Contains(released, activeLockState.ID) { + return false + } + log.Infof(ctx, "Removing stale deploy lock %s acquired at %v: it was released before", activeLockState.ID, activeLockState.AcquisitionTime) + return locker.filer.Delete(ctx, LockFileName) == nil +} + func CreateLocker(user, targetDir string, w *databricks.WorkspaceClient) (*Locker, error) { filer, err := filer.NewWorkspaceFilesClient(w, targetDir) if err != nil { @@ -231,7 +338,8 @@ func CreateLocker(user, targetDir string, w *databricks.WorkspaceClient) (*Locke } locker := &Locker{ - filer: retry.Filer(filer), + filer: retry.Filer(filer), + lockFiler: filer, TargetDir: targetDir, Active: false, diff --git a/libs/locker/locker_test.go b/libs/locker/locker_test.go index c5bb8068840..05e2ba12ff5 100644 --- a/libs/locker/locker_test.go +++ b/libs/locker/locker_test.go @@ -1,9 +1,12 @@ package locker import ( + "bytes" "context" + "errors" "io" "net/url" + "path/filepath" "testing" "uuid" @@ -40,6 +43,7 @@ func (f *eofFiler) Write(ctx context.Context, path string, reader io.Reader, mod func newTestLocker(t *testing.T, f filer.Filer) *Locker { return &Locker{ filer: f, + lockFiler: f, TargetDir: t.TempDir(), State: &LockState{ID: uuid.New(), User: "tester"}, } @@ -86,3 +90,97 @@ func TestLockFailsWhenEveryAttemptHitsUnexpectedEOF(t *testing.T) { assert.False(t, locker.Active) assert.Equal(t, 3, f.writes) } + +// lateFiler fails the first `failures` writes with the SDK's inactivity timeout without applying +// them. applyLate then applies the first of them, as the server does with a request it was still +// processing. +type lateFiler struct { + filer.Filer + failures int + writes int + stalled []byte +} + +func (f *lateFiler) Write(ctx context.Context, path string, reader io.Reader, mode ...filer.WriteMode) error { + f.writes++ + if f.writes > f.failures { + return f.Filer.Write(ctx, path, reader, mode...) + } + if f.stalled == nil { + body, err := io.ReadAll(reader) + if err != nil { + return err + } + f.stalled = body + } + return &url.Error{Op: "Post", URL: "https://example.com/" + path, Err: errors.New("request timed out after 1m30s of inactivity")} +} + +func (f *lateFiler) applyLate(t *testing.T) { + require.NoError(t, f.Filer.Write(t.Context(), LockFileName, bytes.NewReader(f.stalled))) +} + +func TestLockReplacesLockReleasedBeforeLateWrite(t *testing.T) { + ctx := env.Set(t.Context(), retry.IntervalMsVariable, "1") + local, err := filer.NewLocalClient(t.TempDir()) + require.NoError(t, err) + released := filepath.Join(t.TempDir(), "released.json") + + f := &lateFiler{Filer: local, failures: 1} + first := newTestLocker(t, f) + first.ReleasedLocksPath = released + require.NoError(t, first.Lock(ctx, false)) + require.NoError(t, first.Unlock(ctx)) + f.applyLate(t) + + second := newTestLocker(t, local) + second.ReleasedLocksPath = released + require.NoError(t, second.Lock(ctx, false)) + state, err := second.GetActiveLockState(ctx) + require.NoError(t, err) + assert.Equal(t, second.State.ID, state.ID) +} + +func TestLockReplacesLockLeftByFailedLock(t *testing.T) { + ctx := env.Set(t.Context(), retry.IntervalMsVariable, "1") + local, err := filer.NewLocalClient(t.TempDir()) + require.NoError(t, err) + released := filepath.Join(t.TempDir(), "released.json") + + // Every attempt times out; the first is applied after Lock gave up. + f := &lateFiler{Filer: local, failures: 3} + first := newTestLocker(t, f) + first.ReleasedLocksPath = released + require.Error(t, first.Lock(ctx, false)) + f.applyLate(t) + + second := newTestLocker(t, local) + second.ReleasedLocksPath = released + require.NoError(t, second.Lock(ctx, false)) +} + +func TestLockKeepsLockNotReleasedHere(t *testing.T) { + ctx := env.Set(t.Context(), retry.IntervalMsVariable, "1") + local, err := filer.NewLocalClient(t.TempDir()) + require.NoError(t, err) + released := filepath.Join(t.TempDir(), "released.json") + + // A lock released after an uncertain write is recorded, but a different, live lock is kept. + f := &lateFiler{Filer: local, failures: 1} + first := newTestLocker(t, f) + first.ReleasedLocksPath = released + require.NoError(t, first.Lock(ctx, false)) + require.NoError(t, first.Unlock(ctx)) + + other := newTestLocker(t, local) + require.NoError(t, other.Lock(ctx, false)) + + second := newTestLocker(t, local) + second.ReleasedLocksPath = released + err = second.Lock(ctx, false) + require.ErrorContains(t, err, "deploy lock acquired by tester") + assert.True(t, other.Active) + state, err := second.GetActiveLockState(ctx) + require.NoError(t, err) + assert.Equal(t, other.State.ID, state.ID) +} From adaa1fef4f0a6130b88ed18cfc744616165d6118 Mon Sep 17 00:00:00 2001 From: "denis.bilenko" Date: Wed, 18 Nov 2026 08:23:00 +0000 Subject: [PATCH 2/2] Add the PR link to the changelog entry aws: TestAccept/bundle/resources/secrets/basic TestAccept/bundle/select/basic Co-authored-by: Isaac --- .nextchanges/bundles/stale-lock-after-timeout.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.nextchanges/bundles/stale-lock-after-timeout.md b/.nextchanges/bundles/stale-lock-after-timeout.md index d877a9baf7e..a309452c05f 100644 --- a/.nextchanges/bundles/stale-lock-after-timeout.md +++ b/.nextchanges/bundles/stale-lock-after-timeout.md @@ -1 +1 @@ -* Replace a stale deploy lock that a timed-out lock write left behind after this checkout released the lock, instead of failing the next deploy with "deploy lock acquired by ...". +* Replace a stale deploy lock that a timed-out lock write left behind after this checkout released the lock, instead of failing the next deploy with "deploy lock acquired by ...". ([#7753](https://github.com/databricks/cli/pull/7753))