From 67a7ab680e110993302b6487c4a5cad813898ff9 Mon Sep 17 00:00:00 2001 From: "denis.bilenko" Date: Fri, 13 Nov 2026 16:12:57 +0000 Subject: [PATCH 1/7] direct: retry app update and delete while app compute is UPDATING The Apps API rejects update and delete with 400 BAD_REQUEST while the app compute is in a transitional state, and the compute can enter UPDATING on its own shortly after a create or update returned. On such a 400, wait for the compute to settle and retry. Co-authored-by: Isaac --- bundle/direct/dresources/app.go | 70 ++++++++++++++++++++- bundle/direct/dresources/app_test.go | 91 ++++++++++++++++++++++++++++ 2 files changed, 158 insertions(+), 3 deletions(-) diff --git a/bundle/direct/dresources/app.go b/bundle/direct/dresources/app.go index f76681feb98..f9cd6634405 100644 --- a/bundle/direct/dresources/app.go +++ b/bundle/direct/dresources/app.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "net/http" "slices" "strings" "time" @@ -204,7 +205,12 @@ func (r *ResourceApp) doUpdate(ctx context.Context, id string, config *AppState, AppName: id, UpdateMask: appUpdateMask(&app), } - updateWaiter, err := r.client.Apps.CreateUpdate(ctx, request) + var updateWaiter *apps.WaitGetUpdateAppSucceeded[apps.AppUpdate] + err := r.retryWhileComputeSettles(ctx, id, func() error { + var err error + updateWaiter, err = r.client.Apps.CreateUpdate(ctx, request) + return err + }) if err != nil { return err } @@ -343,8 +349,66 @@ func deploymentToAppConfig(d *apps.AppDeployment) *resources.AppConfig { } func (r *ResourceApp) DoDelete(ctx context.Context, id string, _ *AppState) error { - _, err := r.client.Apps.DeleteByName(ctx, id) - return err + return r.retryWhileComputeSettles(ctx, id, func() error { + _, err := r.client.Apps.DeleteByName(ctx, id) + return err + }) +} + +// maxComputeSettleRetries bounds how often retryWhileComputeSettles repeats a call, so a +// compute that keeps going back into a transitional state cannot loop forever. +const maxComputeSettleRetries = 3 + +// retryWhileComputeSettles calls fn and, if it fails with 400 BAD_REQUEST while the app +// compute is in a transitional state, waits for the compute to settle and calls fn again. +// The Apps API rejects update and delete while the compute is UPDATING ("App compute needs +// to be ACTIVE or STOPPED to update", "not terminal with state UPDATING"), and the compute +// can enter UPDATING on its own shortly after a create or update returned. Checking the +// state instead of the error text keeps other 400s failing at once. +func (r *ResourceApp) retryWhileComputeSettles(ctx context.Context, name string, fn func() error) error { + for attempt := 0; ; attempt++ { + err := fn() + apiErr, ok := errors.AsType[*apierr.APIError](err) + if !ok || apiErr.StatusCode != http.StatusBadRequest || attempt >= maxComputeSettleRetries { + return err + } + waited, waitErr := r.waitForComputeToSettle(ctx, name) + if waitErr != nil || !waited { + return err + } + } +} + +func isComputeTransitional(app *apps.App) bool { + if app.ComputeStatus == nil { + return false + } + switch app.ComputeStatus.State { + case apps.ComputeStateUpdating, apps.ComputeStateStarting, apps.ComputeStateStopping: + return true + default: + return false + } +} + +// waitForComputeToSettle polls the app until its compute leaves a transitional state. It +// reports whether it saw a transitional state at all, so a caller can tell a failure caused +// by the compute state from any other one. +func (r *ResourceApp) waitForComputeToSettle(ctx context.Context, name string) (bool, error) { + waited := false + retrier := retries.New[apps.App](retries.WithTimeout(-1), retries.WithRetryFunc(shouldRetry)) + _, err := retrier.Run(ctx, func(ctx context.Context) (*apps.App, error) { + app, err := r.client.Apps.GetByName(ctx, name) + if err != nil { + return nil, retries.Halt(err) + } + if isComputeTransitional(app) { + waited = true + return nil, retries.Continues(string(app.ComputeStatus.State)) + } + return app, nil + }) + return waited, err } // IsGone treats a DELETING app as already-deleted. The Apps DELETE is diff --git a/bundle/direct/dresources/app_test.go b/bundle/direct/dresources/app_test.go index 01b619bae21..4cc291c47f9 100644 --- a/bundle/direct/dresources/app_test.go +++ b/bundle/direct/dresources/app_test.go @@ -248,3 +248,94 @@ func TestWithMissingServicePrincipalHint(t *testing.T) { assert.Contains(t, got.Error(), `delete it with "databricks apps delete myapp"`) } } + +func TestAppDoDelete_RetriesWhileComputeSettles(t *testing.T) { + tests := []struct { + name string + // computeStates are the states GET returns in order; the last one repeats. + computeStates []apps.ComputeState + wantDeletes int + wantErr bool + }{ + { + name: "retries after compute leaves UPDATING", + computeStates: []apps.ComputeState{apps.ComputeStateUpdating, apps.ComputeStateActive}, + wantDeletes: 2, + }, + { + name: "fails at once when compute is not transitional", + computeStates: []apps.ComputeState{apps.ComputeStateActive}, + wantDeletes: 1, + wantErr: true, + }, + { + name: "fails at once when app is already deleting", + computeStates: []apps.ComputeState{apps.ComputeStateDeleting}, + wantDeletes: 1, + wantErr: true, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + server := testserver.New(t) + deletes := 0 + gets := 0 + + server.Handle("DELETE", "/api/2.0/apps/{name}", func(req testserver.Request) any { + deletes++ + if deletes == 1 { + return testserver.Response{ + StatusCode: 400, + Body: map[string]string{ + "error_code": "BAD_REQUEST", + "message": "Cannot delete app test-app as it is not terminal with state UPDATING, and was updated less than 20 minutes ago. Please wait before trying again.", + }, + } + } + return apps.App{Name: req.Vars["name"]} + }) + server.Handle("GET", "/api/2.0/apps/{name}", func(req testserver.Request) any { + state := tc.computeStates[min(gets, len(tc.computeStates)-1)] + gets++ + return apps.App{Name: req.Vars["name"], ComputeStatus: &apps.ComputeStatus{State: state}} + }) + testserver.AddDefaultHandlers(server) + + client, err := databricks.NewWorkspaceClient(&databricks.Config{Host: server.URL, Token: "testtoken"}) + require.NoError(t, err) + + r := (&ResourceApp{}).New(client) + err = r.DoDelete(t.Context(), "test-app", nil) + if tc.wantErr { + require.ErrorContains(t, err, "not terminal with state UPDATING") + } else { + require.NoError(t, err) + } + assert.Equal(t, tc.wantDeletes, deletes) + }) + } +} + +func TestAppRetryWhileComputeSettles_IgnoresNonBadRequest(t *testing.T) { + server := testserver.New(t) + gets := 0 + server.Handle("GET", "/api/2.0/apps/{name}", func(req testserver.Request) any { + gets++ + return apps.App{Name: req.Vars["name"], ComputeStatus: &apps.ComputeStatus{State: apps.ComputeStateUpdating}} + }) + testserver.AddDefaultHandlers(server) + + client, err := databricks.NewWorkspaceClient(&databricks.Config{Host: server.URL, Token: "testtoken"}) + require.NoError(t, err) + + r := (&ResourceApp{}).New(client) + calls := 0 + err = r.retryWhileComputeSettles(t.Context(), "test-app", func() error { + calls++ + return &apierr.APIError{StatusCode: 500, ErrorCode: "INTERNAL_ERROR", Message: "boom"} + }) + require.ErrorContains(t, err, "boom") + assert.Equal(t, 1, calls) + assert.Equal(t, 0, gets) +} From f81ba10f2553e51f36c4c6092195631e9fa08709 Mon Sep 17 00:00:00 2001 From: "denis.bilenko" Date: Fri, 13 Nov 2026 16:14:15 +0000 Subject: [PATCH 2/7] Add changelog fragment cloud: TestAccept/bundle/apps/job_permissions TestAccept/bundle/invariant/no_drift/DMS=true/INPUT_CONFIG=app.yml.tmpl Co-authored-by: Isaac --- .nextchanges/bundles/app-compute-settle.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 .nextchanges/bundles/app-compute-settle.md diff --git a/.nextchanges/bundles/app-compute-settle.md b/.nextchanges/bundles/app-compute-settle.md new file mode 100644 index 00000000000..57322262a6e --- /dev/null +++ b/.nextchanges/bundles/app-compute-settle.md @@ -0,0 +1 @@ +* Retry app updates and deletes in the direct deployment engine while the app compute is `UPDATING`, `STARTING` or `STOPPING`, instead of failing with 400 BAD_REQUEST. ([#7606](https://github.com/databricks/cli/pull/7606)) From 489b214dfa3124fc751bd1280b2fa3b246c536bd Mon Sep 17 00:00:00 2001 From: "denis.bilenko" Date: Fri, 13 Nov 2026 16:23:28 +0000 Subject: [PATCH 3/7] Address review: match the compute-state error, bound the wait, refresh started state Retry on the specific 400 messages even if the compute settled before the first read, use retry.With for the attempt limit and pause, cap each wait at 20 minutes, treat an app gone while waiting as deleted, and refresh the started state used by lifecycle after waiting. Add a DoUpdate test. cloud: TestAccept/bundle/apps/job_permissions TestAccept/bundle/invariant/no_drift/DMS=true/INPUT_CONFIG=app.yml.tmpl Co-authored-by: Isaac --- bundle/direct/dresources/app.go | 94 ++++++++++++------- bundle/direct/dresources/app_test.go | 132 ++++++++++++++++++--------- 2 files changed, 147 insertions(+), 79 deletions(-) diff --git a/bundle/direct/dresources/app.go b/bundle/direct/dresources/app.go index f9cd6634405..86964a8ab2a 100644 --- a/bundle/direct/dresources/app.go +++ b/bundle/direct/dresources/app.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "net/http" + "regexp" "slices" "strings" "time" @@ -12,6 +13,7 @@ import ( "github.com/databricks/cli/bundle/appdeploy" "github.com/databricks/cli/bundle/config/resources" "github.com/databricks/cli/bundle/deployplan" + "github.com/databricks/cli/libs/retry" "github.com/databricks/cli/libs/structs/structpath" "github.com/databricks/databricks-sdk-go" "github.com/databricks/databricks-sdk-go/apierr" @@ -196,6 +198,7 @@ func (r *ResourceApp) DoUpdate(ctx context.Context, id string, config *AppState, } func (r *ResourceApp) doUpdate(ctx context.Context, id string, config *AppState, entry *PlanEntry) error { + alreadyStarted := remoteIsStarted(entry) // Deploy-only fields (source_code_path, config, git_source, lifecycle) are excluded // from the request body; see appRequestBody. if hasAppChanges(entry) { @@ -206,7 +209,7 @@ func (r *ResourceApp) doUpdate(ctx context.Context, id string, config *AppState, UpdateMask: appUpdateMask(&app), } var updateWaiter *apps.WaitGetUpdateAppSucceeded[apps.AppUpdate] - err := r.retryWhileComputeSettles(ctx, id, func() error { + settled, err := r.retryWhileComputeSettles(ctx, id, func() error { var err error updateWaiter, err = r.client.Apps.CreateUpdate(ctx, request) return err @@ -214,6 +217,10 @@ func (r *ResourceApp) doUpdate(ctx context.Context, id string, config *AppState, if err != nil { return err } + // The plan-time compute state is stale if the compute moved while we waited. + if settled != nil { + alreadyStarted = !isComputeStopped(settled) + } response, err := updateWaiter.Get() if err != nil { @@ -225,7 +232,7 @@ func (r *ResourceApp) doUpdate(ctx context.Context, id string, config *AppState, } } - return r.manageLifecycle(ctx, id, config, remoteIsStarted(entry)) + return r.manageLifecycle(ctx, id, config, alreadyStarted) } // missingServicePrincipalMessage is part of the backend error for an app whose service @@ -349,34 +356,58 @@ func deploymentToAppConfig(d *apps.AppDeployment) *resources.AppConfig { } func (r *ResourceApp) DoDelete(ctx context.Context, id string, _ *AppState) error { - return r.retryWhileComputeSettles(ctx, id, func() error { + _, err := r.retryWhileComputeSettles(ctx, id, func() error { _, err := r.client.Apps.DeleteByName(ctx, id) return err }) -} - -// maxComputeSettleRetries bounds how often retryWhileComputeSettles repeats a call, so a -// compute that keeps going back into a transitional state cannot loop forever. -const maxComputeSettleRetries = 3 - -// retryWhileComputeSettles calls fn and, if it fails with 400 BAD_REQUEST while the app -// compute is in a transitional state, waits for the compute to settle and calls fn again. -// The Apps API rejects update and delete while the compute is UPDATING ("App compute needs -// to be ACTIVE or STOPPED to update", "not terminal with state UPDATING"), and the compute -// can enter UPDATING on its own shortly after a create or update returned. Checking the -// state instead of the error text keeps other 400s failing at once. -func (r *ResourceApp) retryWhileComputeSettles(ctx context.Context, name string, fn func() error) error { - for attempt := 0; ; attempt++ { - err := fn() - apiErr, ok := errors.AsType[*apierr.APIError](err) - if !ok || apiErr.StatusCode != http.StatusBadRequest || attempt >= maxComputeSettleRetries { - return err + if errors.Is(err, errAppGoneWhileSettling) { + return nil + } + return err +} + +// computeTransitionalStateError matches the 400 BAD_REQUEST messages the Apps API returns for +// an update or delete while the compute is in a transitional state: "Cannot update app X as +// its compute is in UPDATING state" and "Cannot delete app X as it is not terminal with state +// UPDATING". DELETING is left out: a delete of a DELETING app is handled by IsGone and +// deleteConfirmedGone, and waiting would not help. +var computeTransitionalStateError = regexp.MustCompile(`(?:is in|with state) (?:UPDATING|STARTING|STOPPING)\b`) + +func isComputeTransitionalStateError(err error) bool { + apiErr, ok := errors.AsType[*apierr.APIError](err) + return ok && apiErr.StatusCode == http.StatusBadRequest && computeTransitionalStateError.MatchString(apiErr.Message) +} + +// errAppGoneWhileSettling reports that the app disappeared while waiting for its compute. +var errAppGoneWhileSettling = errors.New("app is gone") + +// computeSettleTimeout bounds one wait for the app compute to leave a transitional state. +const computeSettleTimeout = 20 * time.Minute + +// retryWhileComputeSettles calls fn and, if the Apps API rejected it because the compute is in +// a transitional state, waits for the compute to settle and calls fn again (see retry.With for +// the attempt limit). The compute can enter UPDATING on its own shortly after a create or +// update returned. It returns the app as last read while waiting, or nil if it did not wait, +// so a caller can refresh what it knew about the compute state. +func (r *ResourceApp) retryWhileComputeSettles(ctx context.Context, name string, fn func() error) (*apps.App, error) { + var settled *apps.App + var waitErr error + _, err := retry.With(ctx, func(ctx context.Context, err error) bool { + if !isComputeTransitionalStateError(err) { + return false } - waited, waitErr := r.waitForComputeToSettle(ctx, name) - if waitErr != nil || !waited { - return err + settled, waitErr = r.waitForComputeToSettle(ctx, name) + return waitErr == nil + }, func() (struct{}, error) { + return struct{}{}, fn() + }) + if err != nil && waitErr != nil { + if apierr.IsMissing(waitErr) { + return nil, fmt.Errorf("%w: %w", errAppGoneWhileSettling, waitErr) } + return nil, errors.Join(err, waitErr) } + return settled, err } func isComputeTransitional(app *apps.App) bool { @@ -391,24 +422,19 @@ func isComputeTransitional(app *apps.App) bool { } } -// waitForComputeToSettle polls the app until its compute leaves a transitional state. It -// reports whether it saw a transitional state at all, so a caller can tell a failure caused -// by the compute state from any other one. -func (r *ResourceApp) waitForComputeToSettle(ctx context.Context, name string) (bool, error) { - waited := false - retrier := retries.New[apps.App](retries.WithTimeout(-1), retries.WithRetryFunc(shouldRetry)) - _, err := retrier.Run(ctx, func(ctx context.Context) (*apps.App, error) { +// waitForComputeToSettle polls the app until its compute leaves a transitional state. +func (r *ResourceApp) waitForComputeToSettle(ctx context.Context, name string) (*apps.App, error) { + retrier := retries.New[apps.App](retries.WithTimeout(computeSettleTimeout), retries.WithRetryFunc(shouldRetry)) + return retrier.Run(ctx, func(ctx context.Context) (*apps.App, error) { app, err := r.client.Apps.GetByName(ctx, name) if err != nil { return nil, retries.Halt(err) } if isComputeTransitional(app) { - waited = true - return nil, retries.Continues(string(app.ComputeStatus.State)) + return nil, retries.Continues(fmt.Sprintf("app compute is %s", app.ComputeStatus.State)) } return app, nil }) - return waited, err } // IsGone treats a DELETING app as already-deleted. The Apps DELETE is diff --git a/bundle/direct/dresources/app_test.go b/bundle/direct/dresources/app_test.go index 4cc291c47f9..fec5874f471 100644 --- a/bundle/direct/dresources/app_test.go +++ b/bundle/direct/dresources/app_test.go @@ -9,6 +9,8 @@ import ( "strings" "testing" + "github.com/databricks/cli/bundle/deployplan" + "github.com/databricks/cli/libs/retry" "github.com/databricks/cli/libs/testserver" "github.com/databricks/databricks-sdk-go" "github.com/databricks/databricks-sdk-go/apierr" @@ -249,93 +251,133 @@ func TestWithMissingServicePrincipalHint(t *testing.T) { } } +const appComputeUpdatingDeleteMessage = "Cannot delete app test-app as it is not terminal with state UPDATING, " + + "and was updated less than 20 minutes ago. Please wait before trying again." + +// newAppComputeSettleServer serves GET /api/2.0/apps/{name} with the given compute states in +// order (the last one repeats) and counts the calls. +func newAppComputeSettleServer(t *testing.T, gets *int, states ...apps.ComputeState) *testserver.Server { + t.Setenv(retry.IntervalMsVariable, "1") + server := testserver.New(t) + server.Handle("GET", "/api/2.0/apps/{name}", func(req testserver.Request) any { + if len(states) == 0 { + return testserver.Response{ + StatusCode: 404, + Body: map[string]string{"error_code": "RESOURCE_DOES_NOT_EXIST", "message": "App not found."}, + } + } + state := states[min(*gets, len(states)-1)] + *gets++ + return apps.App{Name: req.Vars["name"], ComputeStatus: &apps.ComputeStatus{State: state}} + }) + return server +} + +func newAppForServer(t *testing.T, server *testserver.Server) *ResourceApp { + testserver.AddDefaultHandlers(server) + client, err := databricks.NewWorkspaceClient(&databricks.Config{Host: server.URL, Token: "testtoken"}) + require.NoError(t, err) + return (&ResourceApp{}).New(client) +} + func TestAppDoDelete_RetriesWhileComputeSettles(t *testing.T) { tests := []struct { - name string - // computeStates are the states GET returns in order; the last one repeats. + name string + message string computeStates []apps.ComputeState wantDeletes int - wantErr bool + wantGets int + wantErr string }{ { name: "retries after compute leaves UPDATING", + message: appComputeUpdatingDeleteMessage, computeStates: []apps.ComputeState{apps.ComputeStateUpdating, apps.ComputeStateActive}, wantDeletes: 2, + wantGets: 2, }, { - name: "fails at once when compute is not transitional", + name: "retries when compute settled before the first read", + message: appComputeUpdatingDeleteMessage, computeStates: []apps.ComputeState{apps.ComputeStateActive}, - wantDeletes: 1, - wantErr: true, + wantDeletes: 2, + wantGets: 1, + }, + { + name: "treats an app gone while waiting as deleted", + message: appComputeUpdatingDeleteMessage, + wantDeletes: 1, }, { - name: "fails at once when app is already deleting", + name: "does not retry a delete of a DELETING app", + message: "Cannot delete app test-app as it is not terminal with state DELETING, and was updated less than 20 minutes ago. Please wait before trying again.", computeStates: []apps.ComputeState{apps.ComputeStateDeleting}, wantDeletes: 1, - wantErr: true, + wantErr: "state DELETING", + }, + { + name: "does not retry other bad requests", + message: "Invalid app name.", + computeStates: []apps.ComputeState{apps.ComputeStateUpdating}, + wantDeletes: 1, + wantErr: "Invalid app name.", }, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { - server := testserver.New(t) - deletes := 0 gets := 0 - + server := newAppComputeSettleServer(t, &gets, tc.computeStates...) + deletes := 0 server.Handle("DELETE", "/api/2.0/apps/{name}", func(req testserver.Request) any { deletes++ if deletes == 1 { return testserver.Response{ StatusCode: 400, - Body: map[string]string{ - "error_code": "BAD_REQUEST", - "message": "Cannot delete app test-app as it is not terminal with state UPDATING, and was updated less than 20 minutes ago. Please wait before trying again.", - }, + Body: map[string]string{"error_code": "BAD_REQUEST", "message": tc.message}, } } return apps.App{Name: req.Vars["name"]} }) - server.Handle("GET", "/api/2.0/apps/{name}", func(req testserver.Request) any { - state := tc.computeStates[min(gets, len(tc.computeStates)-1)] - gets++ - return apps.App{Name: req.Vars["name"], ComputeStatus: &apps.ComputeStatus{State: state}} - }) - testserver.AddDefaultHandlers(server) + r := newAppForServer(t, server) - client, err := databricks.NewWorkspaceClient(&databricks.Config{Host: server.URL, Token: "testtoken"}) - require.NoError(t, err) - - r := (&ResourceApp{}).New(client) - err = r.DoDelete(t.Context(), "test-app", nil) - if tc.wantErr { - require.ErrorContains(t, err, "not terminal with state UPDATING") + err := r.DoDelete(t.Context(), "test-app", nil) + if tc.wantErr != "" { + require.ErrorContains(t, err, tc.wantErr) } else { require.NoError(t, err) } assert.Equal(t, tc.wantDeletes, deletes) + assert.Equal(t, tc.wantGets, gets) }) } } -func TestAppRetryWhileComputeSettles_IgnoresNonBadRequest(t *testing.T) { - server := testserver.New(t) +func TestAppDoUpdate_RetriesWhileComputeSettles(t *testing.T) { gets := 0 - server.Handle("GET", "/api/2.0/apps/{name}", func(req testserver.Request) any { - gets++ - return apps.App{Name: req.Vars["name"], ComputeStatus: &apps.ComputeStatus{State: apps.ComputeStateUpdating}} + server := newAppComputeSettleServer(t, &gets, apps.ComputeStateUpdating, apps.ComputeStateActive) + updates := 0 + server.Handle("POST", "/api/2.0/apps/{name}/update", func(req testserver.Request) any { + updates++ + if updates == 1 { + return testserver.Response{ + StatusCode: 400, + Body: map[string]string{ + "error_code": "BAD_REQUEST", + "message": "Cannot update app test-app as its compute is in UPDATING state. App compute needs to be ACTIVE or STOPPED to update.", + }, + } + } + return apps.AppUpdate{Status: &apps.AppUpdateUpdateStatus{State: apps.AppUpdateUpdateStatusUpdateStateSucceeded}} }) - testserver.AddDefaultHandlers(server) + server.Handle("GET", "/api/2.0/apps/{name}/update", func(req testserver.Request) any { + return apps.AppUpdate{Status: &apps.AppUpdateUpdateStatus{State: apps.AppUpdateUpdateStatusUpdateStateSucceeded}} + }) + r := newAppForServer(t, server) - client, err := databricks.NewWorkspaceClient(&databricks.Config{Host: server.URL, Token: "testtoken"}) + entry := &PlanEntry{Changes: deployplan.Changes{"description": {Action: deployplan.Update}}} + _, err := r.DoUpdate(t.Context(), "test-app", &AppState{Name: "test-app", Description: "new"}, entry) require.NoError(t, err) - - r := (&ResourceApp{}).New(client) - calls := 0 - err = r.retryWhileComputeSettles(t.Context(), "test-app", func() error { - calls++ - return &apierr.APIError{StatusCode: 500, ErrorCode: "INTERNAL_ERROR", Message: "boom"} - }) - require.ErrorContains(t, err, "boom") - assert.Equal(t, 1, calls) - assert.Equal(t, 0, gets) + assert.Equal(t, 2, updates) + assert.Equal(t, 2, gets) } From 9bd0bc43392c4c94b243af7acbf9858172b4c2f0 Mon Sep 17 00:00:00 2001 From: "denis.bilenko" Date: Tue, 17 Nov 2026 13:21:05 +0000 Subject: [PATCH 4/7] Address review: narrow to UPDATING, log the wait, test lifecycle and failure paths Match only the two observed UPDATING messages. Warn before waiting so a paused deploy is explained. Keep the API 400 as the classified error when the wait fails. Test the started-state refresh after a wait (start, no stop, no start), retry exhaustion, a failed wait, and an update whose app is gone while waiting. Co-authored-by: Isaac --- .nextchanges/bundles/app-compute-settle.md | 2 +- bundle/direct/dresources/app.go | 47 ++-- bundle/direct/dresources/app_test.go | 284 ++++++++++++++++----- 3 files changed, 246 insertions(+), 87 deletions(-) diff --git a/.nextchanges/bundles/app-compute-settle.md b/.nextchanges/bundles/app-compute-settle.md index 57322262a6e..4d76493fdb2 100644 --- a/.nextchanges/bundles/app-compute-settle.md +++ b/.nextchanges/bundles/app-compute-settle.md @@ -1 +1 @@ -* Retry app updates and deletes in the direct deployment engine while the app compute is `UPDATING`, `STARTING` or `STOPPING`, instead of failing with 400 BAD_REQUEST. ([#7606](https://github.com/databricks/cli/pull/7606)) +* Retry app updates and deletes in the direct deployment engine when they fail with 400 BAD_REQUEST because the app compute is `UPDATING`, after waiting for the compute to settle. ([#7606](https://github.com/databricks/cli/pull/7606)) diff --git a/bundle/direct/dresources/app.go b/bundle/direct/dresources/app.go index 86964a8ab2a..b5bbf5e90f4 100644 --- a/bundle/direct/dresources/app.go +++ b/bundle/direct/dresources/app.go @@ -5,7 +5,6 @@ import ( "errors" "fmt" "net/http" - "regexp" "slices" "strings" "time" @@ -13,6 +12,7 @@ import ( "github.com/databricks/cli/bundle/appdeploy" "github.com/databricks/cli/bundle/config/resources" "github.com/databricks/cli/bundle/deployplan" + "github.com/databricks/cli/libs/log" "github.com/databricks/cli/libs/retry" "github.com/databricks/cli/libs/structs/structpath" "github.com/databricks/databricks-sdk-go" @@ -366,16 +366,26 @@ func (r *ResourceApp) DoDelete(ctx context.Context, id string, _ *AppState) erro return err } -// computeTransitionalStateError matches the 400 BAD_REQUEST messages the Apps API returns for -// an update or delete while the compute is in a transitional state: "Cannot update app X as -// its compute is in UPDATING state" and "Cannot delete app X as it is not terminal with state -// UPDATING". DELETING is left out: a delete of a DELETING app is handled by IsGone and -// deleteConfirmedGone, and waiting would not help. -var computeTransitionalStateError = regexp.MustCompile(`(?:is in|with state) (?:UPDATING|STARTING|STOPPING)\b`) +// computeUpdatingErrorMessages are the 400 BAD_REQUEST messages the Apps API returns for an +// update or delete while the compute is UPDATING, as seen in cloud test runs: +// - "Cannot update app X as its compute is in UPDATING state. App compute needs to be ACTIVE or STOPPED to update." +// - "Cannot delete app X as it is not terminal with state UPDATING, and was updated less than 20 minutes ago. ..." +// +// The DELETING variant of the delete message is left out: IsGone and deleteConfirmedGone +// handle a delete of a DELETING app, and waiting would not help. +var computeUpdatingErrorMessages = []string{ + "compute is in UPDATING state", + "not terminal with state UPDATING,", +} -func isComputeTransitionalStateError(err error) bool { +func isComputeUpdatingError(err error) bool { apiErr, ok := errors.AsType[*apierr.APIError](err) - return ok && apiErr.StatusCode == http.StatusBadRequest && computeTransitionalStateError.MatchString(apiErr.Message) + if !ok || apiErr.StatusCode != http.StatusBadRequest { + return false + } + return slices.ContainsFunc(computeUpdatingErrorMessages, func(m string) bool { + return strings.Contains(apiErr.Message, m) + }) } // errAppGoneWhileSettling reports that the app disappeared while waiting for its compute. @@ -384,18 +394,23 @@ var errAppGoneWhileSettling = errors.New("app is gone") // computeSettleTimeout bounds one wait for the app compute to leave a transitional state. const computeSettleTimeout = 20 * time.Minute -// retryWhileComputeSettles calls fn and, if the Apps API rejected it because the compute is in -// a transitional state, waits for the compute to settle and calls fn again (see retry.With for -// the attempt limit). The compute can enter UPDATING on its own shortly after a create or -// update returned. It returns the app as last read while waiting, or nil if it did not wait, -// so a caller can refresh what it knew about the compute state. +// retryWhileComputeSettles calls fn and, if the Apps API rejected it because the compute is +// UPDATING, waits for the compute to settle and calls fn again (see retry.With for the attempt +// limit and the pause between attempts). The compute can enter UPDATING on its own shortly +// after a create or update returned. It returns the app as last read while waiting, or nil if +// it did not wait, so a caller can refresh what it knew about the compute state. +// +// If the wait itself fails, the error stays the API's 400 for callers that classify it (for +// example retry.IsTransient in apply.go), with the wait failure only in the text. If the app +// is gone, the error wraps errAppGoneWhileSettling and the 404. func (r *ResourceApp) retryWhileComputeSettles(ctx context.Context, name string, fn func() error) (*apps.App, error) { var settled *apps.App var waitErr error _, err := retry.With(ctx, func(ctx context.Context, err error) bool { - if !isComputeTransitionalStateError(err) { + if !isComputeUpdatingError(err) { return false } + log.Warnf(ctx, "App %s compute is UPDATING; waiting up to %s for it to settle before retrying", name, computeSettleTimeout) settled, waitErr = r.waitForComputeToSettle(ctx, name) return waitErr == nil }, func() (struct{}, error) { @@ -405,7 +420,7 @@ func (r *ResourceApp) retryWhileComputeSettles(ctx context.Context, name string, if apierr.IsMissing(waitErr) { return nil, fmt.Errorf("%w: %w", errAppGoneWhileSettling, waitErr) } - return nil, errors.Join(err, waitErr) + return nil, fmt.Errorf("%w (waiting for the app compute to settle failed: %s)", err, waitErr.Error()) } return settled, err } diff --git a/bundle/direct/dresources/app_test.go b/bundle/direct/dresources/app_test.go index fec5874f471..a76bc9bf1eb 100644 --- a/bundle/direct/dresources/app_test.go +++ b/bundle/direct/dresources/app_test.go @@ -251,26 +251,57 @@ func TestWithMissingServicePrincipalHint(t *testing.T) { } } -const appComputeUpdatingDeleteMessage = "Cannot delete app test-app as it is not terminal with state UPDATING, " + - "and was updated less than 20 minutes ago. Please wait before trying again." +const ( + appComputeUpdatingDeleteMessage = "Cannot delete app test-app as it is not terminal with state UPDATING, " + + "and was updated less than 20 minutes ago. Please wait before trying again." + appComputeUpdatingUpdateMessage = "Cannot update app test-app as its compute is in UPDATING state. " + + "App compute needs to be ACTIVE or STOPPED to update." +) + +func appBadRequest(message string) testserver.Response { + return testserver.Response{ + StatusCode: 400, + Body: map[string]string{"error_code": "BAD_REQUEST", "message": message}, + } +} + +// fakeAppCompute serves GET /api/2.0/apps/{name} and the start/stop endpoints. GET returns the +// states in transitional first, then settled; start and stop change settled. A nil settled makes +// GET return 404 once transitional is used up. +type fakeAppCompute struct { + transitional []apps.ComputeState + settled *apps.ComputeState + gets int + starts int + stops int +} -// newAppComputeSettleServer serves GET /api/2.0/apps/{name} with the given compute states in -// order (the last one repeats) and counts the calls. -func newAppComputeSettleServer(t *testing.T, gets *int, states ...apps.ComputeState) *testserver.Server { +func (f *fakeAppCompute) register(t *testing.T, server *testserver.Server) { t.Setenv(retry.IntervalMsVariable, "1") - server := testserver.New(t) server.Handle("GET", "/api/2.0/apps/{name}", func(req testserver.Request) any { - if len(states) == 0 { + f.gets++ + state := f.settled + if f.gets <= len(f.transitional) { + state = &f.transitional[f.gets-1] + } + if state == nil { return testserver.Response{ StatusCode: 404, Body: map[string]string{"error_code": "RESOURCE_DOES_NOT_EXIST", "message": "App not found."}, } } - state := states[min(*gets, len(states)-1)] - *gets++ - return apps.App{Name: req.Vars["name"], ComputeStatus: &apps.ComputeStatus{State: state}} + return apps.App{Name: req.Vars["name"], ComputeStatus: &apps.ComputeStatus{State: *state}} + }) + server.Handle("POST", "/api/2.0/apps/{name}/start", func(req testserver.Request) any { + f.starts++ + f.settled = new(apps.ComputeStateActive) + return apps.App{Name: req.Vars["name"], ComputeStatus: &apps.ComputeStatus{State: apps.ComputeStateActive}} + }) + server.Handle("POST", "/api/2.0/apps/{name}/stop", func(req testserver.Request) any { + f.stops++ + f.settled = new(apps.ComputeStateStopped) + return apps.App{Name: req.Vars["name"], ComputeStatus: &apps.ComputeStatus{State: apps.ComputeStateStopped}} }) - return server } func newAppForServer(t *testing.T, server *testserver.Server) *ResourceApp { @@ -282,60 +313,69 @@ func newAppForServer(t *testing.T, server *testserver.Server) *ResourceApp { func TestAppDoDelete_RetriesWhileComputeSettles(t *testing.T) { tests := []struct { - name string - message string - computeStates []apps.ComputeState - wantDeletes int - wantGets int - wantErr string + name string + compute fakeAppCompute + // deleteMessages are the 400 messages DELETE returns in order; later calls succeed. + deleteMessages []string + wantDeletes int + wantGets int + wantErr string }{ { - name: "retries after compute leaves UPDATING", - message: appComputeUpdatingDeleteMessage, - computeStates: []apps.ComputeState{apps.ComputeStateUpdating, apps.ComputeStateActive}, - wantDeletes: 2, - wantGets: 2, + name: "retries after compute leaves UPDATING", + compute: fakeAppCompute{transitional: []apps.ComputeState{apps.ComputeStateUpdating}, settled: new(apps.ComputeStateActive)}, + deleteMessages: []string{appComputeUpdatingDeleteMessage}, + wantDeletes: 2, + wantGets: 2, }, { - name: "retries when compute settled before the first read", - message: appComputeUpdatingDeleteMessage, - computeStates: []apps.ComputeState{apps.ComputeStateActive}, - wantDeletes: 2, - wantGets: 1, + name: "retries when compute settled before the first read", + compute: fakeAppCompute{settled: new(apps.ComputeStateActive)}, + deleteMessages: []string{appComputeUpdatingDeleteMessage}, + wantDeletes: 2, + wantGets: 1, }, { - name: "treats an app gone while waiting as deleted", - message: appComputeUpdatingDeleteMessage, - wantDeletes: 1, + name: "treats an app gone while waiting as deleted", + compute: fakeAppCompute{}, + deleteMessages: []string{appComputeUpdatingDeleteMessage}, + wantDeletes: 1, + wantGets: 1, }, { - name: "does not retry a delete of a DELETING app", - message: "Cannot delete app test-app as it is not terminal with state DELETING, and was updated less than 20 minutes ago. Please wait before trying again.", - computeStates: []apps.ComputeState{apps.ComputeStateDeleting}, - wantDeletes: 1, - wantErr: "state DELETING", + name: "returns the last 400 when retries run out", + compute: fakeAppCompute{settled: new(apps.ComputeStateActive)}, + deleteMessages: slices.Repeat([]string{appComputeUpdatingDeleteMessage}, retry.MaxRetries+1), + wantDeletes: retry.MaxRetries + 1, + wantGets: retry.MaxRetries, + wantErr: "not terminal with state UPDATING", }, { - name: "does not retry other bad requests", - message: "Invalid app name.", - computeStates: []apps.ComputeState{apps.ComputeStateUpdating}, - wantDeletes: 1, - wantErr: "Invalid app name.", + name: "does not retry a delete of a DELETING app", + compute: fakeAppCompute{settled: new(apps.ComputeStateDeleting)}, + deleteMessages: []string{"Cannot delete app test-app as it is not terminal with state DELETING, and was updated less than 20 minutes ago. Please wait before trying again."}, + wantDeletes: 1, + wantErr: "state DELETING", + }, + { + name: "does not retry other bad requests", + compute: fakeAppCompute{settled: new(apps.ComputeStateUpdating)}, + deleteMessages: []string{"Invalid app name."}, + wantDeletes: 1, + wantErr: "Invalid app name.", }, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { - gets := 0 - server := newAppComputeSettleServer(t, &gets, tc.computeStates...) + server := testserver.New(t) + compute := tc.compute + compute.register(t, server) deletes := 0 server.Handle("DELETE", "/api/2.0/apps/{name}", func(req testserver.Request) any { deletes++ - if deletes == 1 { - return testserver.Response{ - StatusCode: 400, - Body: map[string]string{"error_code": "BAD_REQUEST", "message": tc.message}, - } + if deletes <= len(tc.deleteMessages) { + return appBadRequest(tc.deleteMessages[deletes-1]) } return apps.App{Name: req.Vars["name"]} }) @@ -348,36 +388,140 @@ func TestAppDoDelete_RetriesWhileComputeSettles(t *testing.T) { require.NoError(t, err) } assert.Equal(t, tc.wantDeletes, deletes) - assert.Equal(t, tc.wantGets, gets) + assert.Equal(t, tc.wantGets, compute.gets) }) } } -func TestAppDoUpdate_RetriesWhileComputeSettles(t *testing.T) { - gets := 0 - server := newAppComputeSettleServer(t, &gets, apps.ComputeStateUpdating, apps.ComputeStateActive) - updates := 0 - server.Handle("POST", "/api/2.0/apps/{name}/update", func(req testserver.Request) any { - updates++ - if updates == 1 { - return testserver.Response{ - StatusCode: 400, - Body: map[string]string{ - "error_code": "BAD_REQUEST", - "message": "Cannot update app test-app as its compute is in UPDATING state. App compute needs to be ACTIVE or STOPPED to update.", - }, - } +// TestAppDoDelete_WaitFailureKeepsTheBadRequest checks that a failed wait does not change how +// the apply layer classifies the error: it stays the 400, not the GET's 403. +func TestAppDoDelete_WaitFailureKeepsTheBadRequest(t *testing.T) { + server := testserver.New(t) + server.Handle("GET", "/api/2.0/apps/{name}", func(req testserver.Request) any { + return testserver.Response{ + StatusCode: 403, + Body: map[string]string{"error_code": "PERMISSION_DENIED", "message": "no access"}, } - return apps.AppUpdate{Status: &apps.AppUpdateUpdateStatus{State: apps.AppUpdateUpdateStatusUpdateStateSucceeded}} }) - server.Handle("GET", "/api/2.0/apps/{name}/update", func(req testserver.Request) any { - return apps.AppUpdate{Status: &apps.AppUpdateUpdateStatus{State: apps.AppUpdateUpdateStatusUpdateStateSucceeded}} + deletes := 0 + server.Handle("DELETE", "/api/2.0/apps/{name}", func(req testserver.Request) any { + deletes++ + return appBadRequest(appComputeUpdatingDeleteMessage) }) r := newAppForServer(t, server) - entry := &PlanEntry{Changes: deployplan.Changes{"description": {Action: deployplan.Update}}} - _, err := r.DoUpdate(t.Context(), "test-app", &AppState{Name: "test-app", Description: "new"}, entry) - require.NoError(t, err) - assert.Equal(t, 2, updates) - assert.Equal(t, 2, gets) + err := r.DoDelete(t.Context(), "test-app", nil) + require.ErrorContains(t, err, "not terminal with state UPDATING") + require.ErrorContains(t, err, "waiting for the app compute to settle failed: no access") + apiErr, ok := errors.AsType[*apierr.APIError](err) + require.True(t, ok) + assert.Equal(t, 400, apiErr.StatusCode) + assert.NotErrorIs(t, err, apierr.ErrPermissionDenied) + assert.Equal(t, 1, deletes) +} + +func TestAppDoUpdate_RetriesWhileComputeSettles(t *testing.T) { + tests := []struct { + name string + // planStarted is the compute started state recorded at plan time. + planStarted bool + // lifecycleStarted is lifecycle.started in the config; nil means not set. + lifecycleStarted *bool + compute fakeAppCompute + wantUpdates int + wantStarts int + wantStops int + wantDeploys int + wantMissing bool + }{ + { + name: "retries after compute leaves UPDATING", + planStarted: true, + compute: fakeAppCompute{transitional: []apps.ComputeState{apps.ComputeStateUpdating}, settled: new(apps.ComputeStateActive)}, + wantUpdates: 2, + }, + { + name: "starts an app that settled to STOPPED", + planStarted: true, + lifecycleStarted: new(true), + compute: fakeAppCompute{transitional: []apps.ComputeState{apps.ComputeStateStopping}, settled: new(apps.ComputeStateStopped)}, + wantUpdates: 2, + wantStarts: 1, + wantDeploys: 1, + }, + { + name: "does not stop an app that settled to STOPPED", + planStarted: true, + lifecycleStarted: new(false), + compute: fakeAppCompute{transitional: []apps.ComputeState{apps.ComputeStateStopping}, settled: new(apps.ComputeStateStopped)}, + wantUpdates: 2, + }, + { + name: "does not start an app that settled to ACTIVE", + planStarted: false, + lifecycleStarted: new(true), + compute: fakeAppCompute{transitional: []apps.ComputeState{apps.ComputeStateStarting}, settled: new(apps.ComputeStateActive)}, + wantUpdates: 2, + wantDeploys: 1, + }, + { + name: "fails with not found when the app is gone while waiting", + planStarted: true, + compute: fakeAppCompute{}, + wantUpdates: 1, + wantMissing: true, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + server := testserver.New(t) + compute := tc.compute + compute.register(t, server) + updates := 0 + server.Handle("POST", "/api/2.0/apps/{name}/update", func(req testserver.Request) any { + updates++ + if updates == 1 { + return appBadRequest(appComputeUpdatingUpdateMessage) + } + return apps.AppUpdate{Status: &apps.AppUpdateUpdateStatus{State: apps.AppUpdateUpdateStatusUpdateStateSucceeded}} + }) + server.Handle("GET", "/api/2.0/apps/{name}/update", func(req testserver.Request) any { + return apps.AppUpdate{Status: &apps.AppUpdateUpdateStatus{State: apps.AppUpdateUpdateStatusUpdateStateSucceeded}} + }) + deploys := 0 + succeeded := apps.AppDeployment{ + DeploymentId: "deploy-1", + Status: &apps.AppDeploymentStatus{State: apps.AppDeploymentStateSucceeded}, + } + server.Handle("POST", "/api/2.0/apps/{name}/deployments", func(req testserver.Request) any { + deploys++ + return succeeded + }) + server.Handle("GET", "/api/2.0/apps/{name}/deployments/{deployment_id}", func(req testserver.Request) any { + return succeeded + }) + r := newAppForServer(t, server) + + config := &AppState{Name: "test-app", Description: "new", SourceCodePath: "/Workspace/app"} + if tc.lifecycleStarted != nil { + config.Lifecycle = &StateLifecycle{Started: tc.lifecycleStarted} + } + entry := &PlanEntry{ + Changes: deployplan.Changes{"description": {Action: deployplan.Update}}, + RemoteState: &AppRemote{Lifecycle: &StateLifecycle{Started: new(tc.planStarted)}}, + } + _, err := r.DoUpdate(t.Context(), "test-app", config, entry) + if tc.wantMissing { + require.ErrorContains(t, err, "app is gone") + assert.True(t, apierr.IsMissing(err), "want a not-found error, got %v", err) + } else { + require.NoError(t, err) + } + assert.Equal(t, tc.wantUpdates, updates) + assert.Equal(t, tc.wantStarts, compute.starts) + assert.Equal(t, tc.wantStops, compute.stops) + assert.Equal(t, tc.wantDeploys, deploys) + }) + } } From 6d8c8259d8905509e5619ee23c2e1383cf7f0592 Mon Sep 17 00:00:00 2001 From: "denis.bilenko" Date: Tue, 17 Nov 2026 13:25:46 +0000 Subject: [PATCH 5/7] Run the affected app tests on cloud cloud: TestAccept/bundle/apps/job_permissions TestAccept/bundle/invariant/no_drift/DMS=true/INPUT_CONFIG=app.yml.tmpl Co-authored-by: Isaac From f4332a4afc28f7978deb67a31a45f4ee74cefaed Mon Sep 17 00:00:00 2001 From: "denis.bilenko" Date: Tue, 17 Nov 2026 14:00:25 +0000 Subject: [PATCH 6/7] Word the compute wait warning so acceptance tests suppress it Whether a cloud run hits the race is timing dependent, so the warning must match SuppressLines.warn_retry like the libs/retry warnings do. A test reads the pattern from acceptance/test.toml. cloud: TestAccept/bundle/apps/job_permissions TestAccept/bundle/invariant/no_drift/DMS=true/INPUT_CONFIG=app.yml.tmpl Co-authored-by: Isaac --- bundle/direct/dresources/app.go | 9 ++++++++- bundle/direct/dresources/app_test.go | 13 +++++++++++++ 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/bundle/direct/dresources/app.go b/bundle/direct/dresources/app.go index b5bbf5e90f4..49e62bf1278 100644 --- a/bundle/direct/dresources/app.go +++ b/bundle/direct/dresources/app.go @@ -410,7 +410,7 @@ func (r *ResourceApp) retryWhileComputeSettles(ctx context.Context, name string, if !isComputeUpdatingError(err) { return false } - log.Warnf(ctx, "App %s compute is UPDATING; waiting up to %s for it to settle before retrying", name, computeSettleTimeout) + log.Warnf(ctx, "%s", computeUpdatingRetryMessage(name)) settled, waitErr = r.waitForComputeToSettle(ctx, name) return waitErr == nil }, func() (struct{}, error) { @@ -425,6 +425,13 @@ func (r *ResourceApp) retryWhileComputeSettles(ctx context.Context, name string, return settled, err } +// computeUpdatingRetryMessage is the warning logged before waiting for the compute. Like +// libs/retry, it says "retrying after" so that acceptance tests suppress it +// (SuppressLines.warn_retry): whether a cloud run hits the race is timing dependent. +func computeUpdatingRetryMessage(name string) string { + return fmt.Sprintf("app %s compute is UPDATING: retrying after it settles (waiting up to %s)", name, computeSettleTimeout) +} + func isComputeTransitional(app *apps.App) bool { if app.ComputeStatus == nil { return false diff --git a/bundle/direct/dresources/app_test.go b/bundle/direct/dresources/app_test.go index a76bc9bf1eb..8b1e9c9d86e 100644 --- a/bundle/direct/dresources/app_test.go +++ b/bundle/direct/dresources/app_test.go @@ -4,7 +4,9 @@ import ( "encoding/json" "errors" "fmt" + "os" "reflect" + "regexp" "slices" "strings" "testing" @@ -525,3 +527,14 @@ func TestAppDoUpdate_RetriesWhileComputeSettles(t *testing.T) { }) } } + +// TestComputeUpdatingRetryMessage_SuppressedInAcceptanceTests guards the wording: the warning +// only appears when a cloud run hits the race, so it must match SuppressLines.warn_retry. +func TestComputeUpdatingRetryMessage_SuppressedInAcceptanceTests(t *testing.T) { + raw, err := os.ReadFile("../../../acceptance/test.toml") + require.NoError(t, err) + m := regexp.MustCompile(`(?m)^SuppressLines\.warn_retry = '(.*)'$`).FindSubmatch(raw) + require.NotNil(t, m, "SuppressLines.warn_retry not found in acceptance/test.toml") + pattern := regexp.MustCompile(string(m[1])) + assert.Regexp(t, pattern, "Warn: "+computeUpdatingRetryMessage("test-app")) +} From 46c254467650e3c7d54579dcee17a6b3fb1b45a0 Mon Sep 17 00:00:00 2001 From: "denis.bilenko" Date: Tue, 17 Nov 2026 14:29:42 +0000 Subject: [PATCH 7/7] Allow CRLF when reading acceptance/test.toml in the warning test Windows checkouts have CRLF line endings, so the pattern line ends in \r. cloud: TestAccept/bundle/apps/job_permissions TestAccept/bundle/invariant/no_drift/DMS=true/INPUT_CONFIG=app.yml.tmpl Co-authored-by: Isaac --- bundle/direct/dresources/app_test.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/bundle/direct/dresources/app_test.go b/bundle/direct/dresources/app_test.go index 8b1e9c9d86e..f689f7c00bb 100644 --- a/bundle/direct/dresources/app_test.go +++ b/bundle/direct/dresources/app_test.go @@ -533,7 +533,8 @@ func TestAppDoUpdate_RetriesWhileComputeSettles(t *testing.T) { func TestComputeUpdatingRetryMessage_SuppressedInAcceptanceTests(t *testing.T) { raw, err := os.ReadFile("../../../acceptance/test.toml") require.NoError(t, err) - m := regexp.MustCompile(`(?m)^SuppressLines\.warn_retry = '(.*)'$`).FindSubmatch(raw) + // The checkout may have CRLF line endings (Windows). + m := regexp.MustCompile(`(?m)^SuppressLines\.warn_retry = '(.*)'\r?$`).FindSubmatch(raw) require.NotNil(t, m, "SuppressLines.warn_retry not found in acceptance/test.toml") pattern := regexp.MustCompile(string(m[1])) assert.Regexp(t, pattern, "Warn: "+computeUpdatingRetryMessage("test-app"))