diff --git a/.nextchanges/bundles/app-compute-settle.md b/.nextchanges/bundles/app-compute-settle.md new file mode 100644 index 00000000000..4d76493fdb2 --- /dev/null +++ b/.nextchanges/bundles/app-compute-settle.md @@ -0,0 +1 @@ +* Retry app updates and deletes in the direct deployment engine when they fail with 400 BAD_REQUEST because the app compute is `UPDATING`, after waiting for the compute to settle. ([#7606](https://github.com/databricks/cli/pull/7606)) diff --git a/bundle/direct/dresources/app.go b/bundle/direct/dresources/app.go index ea705285376..a24b6678aaa 100644 --- a/bundle/direct/dresources/app.go +++ b/bundle/direct/dresources/app.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "net/http" "slices" "strings" "time" @@ -11,6 +12,8 @@ import ( "github.com/databricks/cli/bundle/appdeploy" "github.com/databricks/cli/bundle/config/resources" "github.com/databricks/cli/bundle/deployplan" + "github.com/databricks/cli/libs/log" + "github.com/databricks/cli/libs/retry" "github.com/databricks/cli/libs/structs/structpath" "github.com/databricks/databricks-sdk-go" "github.com/databricks/databricks-sdk-go/apierr" @@ -206,6 +209,7 @@ func (r *ResourceApp) DoUpdate(ctx context.Context, id string, config *AppState, } func (r *ResourceApp) doUpdate(ctx context.Context, id string, config *AppState, entry *PlanEntry) error { + alreadyStarted := remoteIsStarted(entry) // git_source is sent as part of the App body (in the update mask). source_code_path and // config (command/env) are deploy-only on update and applied via manageLifecycle; see // appUpdateBody. @@ -216,10 +220,19 @@ func (r *ResourceApp) doUpdate(ctx context.Context, id string, config *AppState, AppName: id, UpdateMask: appUpdateMask(&app), } - updateWaiter, err := r.client.Apps.CreateUpdate(ctx, request) + var updateWaiter *apps.WaitGetUpdateAppSucceeded[apps.AppUpdate] + settled, err := r.retryWhileComputeSettles(ctx, id, func() error { + var err error + updateWaiter, err = r.client.Apps.CreateUpdate(ctx, request) + return err + }) if err != nil { return err } + // The plan-time compute state is stale if the compute moved while we waited. + if settled != nil { + alreadyStarted = !isComputeStopped(settled) + } response, err := updateWaiter.Get() if err != nil { @@ -231,7 +244,7 @@ func (r *ResourceApp) doUpdate(ctx context.Context, id string, config *AppState, } } - return r.manageLifecycle(ctx, id, config, remoteIsStarted(entry)) + return r.manageLifecycle(ctx, id, config, alreadyStarted) } // missingServicePrincipalMessage is part of the backend error for an app whose service @@ -358,10 +371,109 @@ func deploymentToAppConfig(d *apps.AppDeployment) *resources.AppConfig { } func (r *ResourceApp) DoDelete(ctx context.Context, id string, _ *AppState) error { - _, err := r.client.Apps.DeleteByName(ctx, id) + _, err := r.retryWhileComputeSettles(ctx, id, func() error { + _, err := r.client.Apps.DeleteByName(ctx, id) + return err + }) + if errors.Is(err, errAppGoneWhileSettling) { + return nil + } return err } +// computeUpdatingErrorMessages are the 400 BAD_REQUEST messages the Apps API returns for an +// update or delete while the compute is UPDATING, as seen in cloud test runs: +// - "Cannot update app X as its compute is in UPDATING state. App compute needs to be ACTIVE or STOPPED to update." +// - "Cannot delete app X as it is not terminal with state UPDATING, and was updated less than 20 minutes ago. ..." +// +// The DELETING variant of the delete message is left out: IsGone and deleteConfirmedGone +// handle a delete of a DELETING app, and waiting would not help. +var computeUpdatingErrorMessages = []string{ + "compute is in UPDATING state", + "not terminal with state UPDATING,", +} + +func isComputeUpdatingError(err error) bool { + apiErr, ok := errors.AsType[*apierr.APIError](err) + if !ok || apiErr.StatusCode != http.StatusBadRequest { + return false + } + return slices.ContainsFunc(computeUpdatingErrorMessages, func(m string) bool { + return strings.Contains(apiErr.Message, m) + }) +} + +// errAppGoneWhileSettling reports that the app disappeared while waiting for its compute. +var errAppGoneWhileSettling = errors.New("app is gone") + +// computeSettleTimeout bounds one wait for the app compute to leave a transitional state. +const computeSettleTimeout = 20 * time.Minute + +// retryWhileComputeSettles calls fn and, if the Apps API rejected it because the compute is +// UPDATING, waits for the compute to settle and calls fn again (see retry.With for the attempt +// limit and the pause between attempts). The compute can enter UPDATING on its own shortly +// after a create or update returned. It returns the app as last read while waiting, or nil if +// it did not wait, so a caller can refresh what it knew about the compute state. +// +// If the wait itself fails, the error stays the API's 400 for callers that classify it (for +// example retry.IsTransient in apply.go), with the wait failure only in the text. If the app +// is gone, the error wraps errAppGoneWhileSettling and the 404. +func (r *ResourceApp) retryWhileComputeSettles(ctx context.Context, name string, fn func() error) (*apps.App, error) { + var settled *apps.App + var waitErr error + _, err := retry.With(ctx, func(ctx context.Context, err error) bool { + if !isComputeUpdatingError(err) { + return false + } + log.Warnf(ctx, "%s", computeUpdatingRetryMessage(name)) + settled, waitErr = r.waitForComputeToSettle(ctx, name) + return waitErr == nil + }, func() (struct{}, error) { + return struct{}{}, fn() + }) + if err != nil && waitErr != nil { + if apierr.IsMissing(waitErr) { + return nil, fmt.Errorf("%w: %w", errAppGoneWhileSettling, waitErr) + } + return nil, fmt.Errorf("%w (waiting for the app compute to settle failed: %s)", err, waitErr.Error()) + } + return settled, err +} + +// computeUpdatingRetryMessage is the warning logged before waiting for the compute. Like +// libs/retry, it says "retrying after" so that acceptance tests suppress it +// (SuppressLines.warn_retry): whether a cloud run hits the race is timing dependent. +func computeUpdatingRetryMessage(name string) string { + return fmt.Sprintf("app %s compute is UPDATING: retrying after it settles (waiting up to %s)", name, computeSettleTimeout) +} + +func isComputeTransitional(app *apps.App) bool { + if app.ComputeStatus == nil { + return false + } + switch app.ComputeStatus.State { + case apps.ComputeStateUpdating, apps.ComputeStateStarting, apps.ComputeStateStopping: + return true + default: + return false + } +} + +// waitForComputeToSettle polls the app until its compute leaves a transitional state. +func (r *ResourceApp) waitForComputeToSettle(ctx context.Context, name string) (*apps.App, error) { + retrier := retries.New[apps.App](retries.WithTimeout(computeSettleTimeout), retries.WithRetryFunc(shouldRetry)) + return retrier.Run(ctx, func(ctx context.Context) (*apps.App, error) { + app, err := r.client.Apps.GetByName(ctx, name) + if err != nil { + return nil, retries.Halt(err) + } + if isComputeTransitional(app) { + return nil, retries.Continues(fmt.Sprintf("app compute is %s", app.ComputeStatus.State)) + } + return app, nil + }) +} + // IsGone treats a DELETING app as already-deleted. The Apps DELETE is // fire-and-forget: it returns success while the app sits in // ComputeState=DELETING for up to ~20 minutes, and a GET during that window diff --git a/bundle/direct/dresources/app_test.go b/bundle/direct/dresources/app_test.go index ebc8a7ee7dc..e21728aa328 100644 --- a/bundle/direct/dresources/app_test.go +++ b/bundle/direct/dresources/app_test.go @@ -4,11 +4,15 @@ import ( "encoding/json" "errors" "fmt" + "os" "reflect" + "regexp" "slices" "strings" "testing" + "github.com/databricks/cli/bundle/deployplan" + "github.com/databricks/cli/libs/retry" "github.com/databricks/cli/libs/testserver" "github.com/databricks/databricks-sdk-go" "github.com/databricks/databricks-sdk-go/apierr" @@ -293,3 +297,290 @@ func TestWithMissingServicePrincipalHint(t *testing.T) { assert.Contains(t, got.Error(), `delete it with "databricks apps delete myapp"`) } } + +const ( + appComputeUpdatingDeleteMessage = "Cannot delete app test-app as it is not terminal with state UPDATING, " + + "and was updated less than 20 minutes ago. Please wait before trying again." + appComputeUpdatingUpdateMessage = "Cannot update app test-app as its compute is in UPDATING state. " + + "App compute needs to be ACTIVE or STOPPED to update." +) + +func appBadRequest(message string) testserver.Response { + return testserver.Response{ + StatusCode: 400, + Body: map[string]string{"error_code": "BAD_REQUEST", "message": message}, + } +} + +// fakeAppCompute serves GET /api/2.0/apps/{name} and the start/stop endpoints. GET returns the +// states in transitional first, then settled; start and stop change settled. A nil settled makes +// GET return 404 once transitional is used up. +type fakeAppCompute struct { + transitional []apps.ComputeState + settled *apps.ComputeState + gets int + starts int + stops int +} + +func (f *fakeAppCompute) register(t *testing.T, server *testserver.Server) { + t.Setenv(retry.IntervalMsVariable, "1") + server.Handle("GET", "/api/2.0/apps/{name}", func(req testserver.Request) any { + f.gets++ + state := f.settled + if f.gets <= len(f.transitional) { + state = &f.transitional[f.gets-1] + } + if state == nil { + return testserver.Response{ + StatusCode: 404, + Body: map[string]string{"error_code": "RESOURCE_DOES_NOT_EXIST", "message": "App not found."}, + } + } + return apps.App{Name: req.Vars["name"], ComputeStatus: &apps.ComputeStatus{State: *state}} + }) + server.Handle("POST", "/api/2.0/apps/{name}/start", func(req testserver.Request) any { + f.starts++ + f.settled = new(apps.ComputeStateActive) + return apps.App{Name: req.Vars["name"], ComputeStatus: &apps.ComputeStatus{State: apps.ComputeStateActive}} + }) + server.Handle("POST", "/api/2.0/apps/{name}/stop", func(req testserver.Request) any { + f.stops++ + f.settled = new(apps.ComputeStateStopped) + return apps.App{Name: req.Vars["name"], ComputeStatus: &apps.ComputeStatus{State: apps.ComputeStateStopped}} + }) +} + +func newAppForServer(t *testing.T, server *testserver.Server) *ResourceApp { + testserver.AddDefaultHandlers(server) + client, err := databricks.NewWorkspaceClient(&databricks.Config{Host: server.URL, Token: "testtoken"}) + require.NoError(t, err) + return (&ResourceApp{}).New(client) +} + +func TestAppDoDelete_RetriesWhileComputeSettles(t *testing.T) { + tests := []struct { + name string + compute fakeAppCompute + // deleteMessages are the 400 messages DELETE returns in order; later calls succeed. + deleteMessages []string + wantDeletes int + wantGets int + wantErr string + }{ + { + name: "retries after compute leaves UPDATING", + compute: fakeAppCompute{transitional: []apps.ComputeState{apps.ComputeStateUpdating}, settled: new(apps.ComputeStateActive)}, + deleteMessages: []string{appComputeUpdatingDeleteMessage}, + wantDeletes: 2, + wantGets: 2, + }, + { + name: "retries when compute settled before the first read", + compute: fakeAppCompute{settled: new(apps.ComputeStateActive)}, + deleteMessages: []string{appComputeUpdatingDeleteMessage}, + wantDeletes: 2, + wantGets: 1, + }, + { + name: "treats an app gone while waiting as deleted", + compute: fakeAppCompute{}, + deleteMessages: []string{appComputeUpdatingDeleteMessage}, + wantDeletes: 1, + wantGets: 1, + }, + { + name: "returns the last 400 when retries run out", + compute: fakeAppCompute{settled: new(apps.ComputeStateActive)}, + deleteMessages: slices.Repeat([]string{appComputeUpdatingDeleteMessage}, retry.MaxRetries+1), + wantDeletes: retry.MaxRetries + 1, + wantGets: retry.MaxRetries, + wantErr: "not terminal with state UPDATING", + }, + { + name: "does not retry a delete of a DELETING app", + compute: fakeAppCompute{settled: new(apps.ComputeStateDeleting)}, + deleteMessages: []string{"Cannot delete app test-app as it is not terminal with state DELETING, and was updated less than 20 minutes ago. Please wait before trying again."}, + wantDeletes: 1, + wantErr: "state DELETING", + }, + { + name: "does not retry other bad requests", + compute: fakeAppCompute{settled: new(apps.ComputeStateUpdating)}, + deleteMessages: []string{"Invalid app name."}, + wantDeletes: 1, + wantErr: "Invalid app name.", + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + server := testserver.New(t) + compute := tc.compute + compute.register(t, server) + deletes := 0 + server.Handle("DELETE", "/api/2.0/apps/{name}", func(req testserver.Request) any { + deletes++ + if deletes <= len(tc.deleteMessages) { + return appBadRequest(tc.deleteMessages[deletes-1]) + } + return apps.App{Name: req.Vars["name"]} + }) + r := newAppForServer(t, server) + + err := r.DoDelete(t.Context(), "test-app", nil) + if tc.wantErr != "" { + require.ErrorContains(t, err, tc.wantErr) + } else { + require.NoError(t, err) + } + assert.Equal(t, tc.wantDeletes, deletes) + assert.Equal(t, tc.wantGets, compute.gets) + }) + } +} + +// TestAppDoDelete_WaitFailureKeepsTheBadRequest checks that a failed wait does not change how +// the apply layer classifies the error: it stays the 400, not the GET's 403. +func TestAppDoDelete_WaitFailureKeepsTheBadRequest(t *testing.T) { + server := testserver.New(t) + server.Handle("GET", "/api/2.0/apps/{name}", func(req testserver.Request) any { + return testserver.Response{ + StatusCode: 403, + Body: map[string]string{"error_code": "PERMISSION_DENIED", "message": "no access"}, + } + }) + deletes := 0 + server.Handle("DELETE", "/api/2.0/apps/{name}", func(req testserver.Request) any { + deletes++ + return appBadRequest(appComputeUpdatingDeleteMessage) + }) + r := newAppForServer(t, server) + + err := r.DoDelete(t.Context(), "test-app", nil) + require.ErrorContains(t, err, "not terminal with state UPDATING") + require.ErrorContains(t, err, "waiting for the app compute to settle failed: no access") + apiErr, ok := errors.AsType[*apierr.APIError](err) + require.True(t, ok) + assert.Equal(t, 400, apiErr.StatusCode) + assert.NotErrorIs(t, err, apierr.ErrPermissionDenied) + assert.Equal(t, 1, deletes) +} + +func TestAppDoUpdate_RetriesWhileComputeSettles(t *testing.T) { + tests := []struct { + name string + // planStarted is the compute started state recorded at plan time. + planStarted bool + // lifecycleStarted is lifecycle.started in the config; nil means not set. + lifecycleStarted *bool + compute fakeAppCompute + wantUpdates int + wantStarts int + wantStops int + wantDeploys int + wantMissing bool + }{ + { + name: "retries after compute leaves UPDATING", + planStarted: true, + compute: fakeAppCompute{transitional: []apps.ComputeState{apps.ComputeStateUpdating}, settled: new(apps.ComputeStateActive)}, + wantUpdates: 2, + }, + { + name: "starts an app that settled to STOPPED", + planStarted: true, + lifecycleStarted: new(true), + compute: fakeAppCompute{transitional: []apps.ComputeState{apps.ComputeStateStopping}, settled: new(apps.ComputeStateStopped)}, + wantUpdates: 2, + wantStarts: 1, + wantDeploys: 1, + }, + { + name: "does not stop an app that settled to STOPPED", + planStarted: true, + lifecycleStarted: new(false), + compute: fakeAppCompute{transitional: []apps.ComputeState{apps.ComputeStateStopping}, settled: new(apps.ComputeStateStopped)}, + wantUpdates: 2, + }, + { + name: "does not start an app that settled to ACTIVE", + planStarted: false, + lifecycleStarted: new(true), + compute: fakeAppCompute{transitional: []apps.ComputeState{apps.ComputeStateStarting}, settled: new(apps.ComputeStateActive)}, + wantUpdates: 2, + wantDeploys: 1, + }, + { + name: "fails with not found when the app is gone while waiting", + planStarted: true, + compute: fakeAppCompute{}, + wantUpdates: 1, + wantMissing: true, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + server := testserver.New(t) + compute := tc.compute + compute.register(t, server) + updates := 0 + server.Handle("POST", "/api/2.0/apps/{name}/update", func(req testserver.Request) any { + updates++ + if updates == 1 { + return appBadRequest(appComputeUpdatingUpdateMessage) + } + return apps.AppUpdate{Status: &apps.AppUpdateUpdateStatus{State: apps.AppUpdateUpdateStatusUpdateStateSucceeded}} + }) + server.Handle("GET", "/api/2.0/apps/{name}/update", func(req testserver.Request) any { + return apps.AppUpdate{Status: &apps.AppUpdateUpdateStatus{State: apps.AppUpdateUpdateStatusUpdateStateSucceeded}} + }) + deploys := 0 + succeeded := apps.AppDeployment{ + DeploymentId: "deploy-1", + Status: &apps.AppDeploymentStatus{State: apps.AppDeploymentStateSucceeded}, + } + server.Handle("POST", "/api/2.0/apps/{name}/deployments", func(req testserver.Request) any { + deploys++ + return succeeded + }) + server.Handle("GET", "/api/2.0/apps/{name}/deployments/{deployment_id}", func(req testserver.Request) any { + return succeeded + }) + r := newAppForServer(t, server) + + config := &AppState{Name: "test-app", Description: "new", SourceCodePath: "/Workspace/app"} + if tc.lifecycleStarted != nil { + config.Lifecycle = &StateLifecycle{Started: tc.lifecycleStarted} + } + entry := &PlanEntry{ + Changes: deployplan.Changes{"description": {Action: deployplan.Update}}, + RemoteState: &AppRemote{Lifecycle: &StateLifecycle{Started: new(tc.planStarted)}}, + } + _, err := r.DoUpdate(t.Context(), "test-app", config, entry) + if tc.wantMissing { + require.ErrorContains(t, err, "app is gone") + assert.True(t, apierr.IsMissing(err), "want a not-found error, got %v", err) + } else { + require.NoError(t, err) + } + assert.Equal(t, tc.wantUpdates, updates) + assert.Equal(t, tc.wantStarts, compute.starts) + assert.Equal(t, tc.wantStops, compute.stops) + assert.Equal(t, tc.wantDeploys, deploys) + }) + } +} + +// TestComputeUpdatingRetryMessage_SuppressedInAcceptanceTests guards the wording: the warning +// only appears when a cloud run hits the race, so it must match SuppressLines.warn_retry. +func TestComputeUpdatingRetryMessage_SuppressedInAcceptanceTests(t *testing.T) { + raw, err := os.ReadFile("../../../acceptance/test.toml") + require.NoError(t, err) + // The checkout may have CRLF line endings (Windows). + m := regexp.MustCompile(`(?m)^SuppressLines\.warn_retry = '(.*)'\r?$`).FindSubmatch(raw) + require.NotNil(t, m, "SuppressLines.warn_retry not found in acceptance/test.toml") + pattern := regexp.MustCompile(string(m[1])) + assert.Regexp(t, pattern, "Warn: "+computeUpdatingRetryMessage("test-app")) +}