diff --git a/.nextchanges/bundles/app-start-idempotent.md b/.nextchanges/bundles/app-start-idempotent.md new file mode 100644 index 00000000000..c6829abe913 --- /dev/null +++ b/.nextchanges/bundles/app-start-idempotent.md @@ -0,0 +1 @@ +* Treat starting an app whose compute is already active or starting as success in the direct deployment engine. A retried deploy after a transient error no longer fails with "Cannot start app ... as its compute is in ACTIVE state". ([#7709](https://github.com/databricks/cli/pull/7709)) diff --git a/acceptance/bundle/resources/apps/lifecycle-start-retried/app/app.py b/acceptance/bundle/resources/apps/lifecycle-start-retried/app/app.py new file mode 100644 index 00000000000..ad1e64f9fb0 --- /dev/null +++ b/acceptance/bundle/resources/apps/lifecycle-start-retried/app/app.py @@ -0,0 +1,5 @@ +import http.server, os + +http.server.HTTPServer( + ("", int(os.environ["DATABRICKS_APP_PORT"])), http.server.SimpleHTTPRequestHandler +).serve_forever() diff --git a/acceptance/bundle/resources/apps/lifecycle-start-retried/databricks.yml b/acceptance/bundle/resources/apps/lifecycle-start-retried/databricks.yml new file mode 100644 index 00000000000..474b6b99316 --- /dev/null +++ b/acceptance/bundle/resources/apps/lifecycle-start-retried/databricks.yml @@ -0,0 +1,11 @@ +bundle: + name: lifecycle-start-retried + +resources: + apps: + mykey: + name: myapp + description: my_app_description + source_code_path: ./app + lifecycle: + started: false diff --git a/acceptance/bundle/resources/apps/lifecycle-start-retried/out.test.toml b/acceptance/bundle/resources/apps/lifecycle-start-retried/out.test.toml new file mode 100644 index 00000000000..a927a5fbc06 --- /dev/null +++ b/acceptance/bundle/resources/apps/lifecycle-start-retried/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/resources/apps/lifecycle-start-retried/output.txt b/acceptance/bundle/resources/apps/lifecycle-start-retried/output.txt new file mode 100644 index 00000000000..faef9fa753b --- /dev/null +++ b/acceptance/bundle/resources/apps/lifecycle-start-retried/output.txt @@ -0,0 +1,50 @@ + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/lifecycle-start-retried/default/files... +Created apps.mykey +Files: 2 uploaded, 0 deleted +Resources: 1 created, 0 changed, 0 deleted, 0 unchanged + +=== the deploy succeeds, though its start was answered with a 504 + +>>> update_file.py databricks.yml started: false started: true + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/lifecycle-start-retried/default/files... +✓ Deployment succeeded +Updated apps.mykey +Files: 1 uploaded, 0 deleted +Resources: 0 created, 1 changed, 0 deleted, 0 unchanged + +=== start was sent twice; the second was rejected because the compute was already ACTIVE + +>>> print_requests.py //apps ^//api/2.0/bundle --del-body url +{ + "method": "POST", + "path": "/api/2.0/apps/myapp/start", + "body": {} +} +{ + "method": "POST", + "path": "/api/2.0/apps/myapp/start", + "body": {} +} +{ + "method": "POST", + "path": "/api/2.0/apps/myapp/deployments", + "body": { + "mode": "SNAPSHOT", + "source_code_path": "/Workspace/Users/[USERNAME]/.bundle/lifecycle-start-retried/default/files/app" + } +} + +>>> [CLI] apps get myapp +"ACTIVE" + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.apps.mykey + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/lifecycle-start-retried/default + +Destroy: 1 deleted diff --git a/acceptance/bundle/resources/apps/lifecycle-start-retried/script b/acceptance/bundle/resources/apps/lifecycle-start-retried/script new file mode 100644 index 00000000000..b31fc35ad39 --- /dev/null +++ b/acceptance/bundle/resources/apps/lifecycle-start-retried/script @@ -0,0 +1,20 @@ +cleanup() { + trace $CLI bundle destroy --auto-approve + rm -f "$OUT_REQUESTS" +} +trap cleanup EXIT + +trace $CLI bundle deploy +rm -f "$OUT_REQUESTS" + +# Keep the start the handler applied, replace only the response with a 504, so the start +# is sent again while the compute is already ACTIVE. +fault.py --after-handler "POST /api/2.0/apps/*/start" 504 0 1 + +title "the deploy succeeds, though its start was answered with a 504" +trace update_file.py databricks.yml "started: false" "started: true" +trace $CLI bundle deploy + +title "start was sent twice; the second was rejected because the compute was already ACTIVE" +trace print_requests.py //apps '^//api/2.0/bundle' --del-body url +trace $CLI apps get myapp | jq '.compute_status.state' diff --git a/acceptance/bundle/resources/apps/lifecycle-start-retried/test.toml b/acceptance/bundle/resources/apps/lifecycle-start-retried/test.toml new file mode 100644 index 00000000000..2229b4f446b --- /dev/null +++ b/acceptance/bundle/resources/apps/lifecycle-start-retried/test.toml @@ -0,0 +1,5 @@ +# Local only: the lost response is staged by injecting a fault into the testserver. +Cloud = false +RecordRequests = true +Ignore = [".databricks", "databricks.yml"] +Env.DATABRICKS_BUNDLE_RETRY_INTERVAL_MS = "100" diff --git a/bundle/direct/dresources/app.go b/bundle/direct/dresources/app.go index a24b6678aaa..dec6fea15ce 100644 --- a/bundle/direct/dresources/app.go +++ b/bundle/direct/dresources/app.go @@ -272,11 +272,7 @@ func (r *ResourceApp) manageLifecycle(ctx context.Context, id string, config *Ap if desiredStarted { // lifecycle.started=true: ensure the app compute is running and deploy the latest code. if !alreadyStarted { - startWaiter, err := r.client.Apps.Start(ctx, apps.StartAppRequest{Name: id}) - if err != nil { - return err - } - startedApp, err := startWaiter.Get() + startedApp, err := r.startApp(ctx, id) if err != nil { return err } @@ -307,6 +303,44 @@ func (r *ResourceApp) manageLifecycle(ctx context.Context, id string, config *Ap return nil } +// startApp starts the app compute and waits for it to become active. A start of an app whose +// compute is not STOPPED is rejected with 400 BAD_REQUEST ("Cannot start app X as its compute +// is in ACTIVE state"). That happens when apply retries DoUpdate after a transient error in the +// start waiter: the first start already went through, so the compute is ACTIVE, or still +// STARTING if the retry came before it finished. +func (r *ResourceApp) startApp(ctx context.Context, id string) (*apps.App, error) { + startWaiter, err := r.client.Apps.Start(ctx, apps.StartAppRequest{Name: id}) + if err == nil { + return startWaiter.Get() + } + apiErr, ok := errors.AsType[*apierr.APIError](err) + if !ok || apiErr.StatusCode != http.StatusBadRequest { + return nil, err + } + app, getErr := r.client.Apps.GetByName(ctx, id) + if getErr != nil { + // Wrap only getErr so that apply's retry sees it: a transient read failure leaves + // the compute state unknown, and the 400 alone would make the failure permanent. + return nil, fmt.Errorf("reading app after start was rejected (%s): %w", err.Error(), getErr) + } + if app.ComputeStatus == nil { + return nil, err + } + switch app.ComputeStatus.State { + case apps.ComputeStateActive: + log.Debugf(ctx, "app %s compute is already active, ignoring start error: %s", id, err) + return app, nil + case apps.ComputeStateStarting: + log.Debugf(ctx, "app %s compute is already starting, waiting for it instead: %s", id, err) + return r.client.Apps.WaitGetAppActive(ctx, id, appStartTimeout, nil) + default: + return nil, err + } +} + +// appStartTimeout matches the SDK's default wait for Apps.Start. +const appStartTimeout = 20 * time.Minute + // hasAppChanges reports whether the plan entry contains any Update changes to fields that // belong to the App Update API. source_code_path and config (command/env) are deploy-only // on update and applied via manageLifecycle; lifecycle drives compute start/stop. diff --git a/bundle/direct/dresources/app_test.go b/bundle/direct/dresources/app_test.go index e21728aa328..faca872d4d6 100644 --- a/bundle/direct/dresources/app_test.go +++ b/bundle/direct/dresources/app_test.go @@ -298,6 +298,85 @@ func TestWithMissingServicePrincipalHint(t *testing.T) { } } +// A retried DoUpdate starts the app a second time after the first start went through, and the +// backend rejects the second start. An ACTIVE compute makes that rejection a success, and a +// STARTING one is waited for; any other state keeps the error. +func TestAppStartApp_RejectedStart(t *testing.T) { + tests := []struct { + state apps.ComputeState + wantErr bool + }{ + {state: apps.ComputeStateActive, wantErr: false}, + {state: apps.ComputeStateStarting, wantErr: false}, + {state: apps.ComputeStateStopped, wantErr: true}, + {state: apps.ComputeStateDeleting, wantErr: true}, + {state: "", wantErr: true}, + } + for _, tt := range tests { + t.Run(string(tt.state), func(t *testing.T) { + server := testserver.New(t) + startCalls := 0 + server.Handle("POST", "/api/2.0/apps/{name}/start", func(req testserver.Request) any { + startCalls++ + return testserver.Response{ + StatusCode: 400, + Body: map[string]string{ + "error_code": "BAD_REQUEST", + "message": "Cannot start app " + req.Vars["name"] + " as its compute is in ACTIVE state.", + }, + } + }) + getCalls := 0 + server.Handle("GET", "/api/2.0/apps/{name}", func(req testserver.Request) any { + getCalls++ + app := apps.App{Name: req.Vars["name"]} + state := tt.state + // A STARTING compute becomes ACTIVE on the next poll. + if state == apps.ComputeStateStarting && getCalls > 1 { + state = apps.ComputeStateActive + } + if state != "" { + app.ComputeStatus = &apps.ComputeStatus{State: state} + } + return app + }) + testserver.AddDefaultHandlers(server) + client, err := databricks.NewWorkspaceClient(&databricks.Config{Host: server.URL, Token: "testtoken"}) + require.NoError(t, err) + r := (&ResourceApp{}).New(client) + + app, err := r.startApp(t.Context(), "test-app") + assert.Equal(t, 1, startCalls) + if tt.wantErr { + require.ErrorContains(t, err, "Cannot start app test-app") + return + } + require.NoError(t, err) + assert.Equal(t, apps.ComputeStateActive, app.ComputeStatus.State) + }) + } +} + +// A failed read after a rejected start must stay retryable: the 400 alone is not transient. +func TestAppStartApp_RejectedStartReadFailureIsTransient(t *testing.T) { + server := testserver.New(t) + server.Handle("POST", "/api/2.0/apps/{name}/start", func(req testserver.Request) any { + return appBadRequest("Cannot start app test-app as its compute is in ACTIVE state.") + }) + server.Handle("GET", "/api/2.0/apps/{name}", func(req testserver.Request) any { + return testserver.Response{ + StatusCode: 500, + Body: map[string]string{"error_code": "INTERNAL_ERROR", "message": "boom"}, + } + }) + r := newAppForServer(t, server) + + _, err := r.startApp(t.Context(), "test-app") + require.ErrorContains(t, err, "Cannot start app test-app") + require.ErrorContains(t, err, "boom") + assert.True(t, retry.IsTransient(t.Context(), err)) +} + const ( appComputeUpdatingDeleteMessage = "Cannot delete app test-app as it is not terminal with state UPDATING, " + "and was updated less than 20 minutes ago. Please wait before trying again." diff --git a/libs/testserver/apps.go b/libs/testserver/apps.go index 208c9a33e5a..8396c863128 100644 --- a/libs/testserver/apps.go +++ b/libs/testserver/apps.go @@ -208,6 +208,16 @@ func (s *FakeWorkspace) AppsStart(_ Request, name string) Response { return Response{StatusCode: 404} } + if app.ComputeStatus != nil && app.ComputeStatus.State == apps.ComputeStateActive { + return Response{ + StatusCode: 400, + Body: map[string]string{ + "error_code": "BAD_REQUEST", + "message": fmt.Sprintf("Cannot start app %s as its compute is in ACTIVE state. App compute needs to be STOPPED to start.", name), + }, + } + } + app.ComputeStatus = &apps.ComputeStatus{ State: apps.ComputeStateActive, Message: "App compute is active.",