Repository navigation
direct: treat starting an already-active app as success #7709
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: main
Are you sure you want to change the base?
Changes from all commits
5835905
53dbaa3
17fb92c
8b079df
c296b56
d769cae
13bcdad
640da11
99b39f6
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1 @@ | ||
| * Treat starting an app whose compute is already active or starting as success in the direct deployment engine. A retried deploy after a transient error no longer fails with "Cannot start app ... as its compute is in ACTIVE state". ([#7709](https://github.com/databricks/cli/pull/7709)) |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,5 @@ | ||
| import http.server, os | ||
|
|
||
| http.server.HTTPServer( | ||
| ("", int(os.environ["DATABRICKS_APP_PORT"])), http.server.SimpleHTTPRequestHandler | ||
| ).serve_forever() |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,11 @@ | ||
| bundle: | ||
| name: lifecycle-start-retried | ||
|
|
||
| resources: | ||
| apps: | ||
| mykey: | ||
| name: myapp | ||
| description: my_app_description | ||
| source_code_path: ./app | ||
| lifecycle: | ||
| started: false |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,2 @@ | ||
| Cloud = false | ||
| EnvMatrix.DMS = ["", "true"] |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,50 @@ | ||
|
|
||
| >>> [CLI] bundle deploy | ||
| Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/lifecycle-start-retried/default/files... | ||
| Created apps.mykey | ||
| Files: 2 uploaded, 0 deleted | ||
| Resources: 1 created, 0 changed, 0 deleted, 0 unchanged | ||
|
|
||
| === the deploy succeeds, though its start was answered with a 504 | ||
|
|
||
| >>> update_file.py databricks.yml started: false started: true | ||
|
|
||
| >>> [CLI] bundle deploy | ||
| Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/lifecycle-start-retried/default/files... | ||
| ✓ Deployment succeeded | ||
| Updated apps.mykey | ||
| Files: 1 uploaded, 0 deleted | ||
| Resources: 0 created, 1 changed, 0 deleted, 0 unchanged | ||
|
|
||
| === start was sent twice; the second was rejected because the compute was already ACTIVE | ||
|
|
||
| >>> print_requests.py //apps ^//api/2.0/bundle --del-body url | ||
| { | ||
| "method": "POST", | ||
| "path": "/api/2.0/apps/myapp/start", | ||
| "body": {} | ||
| } | ||
| { | ||
| "method": "POST", | ||
| "path": "/api/2.0/apps/myapp/start", | ||
| "body": {} | ||
| } | ||
| { | ||
| "method": "POST", | ||
| "path": "/api/2.0/apps/myapp/deployments", | ||
| "body": { | ||
| "mode": "SNAPSHOT", | ||
| "source_code_path": "/Workspace/Users/[USERNAME]/.bundle/lifecycle-start-retried/default/files/app" | ||
| } | ||
| } | ||
|
|
||
| >>> [CLI] apps get myapp | ||
| "ACTIVE" | ||
|
|
||
| >>> [CLI] bundle destroy --auto-approve | ||
| The following resources will be deleted: | ||
| delete resources.apps.mykey | ||
|
|
||
| All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/lifecycle-start-retried/default | ||
|
|
||
| Destroy: 1 deleted |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,20 @@ | ||
| cleanup() { | ||
| trace $CLI bundle destroy --auto-approve | ||
| rm -f "$OUT_REQUESTS" | ||
| } | ||
| trap cleanup EXIT | ||
|
|
||
| trace $CLI bundle deploy | ||
| rm -f "$OUT_REQUESTS" | ||
|
|
||
| # Keep the start the handler applied, replace only the response with a 504, so the start | ||
| # is sent again while the compute is already ACTIVE. | ||
| fault.py --after-handler "POST /api/2.0/apps/*/start" 504 0 1 | ||
|
|
||
| title "the deploy succeeds, though its start was answered with a 504" | ||
| trace update_file.py databricks.yml "started: false" "started: true" | ||
| trace $CLI bundle deploy | ||
|
|
||
| title "start was sent twice; the second was rejected because the compute was already ACTIVE" | ||
| trace print_requests.py //apps '^//api/2.0/bundle' --del-body url | ||
| trace $CLI apps get myapp | jq '.compute_status.state' |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,5 @@ | ||
| # Local only: the lost response is staged by injecting a fault into the testserver. | ||
| Cloud = false | ||
| RecordRequests = true | ||
| Ignore = [".databricks", "databricks.yml"] | ||
| Env.DATABRICKS_BUNDLE_RETRY_INTERVAL_MS = "100" | ||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -272,11 +272,7 @@ func (r *ResourceApp) manageLifecycle(ctx context.Context, id string, config *Ap | |
| if desiredStarted { | ||
| // lifecycle.started=true: ensure the app compute is running and deploy the latest code. | ||
| if !alreadyStarted { | ||
| startWaiter, err := r.client.Apps.Start(ctx, apps.StartAppRequest{Name: id}) | ||
| if err != nil { | ||
| return err | ||
| } | ||
| startedApp, err := startWaiter.Get() | ||
| startedApp, err := r.startApp(ctx, id) | ||
| if err != nil { | ||
| return err | ||
| } | ||
|
|
@@ -307,6 +303,44 @@ func (r *ResourceApp) manageLifecycle(ctx context.Context, id string, config *Ap | |
| return nil | ||
| } | ||
|
|
||
| // startApp starts the app compute and waits for it to become active. A start of an app whose | ||
| // compute is not STOPPED is rejected with 400 BAD_REQUEST ("Cannot start app X as its compute | ||
| // is in ACTIVE state"). That happens when apply retries DoUpdate after a transient error in the | ||
| // start waiter: the first start already went through, so the compute is ACTIVE, or still | ||
| // STARTING if the retry came before it finished. | ||
| func (r *ResourceApp) startApp(ctx context.Context, id string) (*apps.App, error) { | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. We have a similar logic in bundle/run/app.go function |
||
| startWaiter, err := r.client.Apps.Start(ctx, apps.StartAppRequest{Name: id}) | ||
| if err == nil { | ||
| return startWaiter.Get() | ||
| } | ||
| apiErr, ok := errors.AsType[*apierr.APIError](err) | ||
| if !ok || apiErr.StatusCode != http.StatusBadRequest { | ||
| return nil, err | ||
| } | ||
| app, getErr := r.client.Apps.GetByName(ctx, id) | ||
| if getErr != nil { | ||
| // Wrap only getErr so that apply's retry sees it: a transient read failure leaves | ||
| // the compute state unknown, and the 400 alone would make the failure permanent. | ||
| return nil, fmt.Errorf("reading app after start was rejected (%s): %w", err.Error(), getErr) | ||
| } | ||
| if app.ComputeStatus == nil { | ||
| return nil, err | ||
| } | ||
| switch app.ComputeStatus.State { | ||
| case apps.ComputeStateActive: | ||
| log.Debugf(ctx, "app %s compute is already active, ignoring start error: %s", id, err) | ||
| return app, nil | ||
| case apps.ComputeStateStarting: | ||
| log.Debugf(ctx, "app %s compute is already starting, waiting for it instead: %s", id, err) | ||
| return r.client.Apps.WaitGetAppActive(ctx, id, appStartTimeout, nil) | ||
| default: | ||
| return nil, err | ||
| } | ||
| } | ||
|
|
||
| // appStartTimeout matches the SDK's default wait for Apps.Start. | ||
| const appStartTimeout = 20 * time.Minute | ||
|
|
||
| // hasAppChanges reports whether the plan entry contains any Update changes to fields that | ||
| // belong to the App Update API. source_code_path and config (command/env) are deploy-only | ||
| // on update and applied via manageLifecycle; lifecycle drives compute start/stop. | ||
|
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -298,6 +298,85 @@ func TestWithMissingServicePrincipalHint(t *testing.T) { | |
| } | ||
| } | ||
|
|
||
| // A retried DoUpdate starts the app a second time after the first start went through, and the | ||
| // backend rejects the second start. An ACTIVE compute makes that rejection a success, and a | ||
| // STARTING one is waited for; any other state keeps the error. | ||
| func TestAppStartApp_RejectedStart(t *testing.T) { | ||
| tests := []struct { | ||
| state apps.ComputeState | ||
| wantErr bool | ||
| }{ | ||
| {state: apps.ComputeStateActive, wantErr: false}, | ||
| {state: apps.ComputeStateStarting, wantErr: false}, | ||
| {state: apps.ComputeStateStopped, wantErr: true}, | ||
| {state: apps.ComputeStateDeleting, wantErr: true}, | ||
| {state: "", wantErr: true}, | ||
| } | ||
| for _, tt := range tests { | ||
| t.Run(string(tt.state), func(t *testing.T) { | ||
| server := testserver.New(t) | ||
| startCalls := 0 | ||
| server.Handle("POST", "/api/2.0/apps/{name}/start", func(req testserver.Request) any { | ||
| startCalls++ | ||
| return testserver.Response{ | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. [AI Generated] nit: nit: this builds the 400 response and the client inline, while the next test uses |
||
| StatusCode: 400, | ||
| Body: map[string]string{ | ||
| "error_code": "BAD_REQUEST", | ||
| "message": "Cannot start app " + req.Vars["name"] + " as its compute is in ACTIVE state.", | ||
| }, | ||
| } | ||
| }) | ||
| getCalls := 0 | ||
| server.Handle("GET", "/api/2.0/apps/{name}", func(req testserver.Request) any { | ||
| getCalls++ | ||
| app := apps.App{Name: req.Vars["name"]} | ||
| state := tt.state | ||
| // A STARTING compute becomes ACTIVE on the next poll. | ||
| if state == apps.ComputeStateStarting && getCalls > 1 { | ||
| state = apps.ComputeStateActive | ||
| } | ||
| if state != "" { | ||
| app.ComputeStatus = &apps.ComputeStatus{State: state} | ||
| } | ||
| return app | ||
| }) | ||
| testserver.AddDefaultHandlers(server) | ||
| client, err := databricks.NewWorkspaceClient(&databricks.Config{Host: server.URL, Token: "testtoken"}) | ||
| require.NoError(t, err) | ||
| r := (&ResourceApp{}).New(client) | ||
|
|
||
| app, err := r.startApp(t.Context(), "test-app") | ||
| assert.Equal(t, 1, startCalls) | ||
| if tt.wantErr { | ||
| require.ErrorContains(t, err, "Cannot start app test-app") | ||
| return | ||
| } | ||
| require.NoError(t, err) | ||
| assert.Equal(t, apps.ComputeStateActive, app.ComputeStatus.State) | ||
| }) | ||
| } | ||
| } | ||
|
|
||
| // A failed read after a rejected start must stay retryable: the 400 alone is not transient. | ||
| func TestAppStartApp_RejectedStartReadFailureIsTransient(t *testing.T) { | ||
| server := testserver.New(t) | ||
| server.Handle("POST", "/api/2.0/apps/{name}/start", func(req testserver.Request) any { | ||
| return appBadRequest("Cannot start app test-app as its compute is in ACTIVE state.") | ||
| }) | ||
| server.Handle("GET", "/api/2.0/apps/{name}", func(req testserver.Request) any { | ||
| return testserver.Response{ | ||
| StatusCode: 500, | ||
| Body: map[string]string{"error_code": "INTERNAL_ERROR", "message": "boom"}, | ||
| } | ||
| }) | ||
| r := newAppForServer(t, server) | ||
|
|
||
| _, err := r.startApp(t.Context(), "test-app") | ||
| require.ErrorContains(t, err, "Cannot start app test-app") | ||
| require.ErrorContains(t, err, "boom") | ||
| assert.True(t, retry.IsTransient(t.Context(), err)) | ||
| } | ||
|
|
||
| const ( | ||
| appComputeUpdatingDeleteMessage = "Cannot delete app test-app as it is not terminal with state UPDATING, " + | ||
| "and was updated less than 20 minutes ago. Please wait before trying again." | ||
|
|
||
Uh oh!
There was an error while loading. Please reload this page.