Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .nextchanges/bundles/app-start-idempotent.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
* Treat starting an app whose compute is already active or starting as success in the direct deployment engine. A retried deploy after a transient error no longer fails with "Cannot start app ... as its compute is in ACTIVE state". ([#7709](https://github.com/databricks/cli/pull/7709))
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
import http.server, os

http.server.HTTPServer(
("", int(os.environ["DATABRICKS_APP_PORT"])), http.server.SimpleHTTPRequestHandler
).serve_forever()
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
bundle:
name: lifecycle-start-retried

resources:
apps:
mykey:
name: myapp
description: my_app_description
source_code_path: ./app
lifecycle:
started: false
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
Cloud = false
EnvMatrix.DMS = ["", "true"]
Original file line number Diff line number Diff line change
@@ -0,0 +1,50 @@

>>> [CLI] bundle deploy
Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/lifecycle-start-retried/default/files...
Created apps.mykey
Files: 2 uploaded, 0 deleted
Resources: 1 created, 0 changed, 0 deleted, 0 unchanged

=== the deploy succeeds, though its start was answered with a 504

>>> update_file.py databricks.yml started: false started: true

>>> [CLI] bundle deploy
Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/lifecycle-start-retried/default/files...
✓ Deployment succeeded
Updated apps.mykey
Files: 1 uploaded, 0 deleted
Resources: 0 created, 1 changed, 0 deleted, 0 unchanged

=== start was sent twice; the second was rejected because the compute was already ACTIVE

>>> print_requests.py //apps ^//api/2.0/bundle --del-body url
{
"method": "POST",
"path": "/api/2.0/apps/myapp/start",
"body": {}
}
{
"method": "POST",
"path": "/api/2.0/apps/myapp/start",
"body": {}
}
{
"method": "POST",
"path": "/api/2.0/apps/myapp/deployments",
"body": {
"mode": "SNAPSHOT",
"source_code_path": "/Workspace/Users/[USERNAME]/.bundle/lifecycle-start-retried/default/files/app"
}
}

>>> [CLI] apps get myapp
"ACTIVE"

>>> [CLI] bundle destroy --auto-approve
The following resources will be deleted:
delete resources.apps.mykey

All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/lifecycle-start-retried/default

Destroy: 1 deleted
20 changes: 20 additions & 0 deletions acceptance/bundle/resources/apps/lifecycle-start-retried/script
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
cleanup() {
trace $CLI bundle destroy --auto-approve
rm -f "$OUT_REQUESTS"
}
trap cleanup EXIT

trace $CLI bundle deploy
rm -f "$OUT_REQUESTS"

# Keep the start the handler applied, replace only the response with a 504, so the start
# is sent again while the compute is already ACTIVE.
fault.py --after-handler "POST /api/2.0/apps/*/start" 504 0 1

title "the deploy succeeds, though its start was answered with a 504"
trace update_file.py databricks.yml "started: false" "started: true"
trace $CLI bundle deploy

title "start was sent twice; the second was rejected because the compute was already ACTIVE"
trace print_requests.py //apps '^//api/2.0/bundle' --del-body url
trace $CLI apps get myapp | jq '.compute_status.state'
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
# Local only: the lost response is staged by injecting a fault into the testserver.
Cloud = false
Comment thread
denik marked this conversation as resolved.
Resolved
RecordRequests = true
Ignore = [".databricks", "databricks.yml"]
Env.DATABRICKS_BUNDLE_RETRY_INTERVAL_MS = "100"
44 changes: 39 additions & 5 deletions bundle/direct/dresources/app.go
Original file line number Diff line number Diff line change
Expand Up @@ -272,11 +272,7 @@ func (r *ResourceApp) manageLifecycle(ctx context.Context, id string, config *Ap
if desiredStarted {
// lifecycle.started=true: ensure the app compute is running and deploy the latest code.
if !alreadyStarted {
startWaiter, err := r.client.Apps.Start(ctx, apps.StartAppRequest{Name: id})
if err != nil {
return err
}
startedApp, err := startWaiter.Get()
startedApp, err := r.startApp(ctx, id)
if err != nil {
return err
}
Expand Down Expand Up @@ -307,6 +303,44 @@ func (r *ResourceApp) manageLifecycle(ctx context.Context, id string, config *Ap
return nil
}

// startApp starts the app compute and waits for it to become active. A start of an app whose
// compute is not STOPPED is rejected with 400 BAD_REQUEST ("Cannot start app X as its compute
// is in ACTIVE state"). That happens when apply retries DoUpdate after a transient error in the
// start waiter: the first start already went through, so the compute is ACTIVE, or still
// STARTING if the retry came before it finished.
func (r *ResourceApp) startApp(ctx context.Context, id string) (*apps.App, error) {

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We have a similar logic in bundle/run/app.go function start, can we consolidate in single helper function maybe?

startWaiter, err := r.client.Apps.Start(ctx, apps.StartAppRequest{Name: id})
if err == nil {
return startWaiter.Get()
}
apiErr, ok := errors.AsType[*apierr.APIError](err)
if !ok || apiErr.StatusCode != http.StatusBadRequest {
return nil, err
}
app, getErr := r.client.Apps.GetByName(ctx, id)
if getErr != nil {
// Wrap only getErr so that apply's retry sees it: a transient read failure leaves
// the compute state unknown, and the 400 alone would make the failure permanent.
return nil, fmt.Errorf("reading app after start was rejected (%s): %w", err.Error(), getErr)
}
if app.ComputeStatus == nil {
return nil, err
}
switch app.ComputeStatus.State {
case apps.ComputeStateActive:
log.Debugf(ctx, "app %s compute is already active, ignoring start error: %s", id, err)
return app, nil
case apps.ComputeStateStarting:
log.Debugf(ctx, "app %s compute is already starting, waiting for it instead: %s", id, err)
return r.client.Apps.WaitGetAppActive(ctx, id, appStartTimeout, nil)
default:
return nil, err
}
}

// appStartTimeout matches the SDK's default wait for Apps.Start.
const appStartTimeout = 20 * time.Minute

// hasAppChanges reports whether the plan entry contains any Update changes to fields that
// belong to the App Update API. source_code_path and config (command/env) are deploy-only
// on update and applied via manageLifecycle; lifecycle drives compute start/stop.
Expand Down
79 changes: 79 additions & 0 deletions bundle/direct/dresources/app_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -298,6 +298,85 @@ func TestWithMissingServicePrincipalHint(t *testing.T) {
}
}

// A retried DoUpdate starts the app a second time after the first start went through, and the
// backend rejects the second start. An ACTIVE compute makes that rejection a success, and a
// STARTING one is waited for; any other state keeps the error.
func TestAppStartApp_RejectedStart(t *testing.T) {
tests := []struct {
state apps.ComputeState
wantErr bool
}{
{state: apps.ComputeStateActive, wantErr: false},
{state: apps.ComputeStateStarting, wantErr: false},
{state: apps.ComputeStateStopped, wantErr: true},
{state: apps.ComputeStateDeleting, wantErr: true},
{state: "", wantErr: true},
}
for _, tt := range tests {
t.Run(string(tt.state), func(t *testing.T) {
server := testserver.New(t)
startCalls := 0
server.Handle("POST", "/api/2.0/apps/{name}/start", func(req testserver.Request) any {
startCalls++
return testserver.Response{

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[AI Generated] nit: nit: this builds the 400 response and the client inline, while the next test uses appBadRequest and newAppForServer from this file. Can this one use them too?

StatusCode: 400,
Body: map[string]string{
"error_code": "BAD_REQUEST",
"message": "Cannot start app " + req.Vars["name"] + " as its compute is in ACTIVE state.",
},
}
})
getCalls := 0
server.Handle("GET", "/api/2.0/apps/{name}", func(req testserver.Request) any {
getCalls++
app := apps.App{Name: req.Vars["name"]}
state := tt.state
// A STARTING compute becomes ACTIVE on the next poll.
if state == apps.ComputeStateStarting && getCalls > 1 {
state = apps.ComputeStateActive
}
if state != "" {
app.ComputeStatus = &apps.ComputeStatus{State: state}
}
return app
})
testserver.AddDefaultHandlers(server)
client, err := databricks.NewWorkspaceClient(&databricks.Config{Host: server.URL, Token: "testtoken"})
require.NoError(t, err)
r := (&ResourceApp{}).New(client)

app, err := r.startApp(t.Context(), "test-app")
assert.Equal(t, 1, startCalls)
if tt.wantErr {
require.ErrorContains(t, err, "Cannot start app test-app")
return
}
require.NoError(t, err)
assert.Equal(t, apps.ComputeStateActive, app.ComputeStatus.State)
})
}
}

// A failed read after a rejected start must stay retryable: the 400 alone is not transient.
func TestAppStartApp_RejectedStartReadFailureIsTransient(t *testing.T) {
server := testserver.New(t)
server.Handle("POST", "/api/2.0/apps/{name}/start", func(req testserver.Request) any {
return appBadRequest("Cannot start app test-app as its compute is in ACTIVE state.")
})
server.Handle("GET", "/api/2.0/apps/{name}", func(req testserver.Request) any {
return testserver.Response{
StatusCode: 500,
Body: map[string]string{"error_code": "INTERNAL_ERROR", "message": "boom"},
}
})
r := newAppForServer(t, server)

_, err := r.startApp(t.Context(), "test-app")
require.ErrorContains(t, err, "Cannot start app test-app")
require.ErrorContains(t, err, "boom")
assert.True(t, retry.IsTransient(t.Context(), err))
}

const (
appComputeUpdatingDeleteMessage = "Cannot delete app test-app as it is not terminal with state UPDATING, " +
"and was updated less than 20 minutes ago. Please wait before trying again."
Expand Down
10 changes: 10 additions & 0 deletions libs/testserver/apps.go
Original file line number Diff line number Diff line change
Expand Up @@ -208,6 +208,16 @@ func (s *FakeWorkspace) AppsStart(_ Request, name string) Response {
return Response{StatusCode: 404}
}

if app.ComputeStatus != nil && app.ComputeStatus.State == apps.ComputeStateActive {
Comment thread
denik marked this conversation as resolved.
Resolved
return Response{
StatusCode: 400,
Body: map[string]string{
"error_code": "BAD_REQUEST",
"message": fmt.Sprintf("Cannot start app %s as its compute is in ACTIVE state. App compute needs to be STOPPED to start.", name),
},
}
}

app.ComputeStatus = &apps.ComputeStatus{
State: apps.ComputeStateActive,
Message: "App compute is active.",
Expand Down
Loading