Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
* When `serving-endpoints create` or `update-config` fails or times out while waiting, the error now includes each served entity's last deployment state (for example `Container build failed. Cause: user_pip_resolution.`) and the `build-logs` and `logs` commands that show more. ([#7669](https://github.com/databricks/cli/pull/7669))
83 changes: 59 additions & 24 deletions cmd/workspace/serving-endpoints/overrides.go
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,8 @@ type waitProgress struct {
Close()
}
last string
// endpoint is the most recent poll result, used to explain a failed wait.
endpoint *serving.ServingEndpointDetailed
}

func newWaitProgress(ctx context.Context) *waitProgress {
Expand All @@ -43,6 +45,7 @@ func newWaitProgress(ctx context.Context) *waitProgress {
}

func (p *waitProgress) report(ctx context.Context, i *serving.ServingEndpointDetailed) {
p.endpoint = i
msg := endpointProgressMessage(i)
line := fmt.Sprintf("%s (%s)", msg, time.Since(p.start).Truncate(time.Second))
if p.spinner != nil {
Expand All @@ -67,35 +70,67 @@ func (p *waitProgress) close() {
}
}

// explain adds what the last poll said about each served entity to a failed or
// timed-out wait, with the commands that show its logs. The generated error is
// only "failed to reach NOT_UPDATING, got UPDATE_FAILED", even when an entity's
// state already names the cause (for example a pip resolution failure), and in
// a terminal the spinner that showed that state is gone by then.
func (p *waitProgress) explain(endpointName string, err error) error {
var details []string
for _, e := range servedEntities(p.endpoint) {
if e.Name == "" || e.State == nil {
continue
}
state := e.State.DeploymentStateMessage
if state == "" {
state = string(e.State.Deployment)
}
details = append(details,
fmt.Sprintf("%s: %s", e.Name, state),
fmt.Sprintf(" build logs: databricks serving-endpoints build-logs %s %s", endpointName, e.Name),
fmt.Sprintf(" server logs: databricks serving-endpoints logs %s %s", endpointName, e.Name))
}
if len(details) == 0 {
return err
}
return fmt.Errorf("%w\n%s", err, strings.Join(details, "\n"))
}

// servedEntities returns the entities of the pending config, or of the active
// config when nothing is pending.
func servedEntities(i *serving.ServingEndpointDetailed) []serving.ServedEntityOutput {
switch {
case i == nil:
return nil
case i.PendingConfig != nil:
return i.PendingConfig.ServedEntities
case i.Config != nil:
return i.Config.ServedEntities
}
return nil
}

// endpointProgressMessage describes where the endpoint is in its update. It
// prefers the per-entity state of the pending config, falls back to the active
// config, and falls back to the generated "current status" text when no entity
// reports a state.
func endpointProgressMessage(i *serving.ServingEndpointDetailed) string {
var parts []string
if i != nil {
var entities []serving.ServedEntityOutput
if i.PendingConfig != nil {
entities = i.PendingConfig.ServedEntities
} else if i.Config != nil {
entities = i.Config.ServedEntities
}
for _, e := range entities {
if e.State == nil {
continue
}
state := e.State.DeploymentStateMessage
if state == "" {
state = string(e.State.Deployment)
}
if state == "" {
continue
}
if e.Name != "" {
state = e.Name + ": " + state
}
parts = append(parts, state)
for _, e := range servedEntities(i) {
if e.State == nil {
continue
}
state := e.State.DeploymentStateMessage
if state == "" {
state = string(e.State.Deployment)
}
if state == "" {
continue
}
if e.Name != "" {
state = e.Name + ": " + state
}
parts = append(parts, state)
}
if len(parts) > 0 {
return strings.Join(parts, "; ")
Expand Down Expand Up @@ -173,7 +208,7 @@ func createOverride(createCmd *cobra.Command, createReq *serving.CreateServingEn
}).GetWithTimeout(timeout)
p.close()
if err != nil {
return err
return p.explain(createReq.Name, err)
}
return cmdio.Render(ctx, info)
}
Expand Down Expand Up @@ -207,7 +242,7 @@ func updateConfigOverride(updateConfigCmd *cobra.Command, updateConfigReq *servi
}).GetWithTimeout(timeout)
p.close()
if err != nil {
return err
return p.explain(updateConfigReq.Name, err)
}
return cmdio.Render(ctx, info)
}
Expand Down
35 changes: 35 additions & 0 deletions cmd/workspace/serving-endpoints/overrides_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ package serving_endpoints

import (
"bytes"
"errors"
"testing"

"github.com/databricks/cli/libs/cmdio"
Expand Down Expand Up @@ -101,3 +102,37 @@ func TestWaitProgressPrintsChangesWithoutTTY(t *testing.T) {
assert.Equal(t, "iris-1: Container image creation pending (0s)\niris-1: Provisioning (0s)\n", stderr.String())
assert.Empty(t, stdout.String())
}

func TestWaitProgressExplain(t *testing.T) {
waitErr := errors.New("failed to reach NOT_UPDATING, got UPDATE_FAILED: current status: UPDATE_FAILED")

t.Run("adds entity state and log commands", func(t *testing.T) {
// Shape of a real endpoint whose container build failed.
p := &waitProgress{endpoint: &serving.ServingEndpointDetailed{
Name: "my-endpoint",
State: &serving.EndpointState{ConfigUpdate: serving.EndpointStateConfigUpdateUpdateFailed},
PendingConfig: &serving.EndpointPendingConfig{ServedEntities: []serving.ServedEntityOutput{
entity("bad_model-1", serving.ServedModelStateDeploymentDeploymentFailed,
"Container build failed. Cause: user_pip_resolution. See Build Logs for details."),
}},
}}
err := p.explain("my-endpoint", waitErr)
assert.ErrorIs(t, err, waitErr)
assert.Equal(t, waitErr.Error()+`
bad_model-1: Container build failed. Cause: user_pip_resolution. See Build Logs for details.
build logs: databricks serving-endpoints build-logs my-endpoint bad_model-1
server logs: databricks serving-endpoints logs my-endpoint bad_model-1`, err.Error())
})

t.Run("no poll result keeps the error", func(t *testing.T) {
p := &waitProgress{}
assert.Equal(t, waitErr, p.explain("my-endpoint", waitErr))
})

t.Run("entities without state keep the error", func(t *testing.T) {
p := &waitProgress{endpoint: &serving.ServingEndpointDetailed{
PendingConfig: &serving.EndpointPendingConfig{ServedEntities: []serving.ServedEntityOutput{{Name: "m-1"}}},
}}
assert.Equal(t, waitErr, p.explain("my-endpoint", waitErr))
})
}
Loading