From a4811e8a7ac560e877caf122e1969a30a197d7c8 Mon Sep 17 00:00:00 2001 From: Anshul Gupta Date: Mon, 16 Nov 2026 08:00:21 +0000 Subject: [PATCH 1/2] serving-endpoints: explain a failed create or update-config wait When the endpoint update failed, the wait only returned "failed to reach NOT_UPDATING, got UPDATE_FAILED: current status: UPDATE_FAILED", even though the served entity's state already named the cause (for example "Container build failed. Cause: user_pip_resolution. See Build Logs for details."). In a terminal, the spinner that showed that state is gone by then. Add each served entity's last state to the error, with the build-logs and logs commands for it. The original error stays first and is still wrapped, and the exit code is unchanged. Co-authored-by: Isaac Signed-off-by: Anshul Gupta --- .../serving-endpoints-wait-failure-details.md | 1 + cmd/workspace/serving-endpoints/overrides.go | 83 +++++++++++++------ .../serving-endpoints/overrides_test.go | 35 ++++++++ 3 files changed, 95 insertions(+), 24 deletions(-) create mode 100644 .nextchanges/cli/serving-endpoints-wait-failure-details.md diff --git a/.nextchanges/cli/serving-endpoints-wait-failure-details.md b/.nextchanges/cli/serving-endpoints-wait-failure-details.md new file mode 100644 index 00000000000..cb9090d4658 --- /dev/null +++ b/.nextchanges/cli/serving-endpoints-wait-failure-details.md @@ -0,0 +1 @@ +* When `serving-endpoints create` or `update-config` fails or times out while waiting, the error now includes each served entity's last deployment state (for example `Container build failed. Cause: user_pip_resolution.`) and the `build-logs` and `logs` commands that show more. diff --git a/cmd/workspace/serving-endpoints/overrides.go b/cmd/workspace/serving-endpoints/overrides.go index 47551f14507..b1dc2ba5177 100644 --- a/cmd/workspace/serving-endpoints/overrides.go +++ b/cmd/workspace/serving-endpoints/overrides.go @@ -32,6 +32,8 @@ type waitProgress struct { Close() } last string + // endpoint is the most recent poll result, used to explain a failed wait. + endpoint *serving.ServingEndpointDetailed } func newWaitProgress(ctx context.Context) *waitProgress { @@ -43,6 +45,7 @@ func newWaitProgress(ctx context.Context) *waitProgress { } func (p *waitProgress) report(ctx context.Context, i *serving.ServingEndpointDetailed) { + p.endpoint = i msg := endpointProgressMessage(i) line := fmt.Sprintf("%s (%s)", msg, time.Since(p.start).Truncate(time.Second)) if p.spinner != nil { @@ -67,35 +70,67 @@ func (p *waitProgress) close() { } } +// explain adds what the last poll said about each served entity to a failed or +// timed-out wait, with the commands that show its logs. The generated error is +// only "failed to reach NOT_UPDATING, got UPDATE_FAILED", even when an entity's +// state already names the cause (for example a pip resolution failure), and in +// a terminal the spinner that showed that state is gone by then. +func (p *waitProgress) explain(endpointName string, err error) error { + var details []string + for _, e := range servedEntities(p.endpoint) { + if e.Name == "" || e.State == nil { + continue + } + state := e.State.DeploymentStateMessage + if state == "" { + state = string(e.State.Deployment) + } + details = append(details, + fmt.Sprintf("%s: %s", e.Name, state), + fmt.Sprintf(" build logs: databricks serving-endpoints build-logs %s %s", endpointName, e.Name), + fmt.Sprintf(" server logs: databricks serving-endpoints logs %s %s", endpointName, e.Name)) + } + if len(details) == 0 { + return err + } + return fmt.Errorf("%w\n%s", err, strings.Join(details, "\n")) +} + +// servedEntities returns the entities of the pending config, or of the active +// config when nothing is pending. +func servedEntities(i *serving.ServingEndpointDetailed) []serving.ServedEntityOutput { + switch { + case i == nil: + return nil + case i.PendingConfig != nil: + return i.PendingConfig.ServedEntities + case i.Config != nil: + return i.Config.ServedEntities + } + return nil +} + // endpointProgressMessage describes where the endpoint is in its update. It // prefers the per-entity state of the pending config, falls back to the active // config, and falls back to the generated "current status" text when no entity // reports a state. func endpointProgressMessage(i *serving.ServingEndpointDetailed) string { var parts []string - if i != nil { - var entities []serving.ServedEntityOutput - if i.PendingConfig != nil { - entities = i.PendingConfig.ServedEntities - } else if i.Config != nil { - entities = i.Config.ServedEntities - } - for _, e := range entities { - if e.State == nil { - continue - } - state := e.State.DeploymentStateMessage - if state == "" { - state = string(e.State.Deployment) - } - if state == "" { - continue - } - if e.Name != "" { - state = e.Name + ": " + state - } - parts = append(parts, state) + for _, e := range servedEntities(i) { + if e.State == nil { + continue + } + state := e.State.DeploymentStateMessage + if state == "" { + state = string(e.State.Deployment) } + if state == "" { + continue + } + if e.Name != "" { + state = e.Name + ": " + state + } + parts = append(parts, state) } if len(parts) > 0 { return strings.Join(parts, "; ") @@ -173,7 +208,7 @@ func createOverride(createCmd *cobra.Command, createReq *serving.CreateServingEn }).GetWithTimeout(timeout) p.close() if err != nil { - return err + return p.explain(createReq.Name, err) } return cmdio.Render(ctx, info) } @@ -207,7 +242,7 @@ func updateConfigOverride(updateConfigCmd *cobra.Command, updateConfigReq *servi }).GetWithTimeout(timeout) p.close() if err != nil { - return err + return p.explain(updateConfigReq.Name, err) } return cmdio.Render(ctx, info) } diff --git a/cmd/workspace/serving-endpoints/overrides_test.go b/cmd/workspace/serving-endpoints/overrides_test.go index b539418628a..10d1940477e 100644 --- a/cmd/workspace/serving-endpoints/overrides_test.go +++ b/cmd/workspace/serving-endpoints/overrides_test.go @@ -2,6 +2,7 @@ package serving_endpoints import ( "bytes" + "errors" "testing" "github.com/databricks/cli/libs/cmdio" @@ -101,3 +102,37 @@ func TestWaitProgressPrintsChangesWithoutTTY(t *testing.T) { assert.Equal(t, "iris-1: Container image creation pending (0s)\niris-1: Provisioning (0s)\n", stderr.String()) assert.Empty(t, stdout.String()) } + +func TestWaitProgressExplain(t *testing.T) { + waitErr := errors.New("failed to reach NOT_UPDATING, got UPDATE_FAILED: current status: UPDATE_FAILED") + + t.Run("adds entity state and log commands", func(t *testing.T) { + // Shape of a real endpoint whose container build failed. + p := &waitProgress{endpoint: &serving.ServingEndpointDetailed{ + Name: "my-endpoint", + State: &serving.EndpointState{ConfigUpdate: serving.EndpointStateConfigUpdateUpdateFailed}, + PendingConfig: &serving.EndpointPendingConfig{ServedEntities: []serving.ServedEntityOutput{ + entity("bad_model-1", serving.ServedModelStateDeploymentDeploymentFailed, + "Container build failed. Cause: user_pip_resolution. See Build Logs for details."), + }}, + }} + err := p.explain("my-endpoint", waitErr) + assert.ErrorIs(t, err, waitErr) + assert.Equal(t, waitErr.Error()+` +bad_model-1: Container build failed. Cause: user_pip_resolution. See Build Logs for details. + build logs: databricks serving-endpoints build-logs my-endpoint bad_model-1 + server logs: databricks serving-endpoints logs my-endpoint bad_model-1`, err.Error()) + }) + + t.Run("no poll result keeps the error", func(t *testing.T) { + p := &waitProgress{} + assert.Equal(t, waitErr, p.explain("my-endpoint", waitErr)) + }) + + t.Run("entities without state keep the error", func(t *testing.T) { + p := &waitProgress{endpoint: &serving.ServingEndpointDetailed{ + PendingConfig: &serving.EndpointPendingConfig{ServedEntities: []serving.ServedEntityOutput{{Name: "m-1"}}}, + }} + assert.Equal(t, waitErr, p.explain("my-endpoint", waitErr)) + }) +} From cfa2897cde211917d411cb9b1c4c9f992dfe3533 Mon Sep 17 00:00:00 2001 From: Anshul Gupta Date: Mon, 16 Nov 2026 08:00:47 +0000 Subject: [PATCH 2/2] Point changelog fragment at #7669 Co-authored-by: Isaac Signed-off-by: Anshul Gupta --- .nextchanges/cli/serving-endpoints-wait-failure-details.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.nextchanges/cli/serving-endpoints-wait-failure-details.md b/.nextchanges/cli/serving-endpoints-wait-failure-details.md index cb9090d4658..ffba025fa44 100644 --- a/.nextchanges/cli/serving-endpoints-wait-failure-details.md +++ b/.nextchanges/cli/serving-endpoints-wait-failure-details.md @@ -1 +1 @@ -* When `serving-endpoints create` or `update-config` fails or times out while waiting, the error now includes each served entity's last deployment state (for example `Container build failed. Cause: user_pip_resolution.`) and the `build-logs` and `logs` commands that show more. +* When `serving-endpoints create` or `update-config` fails or times out while waiting, the error now includes each served entity's last deployment state (for example `Container build failed. Cause: user_pip_resolution.`) and the `build-logs` and `logs` commands that show more. ([#7669](https://github.com/databricks/cli/pull/7669))