From a6aa64482cb9dc01e4cd94d4bbc7dccbb29977b0 Mon Sep 17 00:00:00 2001 From: "denis.bilenko" Date: Thu, 19 Nov 2026 10:12:06 +0000 Subject: [PATCH 1/6] Show status and endpoint for API errors without a message; retry pipeline events reads An error response whose JSON body has no "message" field leaves the SDK's APIError.Message empty, so the CLI printed a bare "Error: ". Print the HTTP status, error_code and endpoint instead. `pipelines run` / `bundle run` aborted when the events API returned a transient 500 ("An unexpected error happened.") while polling an update. Retry that read with libs/retry, as other read paths do. aws: TestAccept/pipelines/run/refresh-flags TestAccept/bundle/deployment/bind/alert Co-authored-by: Isaac --- .../helpers-error/output.txt | 2 +- .../helpers-error/test.toml | 2 - .../cmd/empty-error-message/out.test.toml | 1 + acceptance/cmd/empty-error-message/output.txt | 15 +++++ acceptance/cmd/empty-error-message/script | 4 ++ acceptance/cmd/empty-error-message/test.toml | 18 ++++++ .../pipelines/run/events-500/databricks.yml | 10 ++++ .../pipelines/run/events-500/out.test.toml | 1 + .../pipelines/run/events-500/output.txt | 24 ++++++++ .../pipelines/run/events-500/pipeline_file.py | 7 +++ acceptance/pipelines/run/events-500/script | 5 ++ acceptance/pipelines/run/events-500/test.toml | 1 + bundle/run/progress/pipeline.go | 13 +++-- cmd/root/api_error.go | 56 +++++++++++++++++++ cmd/root/api_error_test.go | 49 ++++++++++++++++ cmd/root/root.go | 1 + 16 files changed, 202 insertions(+), 7 deletions(-) create mode 100644 acceptance/cmd/empty-error-message/out.test.toml create mode 100644 acceptance/cmd/empty-error-message/output.txt create mode 100644 acceptance/cmd/empty-error-message/script create mode 100644 acceptance/cmd/empty-error-message/test.toml create mode 100644 acceptance/pipelines/run/events-500/databricks.yml create mode 100644 acceptance/pipelines/run/events-500/out.test.toml create mode 100644 acceptance/pipelines/run/events-500/output.txt create mode 100644 acceptance/pipelines/run/events-500/pipeline_file.py create mode 100644 acceptance/pipelines/run/events-500/script create mode 100644 acceptance/pipelines/run/events-500/test.toml create mode 100644 cmd/root/api_error.go create mode 100644 cmd/root/api_error_test.go diff --git a/acceptance/bundle/templates-machinery/helpers-error/output.txt b/acceptance/bundle/templates-machinery/helpers-error/output.txt index 6020e944f2d..b0dcd7261a6 100644 --- a/acceptance/bundle/templates-machinery/helpers-error/output.txt +++ b/acceptance/bundle/templates-machinery/helpers-error/output.txt @@ -1,3 +1,3 @@ -Error: failed to compute file content for helpers.txt.tmpl. template: :1:14: executing "" at : error calling user_name: +Error: failed to compute file content for helpers.txt.tmpl. template: :1:14: executing "" at : error calling user_name: API request failed with 500 Internal Server Error from GET /api/2.0/preview/scim/v2/Me Exit code: 1 diff --git a/acceptance/bundle/templates-machinery/helpers-error/test.toml b/acceptance/bundle/templates-machinery/helpers-error/test.toml index 3839635db08..902440a550e 100644 --- a/acceptance/bundle/templates-machinery/helpers-error/test.toml +++ b/acceptance/bundle/templates-machinery/helpers-error/test.toml @@ -1,5 +1,3 @@ -Badness = '''(minor) error message is not great: executing "" at : error calling user_name:''' - [[Server]] Pattern = "GET /api/2.0/preview/scim/v2/Me" Response.Body = '{}' diff --git a/acceptance/cmd/empty-error-message/out.test.toml b/acceptance/cmd/empty-error-message/out.test.toml new file mode 100644 index 00000000000..18b1a88417e --- /dev/null +++ b/acceptance/cmd/empty-error-message/out.test.toml @@ -0,0 +1 @@ +Cloud = false diff --git a/acceptance/cmd/empty-error-message/output.txt b/acceptance/cmd/empty-error-message/output.txt new file mode 100644 index 00000000000..c985471e480 --- /dev/null +++ b/acceptance/cmd/empty-error-message/output.txt @@ -0,0 +1,15 @@ + +>>> errcode [CLI] lakeview create --display-name test +Error: API request failed with 500 Internal Server Error (INTERNAL_ERROR) from POST /api/2.0/lakeview/dashboards + +Exit code: 1 + +>>> errcode [CLI] lakeview get 1234 +Error: API request failed with 404 Not Found from GET /api/2.0/lakeview/dashboards/1234 + +Exit code: 1 + +>>> errcode [CLI] secrets list-scopes +Error: no access + +Exit code: 1 diff --git a/acceptance/cmd/empty-error-message/script b/acceptance/cmd/empty-error-message/script new file mode 100644 index 00000000000..eb50ff01e9d --- /dev/null +++ b/acceptance/cmd/empty-error-message/script @@ -0,0 +1,4 @@ +trace errcode $CLI lakeview create --display-name test +trace errcode $CLI lakeview get 1234 +# A non-empty message is printed as is. +trace errcode $CLI secrets list-scopes diff --git a/acceptance/cmd/empty-error-message/test.toml b/acceptance/cmd/empty-error-message/test.toml new file mode 100644 index 00000000000..cd7c6d28176 --- /dev/null +++ b/acceptance/cmd/empty-error-message/test.toml @@ -0,0 +1,18 @@ +Cloud = false + +# Error responses without a "message" field: the CLI must still say which request failed. + +[[Server]] +Pattern = "POST /api/2.0/lakeview/dashboards" +Response.Body = '{"error_code": "INTERNAL_ERROR"}' +Response.StatusCode = 500 + +[[Server]] +Pattern = "GET /api/2.0/lakeview/dashboards/{dashboard_id}" +Response.Body = '{}' +Response.StatusCode = 404 + +[[Server]] +Pattern = "GET /api/2.0/secrets/scopes/list" +Response.Body = '{"error_code": "PERMISSION_DENIED", "message": "no access"}' +Response.StatusCode = 400 diff --git a/acceptance/pipelines/run/events-500/databricks.yml b/acceptance/pipelines/run/events-500/databricks.yml new file mode 100644 index 00000000000..37e89920ad4 --- /dev/null +++ b/acceptance/pipelines/run/events-500/databricks.yml @@ -0,0 +1,10 @@ +bundle: + name: test-pipeline-run-flags + +resources: + pipelines: + my_pipeline: + name: test-pipeline-flags + libraries: + - file: + path: pipeline_file.py diff --git a/acceptance/pipelines/run/events-500/out.test.toml b/acceptance/pipelines/run/events-500/out.test.toml new file mode 100644 index 00000000000..18b1a88417e --- /dev/null +++ b/acceptance/pipelines/run/events-500/out.test.toml @@ -0,0 +1 @@ +Cloud = false diff --git a/acceptance/pipelines/run/events-500/output.txt b/acceptance/pipelines/run/events-500/output.txt new file mode 100644 index 00000000000..e29bfe84d39 --- /dev/null +++ b/acceptance/pipelines/run/events-500/output.txt @@ -0,0 +1,24 @@ + +>>> [CLI] pipelines deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-pipeline-run-flags/default/files... +Created pipelines.my_pipeline +Files: 2 uploaded, 0 deleted +Resources: 1 created, 0 changed, 0 deleted, 0 unchanged +View your pipeline my_pipeline here: [DATABRICKS_URL]/pipelines/[UUID][0]?w=[WORKSPACE_ID] + +=== A 500 from the events API while polling the update is retried + +>>> [CLI] pipelines run +Recommendation: This command runs the last deployed version of the code + +If you've made local changes, run 'databricks pipelines deploy' first to ensure they are included. + +Update URL: [DATABRICKS_URL]/#joblist/pipelines/[UUID][0]/updates/[UUID][1] + +Warn: retrying after 500 Internal Server Error from GET /api/2.0/pipelines/[UUID][0]/events +Update ID: [UUID][1] + +Update for pipeline completed successfully. + +Pipeline configurations for this update: +• All tables are refreshed diff --git a/acceptance/pipelines/run/events-500/pipeline_file.py b/acceptance/pipelines/run/events-500/pipeline_file.py new file mode 100644 index 00000000000..aa0c44e3f6d --- /dev/null +++ b/acceptance/pipelines/run/events-500/pipeline_file.py @@ -0,0 +1,7 @@ +import dlt +from pyspark.sql import SparkSession + + +@dlt.table +def my_table(): + return spark.range(10) diff --git a/acceptance/pipelines/run/events-500/script b/acceptance/pipelines/run/events-500/script new file mode 100644 index 00000000000..db8636743c3 --- /dev/null +++ b/acceptance/pipelines/run/events-500/script @@ -0,0 +1,5 @@ +trace $CLI pipelines deploy + +title "A 500 from the events API while polling the update is retried" +fault.py "GET /api/2.0/pipelines/*/events" 500 0 1 INTERNAL_ERROR +trace $CLI pipelines run diff --git a/acceptance/pipelines/run/events-500/test.toml b/acceptance/pipelines/run/events-500/test.toml new file mode 100644 index 00000000000..2889b56fb03 --- /dev/null +++ b/acceptance/pipelines/run/events-500/test.toml @@ -0,0 +1 @@ +Env.DATABRICKS_BUNDLE_RETRY_INTERVAL_MS = "100" diff --git a/bundle/run/progress/pipeline.go b/bundle/run/progress/pipeline.go index cce17965da8..1a8d6e1e779 100644 --- a/bundle/run/progress/pipeline.go +++ b/bundle/run/progress/pipeline.go @@ -6,6 +6,7 @@ import ( "slices" "strings" + "github.com/databricks/cli/libs/retry" "github.com/databricks/databricks-sdk-go" "github.com/databricks/databricks-sdk-go/service/pipelines" ) @@ -74,10 +75,14 @@ func (l *UpdateTracker) Events(ctx context.Context) ([]ProgressEvent, error) { } // we only check the most recent 100 events for progress - events, err := l.w.Pipelines.ListPipelineEventsAll(ctx, pipelines.ListPipelineEventsRequest{ - PipelineId: l.PipelineId, - MaxResults: 100, - Filter: filter, + // The events API intermittently fails with a 500 ("An unexpected error happened."). Listing is a + // read, so retry it instead of aborting the whole run. + events, err := retry.With(ctx, retry.IsTransientRead, func() ([]pipelines.PipelineEvent, error) { + return l.w.Pipelines.ListPipelineEventsAll(ctx, pipelines.ListPipelineEventsRequest{ + PipelineId: l.PipelineId, + MaxResults: 100, + Filter: filter, + }) }) if err != nil { return nil, err diff --git a/cmd/root/api_error.go b/cmd/root/api_error.go new file mode 100644 index 00000000000..8625d452ae0 --- /dev/null +++ b/cmd/root/api_error.go @@ -0,0 +1,56 @@ +package root + +import ( + "errors" + "fmt" + "net/http" + "strconv" + "strings" + + "github.com/databricks/databricks-sdk-go/apierr" +) + +// describedError replaces the text of an error while keeping it unwrappable. +type describedError struct { + msg string + err error +} + +func (e *describedError) Error() string { return e.msg } +func (e *describedError) Unwrap() error { return e.err } + +// describeEmptyAPIError fills in API errors whose response had no "message" field. +// The SDK then leaves the message empty and the CLI would print a bare "Error: ", +// which says nothing about which request failed or how. +func describeEmptyAPIError(err error) error { + apiErr, ok := errors.AsType[*apierr.APIError](err) + if !ok || strings.TrimSpace(apiErr.Message) != "" { + return err + } + // Only fill in when the empty message is what ends up printed, e.g. "creating thing: ". + // Callers that already explain the error in their own words are left alone. + prefix := strings.TrimRight(err.Error(), " ") + if prefix != "" && !strings.HasSuffix(prefix, ":") { + return err + } + prefix = strings.TrimSuffix(prefix, ":") + + status := strconv.Itoa(apiErr.StatusCode) + if text := http.StatusText(apiErr.StatusCode); text != "" { + status += " " + text + } + desc := "API request failed with " + status + // The SDK formats a missing error_code as "". + if code := apiErr.ErrorCode; code != "" && code != "" { + desc += " (" + code + ")" + } + if rw := apiErr.ResponseWrapper; rw != nil && rw.Response != nil && rw.Response.Request != nil { + req := rw.Response.Request + desc += fmt.Sprintf(" from %s %s", req.Method, req.URL.Path) + } + + if prefix != "" { + desc = prefix + ": " + desc + } + return &describedError{msg: desc, err: err} +} diff --git a/cmd/root/api_error_test.go b/cmd/root/api_error_test.go new file mode 100644 index 00000000000..28de106736a --- /dev/null +++ b/cmd/root/api_error_test.go @@ -0,0 +1,49 @@ +package root + +import ( + "errors" + "fmt" + "net/http" + "net/url" + "testing" + + "github.com/databricks/databricks-sdk-go/apierr" + "github.com/databricks/databricks-sdk-go/common" + "github.com/stretchr/testify/assert" +) + +func newAPIError(message, code string) *apierr.APIError { + req := &http.Request{Method: http.MethodPost, URL: &url.URL{Path: "/api/2.0/x", RawQuery: "q=1"}} + return &apierr.APIError{ + Message: message, + ErrorCode: code, + StatusCode: 500, + ResponseWrapper: &common.ResponseWrapper{Response: &http.Response{Request: req}}, + } +} + +func TestDescribeEmptyAPIError(t *testing.T) { + for _, tc := range []struct { + name string + err error + want string + }{ + {"not an API error", errors.New("boom"), "boom"}, + {"has a message", newAPIError("no access", "PERMISSION_DENIED"), "no access"}, + {"empty message", newAPIError("", "INTERNAL_ERROR"), "API request failed with 500 Internal Server Error (INTERNAL_ERROR) from POST /api/2.0/x"}, + {"no error code", newAPIError("", ""), "API request failed with 500 Internal Server Error from POST /api/2.0/x"}, + {"wrapped with own explanation", fmt.Errorf("backend failed (500 with no details)%w", newAPIError("", "")), "backend failed (500 with no details)"}, + {"wrapped", fmt.Errorf("creating thing: %w", newAPIError("", "")), "creating thing: API request failed with 500 Internal Server Error from POST /api/2.0/x"}, + } { + t.Run(tc.name, func(t *testing.T) { + got := describeEmptyAPIError(tc.err) + assert.Equal(t, tc.want, got.Error()) + assert.ErrorIs(t, got, tc.err) + }) + } +} + +func TestDescribeEmptyAPIErrorWithoutRequest(t *testing.T) { + err := &apierr.APIError{StatusCode: 499} + assert.Equal(t, "API request failed with 499", describeEmptyAPIError(err).Error()) +} diff --git a/cmd/root/root.go b/cmd/root/root.go index 1a9ca7c62e6..65bc5dc8019 100644 --- a/cmd/root/root.go +++ b/cmd/root/root.go @@ -176,6 +176,7 @@ Stack Trace: // Run the command cmd, err = cmd.ExecuteContextC(ctx) if err != nil && !errors.Is(err, ErrAlreadyPrinted) { + err = describeEmptyAPIError(err) if cmdctx.HasConfigUsed(cmd.Context()) { cfg := cmdctx.ConfigUsed(cmd.Context()) err = auth.EnrichAuthError(cmd.Context(), cfg, err) From 89458de0a3e4545417666a57d83a38140d88b94b Mon Sep 17 00:00:00 2001 From: "denis.bilenko" Date: Thu, 19 Nov 2026 11:06:37 +0000 Subject: [PATCH 2/6] Add changelog fragments aws: TestAccept/pipelines/run/refresh-flags TestAccept/bundle/deployment/bind/alert Co-authored-by: Isaac --- .nextchanges/bundles/pipeline-events-retry.md | 1 + .nextchanges/cli/empty-api-error-message.md | 1 + 2 files changed, 2 insertions(+) create mode 100644 .nextchanges/bundles/pipeline-events-retry.md create mode 100644 .nextchanges/cli/empty-api-error-message.md diff --git a/.nextchanges/bundles/pipeline-events-retry.md b/.nextchanges/bundles/pipeline-events-retry.md new file mode 100644 index 00000000000..d7d2d2de100 --- /dev/null +++ b/.nextchanges/bundles/pipeline-events-retry.md @@ -0,0 +1 @@ +* `bundle run` and `pipelines run` now retry transient errors from the pipeline events API while waiting for an update. ([#7800](https://github.com/databricks/cli/pull/7800)) diff --git a/.nextchanges/cli/empty-api-error-message.md b/.nextchanges/cli/empty-api-error-message.md new file mode 100644 index 00000000000..22338143a6b --- /dev/null +++ b/.nextchanges/cli/empty-api-error-message.md @@ -0,0 +1 @@ +* Show the HTTP status, error code and endpoint when an API error response has no message, instead of a bare `Error: `. ([#7800](https://github.com/databricks/cli/pull/7800)) From 27309acbcb8762886ae8605927562894b8493099 Mon Sep 17 00:00:00 2001 From: "denis.bilenko" Date: Thu, 19 Nov 2026 11:20:04 +0000 Subject: [PATCH 3/6] Describe only bare API errors; retry the other pipeline polling reads Don't branch on the text of wrapping errors: only a bare API error with an empty message is described. Also retry the update status read in the poll loop and the events read after the update completes. aws: TestAccept/pipelines/run/refresh-flags TestAccept/bundle/deployment/bind/alert Co-authored-by: Isaac --- .../helpers-error/output.txt | 2 +- .../helpers-error/test.toml | 2 ++ bundle/run/pipeline.go | 5 ++++- cmd/pipelines/run.go | 11 +++++++---- cmd/root/api_error.go | 19 ++++++------------- cmd/root/api_error_test.go | 4 ++-- 6 files changed, 22 insertions(+), 21 deletions(-) diff --git a/acceptance/bundle/templates-machinery/helpers-error/output.txt b/acceptance/bundle/templates-machinery/helpers-error/output.txt index b0dcd7261a6..6020e944f2d 100644 --- a/acceptance/bundle/templates-machinery/helpers-error/output.txt +++ b/acceptance/bundle/templates-machinery/helpers-error/output.txt @@ -1,3 +1,3 @@ -Error: failed to compute file content for helpers.txt.tmpl. template: :1:14: executing "" at : error calling user_name: API request failed with 500 Internal Server Error from GET /api/2.0/preview/scim/v2/Me +Error: failed to compute file content for helpers.txt.tmpl. template: :1:14: executing "" at : error calling user_name: Exit code: 1 diff --git a/acceptance/bundle/templates-machinery/helpers-error/test.toml b/acceptance/bundle/templates-machinery/helpers-error/test.toml index 902440a550e..3839635db08 100644 --- a/acceptance/bundle/templates-machinery/helpers-error/test.toml +++ b/acceptance/bundle/templates-machinery/helpers-error/test.toml @@ -1,3 +1,5 @@ +Badness = '''(minor) error message is not great: executing "" at : error calling user_name:''' + [[Server]] Pattern = "GET /api/2.0/preview/scim/v2/Me" Response.Body = '{}' diff --git a/bundle/run/pipeline.go b/bundle/run/pipeline.go index ad0860c3356..7960bfee071 100644 --- a/bundle/run/pipeline.go +++ b/bundle/run/pipeline.go @@ -15,6 +15,7 @@ import ( "github.com/databricks/cli/bundle/run/progress" "github.com/databricks/cli/libs/cmdio" "github.com/databricks/cli/libs/log" + "github.com/databricks/cli/libs/retry" "github.com/databricks/databricks-sdk-go/service/pipelines" "github.com/spf13/cobra" ) @@ -154,7 +155,9 @@ func (r *pipelineRunner) Run(ctx context.Context, opts *Options) (output.RunOutp log.Info(ctx, event.String()) } - update, err := w.Pipelines.GetUpdateByPipelineIdAndUpdateId(ctx, pipelineID, updateID) + update, err := retry.With(ctx, retry.IsTransientRead, func() (*pipelines.GetUpdateResponse, error) { + return w.Pipelines.GetUpdateByPipelineIdAndUpdateId(ctx, pipelineID, updateID) + }) if err != nil { return nil, err } diff --git a/cmd/pipelines/run.go b/cmd/pipelines/run.go index cc85f32e6a0..2c219ebcbb0 100644 --- a/cmd/pipelines/run.go +++ b/cmd/pipelines/run.go @@ -24,6 +24,7 @@ import ( "github.com/databricks/cli/libs/cmdio" "github.com/databricks/cli/libs/flags" "github.com/databricks/cli/libs/logdiag" + "github.com/databricks/cli/libs/retry" "github.com/databricks/databricks-sdk-go" "github.com/databricks/databricks-sdk-go/service/pipelines" "github.com/spf13/cobra" @@ -181,10 +182,12 @@ func fetchAndDisplayPipelineUpdate(ctx context.Context, w *databricks.WorkspaceC latestUpdate := *getUpdateResponse.Update // The public events API rejects event_type filters. - events, err := w.Pipelines.ListPipelineEventsAll(ctx, pipelines.ListPipelineEventsRequest{ - PipelineId: pipelineId, - Filter: fmt.Sprintf("update_id='%s'", updateId), - OrderBy: []string{"timestamp asc"}, + events, err := retry.With(ctx, retry.IsTransientRead, func() ([]pipelines.PipelineEvent, error) { + return w.Pipelines.ListPipelineEventsAll(ctx, pipelines.ListPipelineEventsRequest{ + PipelineId: pipelineId, + Filter: fmt.Sprintf("update_id='%s'", updateId), + OrderBy: []string{"timestamp asc"}, + }) }) if err != nil { return err diff --git a/cmd/root/api_error.go b/cmd/root/api_error.go index 8625d452ae0..f8025d32437 100644 --- a/cmd/root/api_error.go +++ b/cmd/root/api_error.go @@ -19,21 +19,18 @@ type describedError struct { func (e *describedError) Error() string { return e.msg } func (e *describedError) Unwrap() error { return e.err } -// describeEmptyAPIError fills in API errors whose response had no "message" field. +// describeEmptyAPIError fills in an API error whose response had no "message" field. // The SDK then leaves the message empty and the CLI would print a bare "Error: ", // which says nothing about which request failed or how. +// +// Only a bare API error is described: wrappers either explain the error in their +// own words or would need their text to be parsed. func describeEmptyAPIError(err error) error { - apiErr, ok := errors.AsType[*apierr.APIError](err) + apiErr := &apierr.APIError{} + ok := errors.As(err, &apiErr) if !ok || strings.TrimSpace(apiErr.Message) != "" { return err } - // Only fill in when the empty message is what ends up printed, e.g. "creating thing: ". - // Callers that already explain the error in their own words are left alone. - prefix := strings.TrimRight(err.Error(), " ") - if prefix != "" && !strings.HasSuffix(prefix, ":") { - return err - } - prefix = strings.TrimSuffix(prefix, ":") status := strconv.Itoa(apiErr.StatusCode) if text := http.StatusText(apiErr.StatusCode); text != "" { @@ -48,9 +45,5 @@ func describeEmptyAPIError(err error) error { req := rw.Response.Request desc += fmt.Sprintf(" from %s %s", req.Method, req.URL.Path) } - - if prefix != "" { - desc = prefix + ": " + desc - } return &describedError{msg: desc, err: err} } diff --git a/cmd/root/api_error_test.go b/cmd/root/api_error_test.go index 28de106736a..0e009d0a231 100644 --- a/cmd/root/api_error_test.go +++ b/cmd/root/api_error_test.go @@ -32,8 +32,8 @@ func TestDescribeEmptyAPIError(t *testing.T) { {"has a message", newAPIError("no access", "PERMISSION_DENIED"), "no access"}, {"empty message", newAPIError("", "INTERNAL_ERROR"), "API request failed with 500 Internal Server Error (INTERNAL_ERROR) from POST /api/2.0/x"}, {"no error code", newAPIError("", ""), "API request failed with 500 Internal Server Error from POST /api/2.0/x"}, - {"wrapped with own explanation", fmt.Errorf("backend failed (500 with no details)%w", newAPIError("", "")), "backend failed (500 with no details)"}, - {"wrapped", fmt.Errorf("creating thing: %w", newAPIError("", "")), "creating thing: API request failed with 500 Internal Server Error from POST /api/2.0/x"}, + {"whitespace message", newAPIError("\n", ""), "API request failed with 500 Internal Server Error from POST /api/2.0/x"}, + {"wrapped", fmt.Errorf("backend failed (500 with no details)%w", newAPIError("", "")), "backend failed (500 with no details)"}, } { t.Run(tc.name, func(t *testing.T) { got := describeEmptyAPIError(tc.err) From 7a0f596dcced7367b5210f9b14a5b4af60cea50b Mon Sep 17 00:00:00 2001 From: "denis.bilenko" Date: Thu, 19 Nov 2026 11:54:05 +0000 Subject: [PATCH 4/6] Keep the bare-error type assertion that lint --fix rewrote to errors.As aws: TestAccept/pipelines/run/refresh-flags TestAccept/bundle/deployment/bind/alert Co-authored-by: Isaac --- cmd/root/api_error.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/cmd/root/api_error.go b/cmd/root/api_error.go index f8025d32437..6ca855a38ee 100644 --- a/cmd/root/api_error.go +++ b/cmd/root/api_error.go @@ -1,7 +1,6 @@ package root import ( - "errors" "fmt" "net/http" "strconv" @@ -26,8 +25,7 @@ func (e *describedError) Unwrap() error { return e.err } // Only a bare API error is described: wrappers either explain the error in their // own words or would need their text to be parsed. func describeEmptyAPIError(err error) error { - apiErr := &apierr.APIError{} - ok := errors.As(err, &apiErr) + apiErr, ok := err.(*apierr.APIError) //nolint:errorlint // only the bare error, see above if !ok || strings.TrimSpace(apiErr.Message) != "" { return err } From df9f9a5726d872fafad129e2f832e857f5327c72 Mon Sep 17 00:00:00 2001 From: "denis.bilenko" Date: Thu, 19 Nov 2026 11:58:10 +0000 Subject: [PATCH 5/6] Describe an API error only when the printed message would be empty Generated commands return the API error wrapped without extra text, so a bare type check missed it. aws: TestAccept/pipelines/run/refresh-flags TestAccept/bundle/deployment/bind/alert Co-authored-by: Isaac --- cmd/root/api_error.go | 12 ++++++++---- cmd/root/api_error_test.go | 3 ++- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/cmd/root/api_error.go b/cmd/root/api_error.go index 6ca855a38ee..6349dac1f09 100644 --- a/cmd/root/api_error.go +++ b/cmd/root/api_error.go @@ -1,6 +1,7 @@ package root import ( + "errors" "fmt" "net/http" "strconv" @@ -22,11 +23,14 @@ func (e *describedError) Unwrap() error { return e.err } // The SDK then leaves the message empty and the CLI would print a bare "Error: ", // which says nothing about which request failed or how. // -// Only a bare API error is described: wrappers either explain the error in their -// own words or would need their text to be parsed. +// Only an error that would print as nothing at all is described; wrappers that add +// their own text already say something about the failure. func describeEmptyAPIError(err error) error { - apiErr, ok := err.(*apierr.APIError) //nolint:errorlint // only the bare error, see above - if !ok || strings.TrimSpace(apiErr.Message) != "" { + if strings.TrimSpace(err.Error()) != "" { + return err + } + apiErr, ok := errors.AsType[*apierr.APIError](err) + if !ok { return err } diff --git a/cmd/root/api_error_test.go b/cmd/root/api_error_test.go index 0e009d0a231..da8b53c24b4 100644 --- a/cmd/root/api_error_test.go +++ b/cmd/root/api_error_test.go @@ -33,7 +33,8 @@ func TestDescribeEmptyAPIError(t *testing.T) { {"empty message", newAPIError("", "INTERNAL_ERROR"), "API request failed with 500 Internal Server Error (INTERNAL_ERROR) from POST /api/2.0/x"}, {"no error code", newAPIError("", ""), "API request failed with 500 Internal Server Error from POST /api/2.0/x"}, {"whitespace message", newAPIError("\n", ""), "API request failed with 500 Internal Server Error from POST /api/2.0/x"}, - {"wrapped", fmt.Errorf("backend failed (500 with no details)%w", newAPIError("", "")), "backend failed (500 with no details)"}, + {"wrapped with own text", fmt.Errorf("creating thing: %w", newAPIError("", "")), "creating thing: "}, + {"wrapped without own text", fmt.Errorf("%w", newAPIError("", "INTERNAL_ERROR")), "API request failed with 500 Internal Server Error (INTERNAL_ERROR) from POST /api/2.0/x"}, } { t.Run(tc.name, func(t *testing.T) { got := describeEmptyAPIError(tc.err) From 265587f8404c0c099987b555376114a299553350 Mon Sep 17 00:00:00 2001 From: "denis.bilenko" Date: Thu, 19 Nov 2026 13:46:03 +0000 Subject: [PATCH 6/6] Address review: share API error formatting, retry remaining pipeline reads - Share the status and endpoint formatting between libs/retry and cmd/root via libs/diag. - Retry the error-events read and the post-run GetUpdate too. - Assert the retry warning in pipelines/run/events-500. - Mention update status reads in the changelog entry. aws: TestAccept/pipelines/run/refresh-flags Co-authored-by: Isaac --- .nextchanges/bundles/pipeline-events-retry.md | 2 +- acceptance/pipelines/run/events-500/test.toml | 2 ++ bundle/run/pipeline.go | 10 +++++---- cmd/pipelines/run.go | 8 ++++--- cmd/root/api_error.go | 15 ++++--------- libs/diag/sdk_error.go | 22 +++++++++++++++++++ libs/retry/retry.go | 12 ++++------ 7 files changed, 44 insertions(+), 27 deletions(-) diff --git a/.nextchanges/bundles/pipeline-events-retry.md b/.nextchanges/bundles/pipeline-events-retry.md index d7d2d2de100..ebf063eafc6 100644 --- a/.nextchanges/bundles/pipeline-events-retry.md +++ b/.nextchanges/bundles/pipeline-events-retry.md @@ -1 +1 @@ -* `bundle run` and `pipelines run` now retry transient errors from the pipeline events API while waiting for an update. ([#7800](https://github.com/databricks/cli/pull/7800)) +* `bundle run` and `pipelines run` now retry transient errors from the pipeline events and update status reads while waiting for an update. ([#7800](https://github.com/databricks/cli/pull/7800)) diff --git a/acceptance/pipelines/run/events-500/test.toml b/acceptance/pipelines/run/events-500/test.toml index 2889b56fb03..867aee20983 100644 --- a/acceptance/pipelines/run/events-500/test.toml +++ b/acceptance/pipelines/run/events-500/test.toml @@ -1 +1,3 @@ Env.DATABRICKS_BUNDLE_RETRY_INTERVAL_MS = "100" +# Assert the retry warning, which acceptance/test.toml suppresses by default. +SuppressLines.warn_retry = "" diff --git a/bundle/run/pipeline.go b/bundle/run/pipeline.go index 7960bfee071..6a7ddf49ff7 100644 --- a/bundle/run/pipeline.go +++ b/bundle/run/pipeline.go @@ -59,10 +59,12 @@ func (r *pipelineRunner) logErrorEvent(ctx context.Context, pipelineId, updateId // Otherwise for long lived pipelines, there can be a lot of unnecessary // latency due to multiple pagination API calls needed underneath the hood for // ListPipelineEventsAll - events, err := w.Pipelines.ListPipelineEventsAll(ctx, pipelines.ListPipelineEventsRequest{ - Filter: `level='ERROR'`, - MaxResults: 100, - PipelineId: pipelineId, + events, err := retry.With(ctx, retry.IsTransientRead, func() ([]pipelines.PipelineEvent, error) { + return w.Pipelines.ListPipelineEventsAll(ctx, pipelines.ListPipelineEventsRequest{ + Filter: `level='ERROR'`, + MaxResults: 100, + PipelineId: pipelineId, + }) }) if err != nil { return err diff --git a/cmd/pipelines/run.go b/cmd/pipelines/run.go index 2c219ebcbb0..e4edae8f182 100644 --- a/cmd/pipelines/run.go +++ b/cmd/pipelines/run.go @@ -167,9 +167,11 @@ func fetchAndDisplayPipelineUpdate(ctx context.Context, w *databricks.WorkspaceC return errors.New("no update ID provided") } - getUpdateResponse, err := w.Pipelines.GetUpdate(ctx, pipelines.GetUpdateRequest{ - PipelineId: pipelineId, - UpdateId: updateId, + getUpdateResponse, err := retry.With(ctx, retry.IsTransientRead, func() (*pipelines.GetUpdateResponse, error) { + return w.Pipelines.GetUpdate(ctx, pipelines.GetUpdateRequest{ + PipelineId: pipelineId, + UpdateId: updateId, + }) }) if err != nil { return err diff --git a/cmd/root/api_error.go b/cmd/root/api_error.go index 6349dac1f09..29f20ae9ba3 100644 --- a/cmd/root/api_error.go +++ b/cmd/root/api_error.go @@ -2,11 +2,9 @@ package root import ( "errors" - "fmt" - "net/http" - "strconv" "strings" + "github.com/databricks/cli/libs/diag" "github.com/databricks/databricks-sdk-go/apierr" ) @@ -34,18 +32,13 @@ func describeEmptyAPIError(err error) error { return err } - status := strconv.Itoa(apiErr.StatusCode) - if text := http.StatusText(apiErr.StatusCode); text != "" { - status += " " + text - } - desc := "API request failed with " + status + desc := "API request failed with " + diag.FormatAPIErrorStatus(apiErr) // The SDK formats a missing error_code as "". if code := apiErr.ErrorCode; code != "" && code != "" { desc += " (" + code + ")" } - if rw := apiErr.ResponseWrapper; rw != nil && rw.Response != nil && rw.Response.Request != nil { - req := rw.Response.Request - desc += fmt.Sprintf(" from %s %s", req.Method, req.URL.Path) + if endpoint := diag.FormatAPIErrorEndpoint(apiErr); endpoint != "" { + desc += " from " + endpoint } return &describedError{msg: desc, err: err} } diff --git a/libs/diag/sdk_error.go b/libs/diag/sdk_error.go index d190498d6bf..c0183190882 100644 --- a/libs/diag/sdk_error.go +++ b/libs/diag/sdk_error.go @@ -3,6 +3,7 @@ package diag import ( "errors" "fmt" + "net/http" "strconv" "strings" @@ -42,3 +43,24 @@ func FormatAPIErrorDetails(e error) string { } return fmt.Sprintf("Endpoint: %s\nHTTP Status: %s\nAPI error_code: %s\nAPI message: %s", endpoint, httpStatus, apiErr.ErrorCode, apiErr.Message) } + +// FormatAPIErrorStatus returns the HTTP status of an API error, e.g. "500 Internal Server Error". +func FormatAPIErrorStatus(apiErr *apierr.APIError) string { + status := strconv.Itoa(apiErr.StatusCode) + // Non-standard codes such as 499 have no status text. + if text := http.StatusText(apiErr.StatusCode); text != "" { + status += " " + text + } + return status +} + +// FormatAPIErrorEndpoint returns the method and path of the request that failed, e.g. +// "GET /api/2.0/clusters/get", or "" if the error carries no request. +func FormatAPIErrorEndpoint(apiErr *apierr.APIError) string { + rw := apiErr.ResponseWrapper + if rw == nil || rw.Response == nil || rw.Response.Request == nil { + return "" + } + req := rw.Response.Request + return req.Method + " " + req.URL.Path +} diff --git a/libs/retry/retry.go b/libs/retry/retry.go index b575e67309a..4c32476dfb9 100644 --- a/libs/retry/retry.go +++ b/libs/retry/retry.go @@ -12,6 +12,7 @@ import ( "strings" "time" + "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/env" "github.com/databricks/cli/libs/log" "github.com/databricks/databricks-sdk-go/apierr" @@ -132,15 +133,10 @@ func retryMessage(err error) string { msg := "retrying after " + err.Error() if apiErr, ok := errors.AsType[*apierr.APIError](err); ok { endpoint := "" - if rw := apiErr.ResponseWrapper; rw != nil && rw.Response != nil && rw.Response.Request != nil { - req := rw.Response.Request - endpoint = fmt.Sprintf(" from %s %s", req.Method, req.URL.Path) - } - status := strconv.Itoa(apiErr.StatusCode) - // Non-standard codes such as 499 have no status text. - if text := http.StatusText(apiErr.StatusCode); text != "" { - status += " " + text + if e := diag.FormatAPIErrorEndpoint(apiErr); e != "" { + endpoint = " from " + e } + status := diag.FormatAPIErrorStatus(apiErr) msg = fmt.Sprintf("retrying after %s%s", status, endpoint) // Include the context the caller added to the error, e.g. the path of the file being written. if err.Error() != apiErr.Error() {