diff --git a/.nextchanges/bundles/pipeline-events-retry.md b/.nextchanges/bundles/pipeline-events-retry.md new file mode 100644 index 00000000000..ebf063eafc6 --- /dev/null +++ b/.nextchanges/bundles/pipeline-events-retry.md @@ -0,0 +1 @@ +* `bundle run` and `pipelines run` now retry transient errors from the pipeline events and update status reads while waiting for an update. ([#7800](https://github.com/databricks/cli/pull/7800)) diff --git a/.nextchanges/cli/empty-api-error-message.md b/.nextchanges/cli/empty-api-error-message.md new file mode 100644 index 00000000000..22338143a6b --- /dev/null +++ b/.nextchanges/cli/empty-api-error-message.md @@ -0,0 +1 @@ +* Show the HTTP status, error code and endpoint when an API error response has no message, instead of a bare `Error: `. ([#7800](https://github.com/databricks/cli/pull/7800)) diff --git a/acceptance/cmd/empty-error-message/out.test.toml b/acceptance/cmd/empty-error-message/out.test.toml new file mode 100644 index 00000000000..18b1a88417e --- /dev/null +++ b/acceptance/cmd/empty-error-message/out.test.toml @@ -0,0 +1 @@ +Cloud = false diff --git a/acceptance/cmd/empty-error-message/output.txt b/acceptance/cmd/empty-error-message/output.txt new file mode 100644 index 00000000000..c985471e480 --- /dev/null +++ b/acceptance/cmd/empty-error-message/output.txt @@ -0,0 +1,15 @@ + +>>> errcode [CLI] lakeview create --display-name test +Error: API request failed with 500 Internal Server Error (INTERNAL_ERROR) from POST /api/2.0/lakeview/dashboards + +Exit code: 1 + +>>> errcode [CLI] lakeview get 1234 +Error: API request failed with 404 Not Found from GET /api/2.0/lakeview/dashboards/1234 + +Exit code: 1 + +>>> errcode [CLI] secrets list-scopes +Error: no access + +Exit code: 1 diff --git a/acceptance/cmd/empty-error-message/script b/acceptance/cmd/empty-error-message/script new file mode 100644 index 00000000000..eb50ff01e9d --- /dev/null +++ b/acceptance/cmd/empty-error-message/script @@ -0,0 +1,4 @@ +trace errcode $CLI lakeview create --display-name test +trace errcode $CLI lakeview get 1234 +# A non-empty message is printed as is. +trace errcode $CLI secrets list-scopes diff --git a/acceptance/cmd/empty-error-message/test.toml b/acceptance/cmd/empty-error-message/test.toml new file mode 100644 index 00000000000..cd7c6d28176 --- /dev/null +++ b/acceptance/cmd/empty-error-message/test.toml @@ -0,0 +1,18 @@ +Cloud = false + +# Error responses without a "message" field: the CLI must still say which request failed. + +[[Server]] +Pattern = "POST /api/2.0/lakeview/dashboards" +Response.Body = '{"error_code": "INTERNAL_ERROR"}' +Response.StatusCode = 500 + +[[Server]] +Pattern = "GET /api/2.0/lakeview/dashboards/{dashboard_id}" +Response.Body = '{}' +Response.StatusCode = 404 + +[[Server]] +Pattern = "GET /api/2.0/secrets/scopes/list" +Response.Body = '{"error_code": "PERMISSION_DENIED", "message": "no access"}' +Response.StatusCode = 400 diff --git a/acceptance/pipelines/run/events-500/databricks.yml b/acceptance/pipelines/run/events-500/databricks.yml new file mode 100644 index 00000000000..37e89920ad4 --- /dev/null +++ b/acceptance/pipelines/run/events-500/databricks.yml @@ -0,0 +1,10 @@ +bundle: + name: test-pipeline-run-flags + +resources: + pipelines: + my_pipeline: + name: test-pipeline-flags + libraries: + - file: + path: pipeline_file.py diff --git a/acceptance/pipelines/run/events-500/out.test.toml b/acceptance/pipelines/run/events-500/out.test.toml new file mode 100644 index 00000000000..18b1a88417e --- /dev/null +++ b/acceptance/pipelines/run/events-500/out.test.toml @@ -0,0 +1 @@ +Cloud = false diff --git a/acceptance/pipelines/run/events-500/output.txt b/acceptance/pipelines/run/events-500/output.txt new file mode 100644 index 00000000000..e29bfe84d39 --- /dev/null +++ b/acceptance/pipelines/run/events-500/output.txt @@ -0,0 +1,24 @@ + +>>> [CLI] pipelines deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-pipeline-run-flags/default/files... +Created pipelines.my_pipeline +Files: 2 uploaded, 0 deleted +Resources: 1 created, 0 changed, 0 deleted, 0 unchanged +View your pipeline my_pipeline here: [DATABRICKS_URL]/pipelines/[UUID][0]?w=[WORKSPACE_ID] + +=== A 500 from the events API while polling the update is retried + +>>> [CLI] pipelines run +Recommendation: This command runs the last deployed version of the code + +If you've made local changes, run 'databricks pipelines deploy' first to ensure they are included. + +Update URL: [DATABRICKS_URL]/#joblist/pipelines/[UUID][0]/updates/[UUID][1] + +Warn: retrying after 500 Internal Server Error from GET /api/2.0/pipelines/[UUID][0]/events +Update ID: [UUID][1] + +Update for pipeline completed successfully. + +Pipeline configurations for this update: +• All tables are refreshed diff --git a/acceptance/pipelines/run/events-500/pipeline_file.py b/acceptance/pipelines/run/events-500/pipeline_file.py new file mode 100644 index 00000000000..aa0c44e3f6d --- /dev/null +++ b/acceptance/pipelines/run/events-500/pipeline_file.py @@ -0,0 +1,7 @@ +import dlt +from pyspark.sql import SparkSession + + +@dlt.table +def my_table(): + return spark.range(10) diff --git a/acceptance/pipelines/run/events-500/script b/acceptance/pipelines/run/events-500/script new file mode 100644 index 00000000000..db8636743c3 --- /dev/null +++ b/acceptance/pipelines/run/events-500/script @@ -0,0 +1,5 @@ +trace $CLI pipelines deploy + +title "A 500 from the events API while polling the update is retried" +fault.py "GET /api/2.0/pipelines/*/events" 500 0 1 INTERNAL_ERROR +trace $CLI pipelines run diff --git a/acceptance/pipelines/run/events-500/test.toml b/acceptance/pipelines/run/events-500/test.toml new file mode 100644 index 00000000000..867aee20983 --- /dev/null +++ b/acceptance/pipelines/run/events-500/test.toml @@ -0,0 +1,3 @@ +Env.DATABRICKS_BUNDLE_RETRY_INTERVAL_MS = "100" +# Assert the retry warning, which acceptance/test.toml suppresses by default. +SuppressLines.warn_retry = "" diff --git a/bundle/run/pipeline.go b/bundle/run/pipeline.go index ad0860c3356..6a7ddf49ff7 100644 --- a/bundle/run/pipeline.go +++ b/bundle/run/pipeline.go @@ -15,6 +15,7 @@ import ( "github.com/databricks/cli/bundle/run/progress" "github.com/databricks/cli/libs/cmdio" "github.com/databricks/cli/libs/log" + "github.com/databricks/cli/libs/retry" "github.com/databricks/databricks-sdk-go/service/pipelines" "github.com/spf13/cobra" ) @@ -58,10 +59,12 @@ func (r *pipelineRunner) logErrorEvent(ctx context.Context, pipelineId, updateId // Otherwise for long lived pipelines, there can be a lot of unnecessary // latency due to multiple pagination API calls needed underneath the hood for // ListPipelineEventsAll - events, err := w.Pipelines.ListPipelineEventsAll(ctx, pipelines.ListPipelineEventsRequest{ - Filter: `level='ERROR'`, - MaxResults: 100, - PipelineId: pipelineId, + events, err := retry.With(ctx, retry.IsTransientRead, func() ([]pipelines.PipelineEvent, error) { + return w.Pipelines.ListPipelineEventsAll(ctx, pipelines.ListPipelineEventsRequest{ + Filter: `level='ERROR'`, + MaxResults: 100, + PipelineId: pipelineId, + }) }) if err != nil { return err @@ -154,7 +157,9 @@ func (r *pipelineRunner) Run(ctx context.Context, opts *Options) (output.RunOutp log.Info(ctx, event.String()) } - update, err := w.Pipelines.GetUpdateByPipelineIdAndUpdateId(ctx, pipelineID, updateID) + update, err := retry.With(ctx, retry.IsTransientRead, func() (*pipelines.GetUpdateResponse, error) { + return w.Pipelines.GetUpdateByPipelineIdAndUpdateId(ctx, pipelineID, updateID) + }) if err != nil { return nil, err } diff --git a/bundle/run/progress/pipeline.go b/bundle/run/progress/pipeline.go index cce17965da8..1a8d6e1e779 100644 --- a/bundle/run/progress/pipeline.go +++ b/bundle/run/progress/pipeline.go @@ -6,6 +6,7 @@ import ( "slices" "strings" + "github.com/databricks/cli/libs/retry" "github.com/databricks/databricks-sdk-go" "github.com/databricks/databricks-sdk-go/service/pipelines" ) @@ -74,10 +75,14 @@ func (l *UpdateTracker) Events(ctx context.Context) ([]ProgressEvent, error) { } // we only check the most recent 100 events for progress - events, err := l.w.Pipelines.ListPipelineEventsAll(ctx, pipelines.ListPipelineEventsRequest{ - PipelineId: l.PipelineId, - MaxResults: 100, - Filter: filter, + // The events API intermittently fails with a 500 ("An unexpected error happened."). Listing is a + // read, so retry it instead of aborting the whole run. + events, err := retry.With(ctx, retry.IsTransientRead, func() ([]pipelines.PipelineEvent, error) { + return l.w.Pipelines.ListPipelineEventsAll(ctx, pipelines.ListPipelineEventsRequest{ + PipelineId: l.PipelineId, + MaxResults: 100, + Filter: filter, + }) }) if err != nil { return nil, err diff --git a/cmd/pipelines/run.go b/cmd/pipelines/run.go index cc85f32e6a0..e4edae8f182 100644 --- a/cmd/pipelines/run.go +++ b/cmd/pipelines/run.go @@ -24,6 +24,7 @@ import ( "github.com/databricks/cli/libs/cmdio" "github.com/databricks/cli/libs/flags" "github.com/databricks/cli/libs/logdiag" + "github.com/databricks/cli/libs/retry" "github.com/databricks/databricks-sdk-go" "github.com/databricks/databricks-sdk-go/service/pipelines" "github.com/spf13/cobra" @@ -166,9 +167,11 @@ func fetchAndDisplayPipelineUpdate(ctx context.Context, w *databricks.WorkspaceC return errors.New("no update ID provided") } - getUpdateResponse, err := w.Pipelines.GetUpdate(ctx, pipelines.GetUpdateRequest{ - PipelineId: pipelineId, - UpdateId: updateId, + getUpdateResponse, err := retry.With(ctx, retry.IsTransientRead, func() (*pipelines.GetUpdateResponse, error) { + return w.Pipelines.GetUpdate(ctx, pipelines.GetUpdateRequest{ + PipelineId: pipelineId, + UpdateId: updateId, + }) }) if err != nil { return err @@ -181,10 +184,12 @@ func fetchAndDisplayPipelineUpdate(ctx context.Context, w *databricks.WorkspaceC latestUpdate := *getUpdateResponse.Update // The public events API rejects event_type filters. - events, err := w.Pipelines.ListPipelineEventsAll(ctx, pipelines.ListPipelineEventsRequest{ - PipelineId: pipelineId, - Filter: fmt.Sprintf("update_id='%s'", updateId), - OrderBy: []string{"timestamp asc"}, + events, err := retry.With(ctx, retry.IsTransientRead, func() ([]pipelines.PipelineEvent, error) { + return w.Pipelines.ListPipelineEventsAll(ctx, pipelines.ListPipelineEventsRequest{ + PipelineId: pipelineId, + Filter: fmt.Sprintf("update_id='%s'", updateId), + OrderBy: []string{"timestamp asc"}, + }) }) if err != nil { return err diff --git a/cmd/root/api_error.go b/cmd/root/api_error.go new file mode 100644 index 00000000000..29f20ae9ba3 --- /dev/null +++ b/cmd/root/api_error.go @@ -0,0 +1,44 @@ +package root + +import ( + "errors" + "strings" + + "github.com/databricks/cli/libs/diag" + "github.com/databricks/databricks-sdk-go/apierr" +) + +// describedError replaces the text of an error while keeping it unwrappable. +type describedError struct { + msg string + err error +} + +func (e *describedError) Error() string { return e.msg } +func (e *describedError) Unwrap() error { return e.err } + +// describeEmptyAPIError fills in an API error whose response had no "message" field. +// The SDK then leaves the message empty and the CLI would print a bare "Error: ", +// which says nothing about which request failed or how. +// +// Only an error that would print as nothing at all is described; wrappers that add +// their own text already say something about the failure. +func describeEmptyAPIError(err error) error { + if strings.TrimSpace(err.Error()) != "" { + return err + } + apiErr, ok := errors.AsType[*apierr.APIError](err) + if !ok { + return err + } + + desc := "API request failed with " + diag.FormatAPIErrorStatus(apiErr) + // The SDK formats a missing error_code as "". + if code := apiErr.ErrorCode; code != "" && code != "" { + desc += " (" + code + ")" + } + if endpoint := diag.FormatAPIErrorEndpoint(apiErr); endpoint != "" { + desc += " from " + endpoint + } + return &describedError{msg: desc, err: err} +} diff --git a/cmd/root/api_error_test.go b/cmd/root/api_error_test.go new file mode 100644 index 00000000000..da8b53c24b4 --- /dev/null +++ b/cmd/root/api_error_test.go @@ -0,0 +1,50 @@ +package root + +import ( + "errors" + "fmt" + "net/http" + "net/url" + "testing" + + "github.com/databricks/databricks-sdk-go/apierr" + "github.com/databricks/databricks-sdk-go/common" + "github.com/stretchr/testify/assert" +) + +func newAPIError(message, code string) *apierr.APIError { + req := &http.Request{Method: http.MethodPost, URL: &url.URL{Path: "/api/2.0/x", RawQuery: "q=1"}} + return &apierr.APIError{ + Message: message, + ErrorCode: code, + StatusCode: 500, + ResponseWrapper: &common.ResponseWrapper{Response: &http.Response{Request: req}}, + } +} + +func TestDescribeEmptyAPIError(t *testing.T) { + for _, tc := range []struct { + name string + err error + want string + }{ + {"not an API error", errors.New("boom"), "boom"}, + {"has a message", newAPIError("no access", "PERMISSION_DENIED"), "no access"}, + {"empty message", newAPIError("", "INTERNAL_ERROR"), "API request failed with 500 Internal Server Error (INTERNAL_ERROR) from POST /api/2.0/x"}, + {"no error code", newAPIError("", ""), "API request failed with 500 Internal Server Error from POST /api/2.0/x"}, + {"whitespace message", newAPIError("\n", ""), "API request failed with 500 Internal Server Error from POST /api/2.0/x"}, + {"wrapped with own text", fmt.Errorf("creating thing: %w", newAPIError("", "")), "creating thing: "}, + {"wrapped without own text", fmt.Errorf("%w", newAPIError("", "INTERNAL_ERROR")), "API request failed with 500 Internal Server Error (INTERNAL_ERROR) from POST /api/2.0/x"}, + } { + t.Run(tc.name, func(t *testing.T) { + got := describeEmptyAPIError(tc.err) + assert.Equal(t, tc.want, got.Error()) + assert.ErrorIs(t, got, tc.err) + }) + } +} + +func TestDescribeEmptyAPIErrorWithoutRequest(t *testing.T) { + err := &apierr.APIError{StatusCode: 499} + assert.Equal(t, "API request failed with 499", describeEmptyAPIError(err).Error()) +} diff --git a/cmd/root/root.go b/cmd/root/root.go index 1a9ca7c62e6..65bc5dc8019 100644 --- a/cmd/root/root.go +++ b/cmd/root/root.go @@ -176,6 +176,7 @@ Stack Trace: // Run the command cmd, err = cmd.ExecuteContextC(ctx) if err != nil && !errors.Is(err, ErrAlreadyPrinted) { + err = describeEmptyAPIError(err) if cmdctx.HasConfigUsed(cmd.Context()) { cfg := cmdctx.ConfigUsed(cmd.Context()) err = auth.EnrichAuthError(cmd.Context(), cfg, err) diff --git a/libs/diag/sdk_error.go b/libs/diag/sdk_error.go index d190498d6bf..c0183190882 100644 --- a/libs/diag/sdk_error.go +++ b/libs/diag/sdk_error.go @@ -3,6 +3,7 @@ package diag import ( "errors" "fmt" + "net/http" "strconv" "strings" @@ -42,3 +43,24 @@ func FormatAPIErrorDetails(e error) string { } return fmt.Sprintf("Endpoint: %s\nHTTP Status: %s\nAPI error_code: %s\nAPI message: %s", endpoint, httpStatus, apiErr.ErrorCode, apiErr.Message) } + +// FormatAPIErrorStatus returns the HTTP status of an API error, e.g. "500 Internal Server Error". +func FormatAPIErrorStatus(apiErr *apierr.APIError) string { + status := strconv.Itoa(apiErr.StatusCode) + // Non-standard codes such as 499 have no status text. + if text := http.StatusText(apiErr.StatusCode); text != "" { + status += " " + text + } + return status +} + +// FormatAPIErrorEndpoint returns the method and path of the request that failed, e.g. +// "GET /api/2.0/clusters/get", or "" if the error carries no request. +func FormatAPIErrorEndpoint(apiErr *apierr.APIError) string { + rw := apiErr.ResponseWrapper + if rw == nil || rw.Response == nil || rw.Response.Request == nil { + return "" + } + req := rw.Response.Request + return req.Method + " " + req.URL.Path +} diff --git a/libs/retry/retry.go b/libs/retry/retry.go index b575e67309a..4c32476dfb9 100644 --- a/libs/retry/retry.go +++ b/libs/retry/retry.go @@ -12,6 +12,7 @@ import ( "strings" "time" + "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/env" "github.com/databricks/cli/libs/log" "github.com/databricks/databricks-sdk-go/apierr" @@ -132,15 +133,10 @@ func retryMessage(err error) string { msg := "retrying after " + err.Error() if apiErr, ok := errors.AsType[*apierr.APIError](err); ok { endpoint := "" - if rw := apiErr.ResponseWrapper; rw != nil && rw.Response != nil && rw.Response.Request != nil { - req := rw.Response.Request - endpoint = fmt.Sprintf(" from %s %s", req.Method, req.URL.Path) - } - status := strconv.Itoa(apiErr.StatusCode) - // Non-standard codes such as 499 have no status text. - if text := http.StatusText(apiErr.StatusCode); text != "" { - status += " " + text + if e := diag.FormatAPIErrorEndpoint(apiErr); e != "" { + endpoint = " from " + e } + status := diag.FormatAPIErrorStatus(apiErr) msg = fmt.Sprintf("retrying after %s%s", status, endpoint) // Include the context the caller added to the error, e.g. the path of the file being written. if err.Error() != apiErr.Error() {