Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .nextchanges/bundles/pipeline-events-retry.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
* `bundle run` and `pipelines run` now retry transient errors from the pipeline events and update status reads while waiting for an update. ([#7800](https://github.com/databricks/cli/pull/7800))
1 change: 1 addition & 0 deletions .nextchanges/cli/empty-api-error-message.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
* Show the HTTP status, error code and endpoint when an API error response has no message, instead of a bare `Error: `. ([#7800](https://github.com/databricks/cli/pull/7800))
1 change: 1 addition & 0 deletions acceptance/cmd/empty-error-message/out.test.toml
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
Cloud = false
15 changes: 15 additions & 0 deletions acceptance/cmd/empty-error-message/output.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@

>>> errcode [CLI] lakeview create --display-name test
Error: API request failed with 500 Internal Server Error (INTERNAL_ERROR) from POST /api/2.0/lakeview/dashboards

Exit code: 1

>>> errcode [CLI] lakeview get 1234
Error: API request failed with 404 Not Found from GET /api/2.0/lakeview/dashboards/1234

Exit code: 1

>>> errcode [CLI] secrets list-scopes
Error: no access

Exit code: 1
4 changes: 4 additions & 0 deletions acceptance/cmd/empty-error-message/script
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
trace errcode $CLI lakeview create --display-name test
trace errcode $CLI lakeview get 1234
# A non-empty message is printed as is.
trace errcode $CLI secrets list-scopes
18 changes: 18 additions & 0 deletions acceptance/cmd/empty-error-message/test.toml
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
Cloud = false

# Error responses without a "message" field: the CLI must still say which request failed.

[[Server]]
Pattern = "POST /api/2.0/lakeview/dashboards"
Response.Body = '{"error_code": "INTERNAL_ERROR"}'
Response.StatusCode = 500

[[Server]]
Pattern = "GET /api/2.0/lakeview/dashboards/{dashboard_id}"
Response.Body = '{}'
Response.StatusCode = 404

[[Server]]
Pattern = "GET /api/2.0/secrets/scopes/list"
Response.Body = '{"error_code": "PERMISSION_DENIED", "message": "no access"}'
Response.StatusCode = 400
10 changes: 10 additions & 0 deletions acceptance/pipelines/run/events-500/databricks.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
bundle:
name: test-pipeline-run-flags

resources:
pipelines:
my_pipeline:
name: test-pipeline-flags
libraries:
- file:
path: pipeline_file.py
1 change: 1 addition & 0 deletions acceptance/pipelines/run/events-500/out.test.toml
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
Cloud = false
24 changes: 24 additions & 0 deletions acceptance/pipelines/run/events-500/output.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@

>>> [CLI] pipelines deploy
Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-pipeline-run-flags/default/files...
Created pipelines.my_pipeline
Files: 2 uploaded, 0 deleted
Resources: 1 created, 0 changed, 0 deleted, 0 unchanged
View your pipeline my_pipeline here: [DATABRICKS_URL]/pipelines/[UUID][0]?w=[WORKSPACE_ID]

=== A 500 from the events API while polling the update is retried

>>> [CLI] pipelines run
Recommendation: This command runs the last deployed version of the code

If you've made local changes, run 'databricks pipelines deploy' first to ensure they are included.

Update URL: [DATABRICKS_URL]/#joblist/pipelines/[UUID][0]/updates/[UUID][1]

Warn: retrying after 500 Internal Server Error from GET /api/2.0/pipelines/[UUID][0]/events
Update ID: [UUID][1]

Update for pipeline completed successfully.

Pipeline configurations for this update:
• All tables are refreshed
7 changes: 7 additions & 0 deletions acceptance/pipelines/run/events-500/pipeline_file.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
import dlt
from pyspark.sql import SparkSession


@dlt.table
def my_table():
return spark.range(10)
5 changes: 5 additions & 0 deletions acceptance/pipelines/run/events-500/script
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
trace $CLI pipelines deploy

title "A 500 from the events API while polling the update is retried"
fault.py "GET /api/2.0/pipelines/*/events" 500 0 1 INTERNAL_ERROR
trace $CLI pipelines run
3 changes: 3 additions & 0 deletions acceptance/pipelines/run/events-500/test.toml
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
Env.DATABRICKS_BUNDLE_RETRY_INTERVAL_MS = "100"
Comment thread
denik marked this conversation as resolved.
Resolved
# Assert the retry warning, which acceptance/test.toml suppresses by default.
SuppressLines.warn_retry = ""
15 changes: 10 additions & 5 deletions bundle/run/pipeline.go
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ import (
"github.com/databricks/cli/bundle/run/progress"
"github.com/databricks/cli/libs/cmdio"
"github.com/databricks/cli/libs/log"
"github.com/databricks/cli/libs/retry"
"github.com/databricks/databricks-sdk-go/service/pipelines"
"github.com/spf13/cobra"
)
Expand Down Expand Up @@ -58,10 +59,12 @@ func (r *pipelineRunner) logErrorEvent(ctx context.Context, pipelineId, updateId
// Otherwise for long lived pipelines, there can be a lot of unnecessary
// latency due to multiple pagination API calls needed underneath the hood for
// ListPipelineEventsAll
events, err := w.Pipelines.ListPipelineEventsAll(ctx, pipelines.ListPipelineEventsRequest{
Filter: `level='ERROR'`,
MaxResults: 100,
PipelineId: pipelineId,
events, err := retry.With(ctx, retry.IsTransientRead, func() ([]pipelines.PipelineEvent, error) {
return w.Pipelines.ListPipelineEventsAll(ctx, pipelines.ListPipelineEventsRequest{
Filter: `level='ERROR'`,
MaxResults: 100,
PipelineId: pipelineId,
})
})
if err != nil {
return err
Expand Down Expand Up @@ -154,7 +157,9 @@ func (r *pipelineRunner) Run(ctx context.Context, opts *Options) (output.RunOutp
log.Info(ctx, event.String())
}

update, err := w.Pipelines.GetUpdateByPipelineIdAndUpdateId(ctx, pipelineID, updateID)
update, err := retry.With(ctx, retry.IsTransientRead, func() (*pipelines.GetUpdateResponse, error) {
Comment thread
denik marked this conversation as resolved.
Resolved
return w.Pipelines.GetUpdateByPipelineIdAndUpdateId(ctx, pipelineID, updateID)
})
if err != nil {
return nil, err
}
Expand Down
13 changes: 9 additions & 4 deletions bundle/run/progress/pipeline.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import (
"slices"
"strings"

"github.com/databricks/cli/libs/retry"
"github.com/databricks/databricks-sdk-go"
"github.com/databricks/databricks-sdk-go/service/pipelines"
)
Expand Down Expand Up @@ -74,10 +75,14 @@ func (l *UpdateTracker) Events(ctx context.Context) ([]ProgressEvent, error) {
}

// we only check the most recent 100 events for progress
events, err := l.w.Pipelines.ListPipelineEventsAll(ctx, pipelines.ListPipelineEventsRequest{
PipelineId: l.PipelineId,
MaxResults: 100,
Filter: filter,
// The events API intermittently fails with a 500 ("An unexpected error happened."). Listing is a
// read, so retry it instead of aborting the whole run.
events, err := retry.With(ctx, retry.IsTransientRead, func() ([]pipelines.PipelineEvent, error) {
return l.w.Pipelines.ListPipelineEventsAll(ctx, pipelines.ListPipelineEventsRequest{
PipelineId: l.PipelineId,
MaxResults: 100,
Filter: filter,
})
})
if err != nil {
return nil, err
Expand Down
19 changes: 12 additions & 7 deletions cmd/pipelines/run.go
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ import (
"github.com/databricks/cli/libs/cmdio"
"github.com/databricks/cli/libs/flags"
"github.com/databricks/cli/libs/logdiag"
"github.com/databricks/cli/libs/retry"
"github.com/databricks/databricks-sdk-go"
"github.com/databricks/databricks-sdk-go/service/pipelines"
"github.com/spf13/cobra"
Expand Down Expand Up @@ -166,9 +167,11 @@ func fetchAndDisplayPipelineUpdate(ctx context.Context, w *databricks.WorkspaceC
return errors.New("no update ID provided")
}

getUpdateResponse, err := w.Pipelines.GetUpdate(ctx, pipelines.GetUpdateRequest{
PipelineId: pipelineId,
UpdateId: updateId,
getUpdateResponse, err := retry.With(ctx, retry.IsTransientRead, func() (*pipelines.GetUpdateResponse, error) {
return w.Pipelines.GetUpdate(ctx, pipelines.GetUpdateRequest{
PipelineId: pipelineId,
UpdateId: updateId,
})
})
if err != nil {
return err
Expand All @@ -181,10 +184,12 @@ func fetchAndDisplayPipelineUpdate(ctx context.Context, w *databricks.WorkspaceC
latestUpdate := *getUpdateResponse.Update
Comment thread
denik marked this conversation as resolved.
Resolved

// The public events API rejects event_type filters.
events, err := w.Pipelines.ListPipelineEventsAll(ctx, pipelines.ListPipelineEventsRequest{
PipelineId: pipelineId,
Filter: fmt.Sprintf("update_id='%s'", updateId),
OrderBy: []string{"timestamp asc"},
events, err := retry.With(ctx, retry.IsTransientRead, func() ([]pipelines.PipelineEvent, error) {
return w.Pipelines.ListPipelineEventsAll(ctx, pipelines.ListPipelineEventsRequest{
PipelineId: pipelineId,
Filter: fmt.Sprintf("update_id='%s'", updateId),
OrderBy: []string{"timestamp asc"},
})
})
if err != nil {
return err
Expand Down
44 changes: 44 additions & 0 deletions cmd/root/api_error.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,44 @@
package root

import (
"errors"
"strings"

"github.com/databricks/cli/libs/diag"
"github.com/databricks/databricks-sdk-go/apierr"
)

// describedError replaces the text of an error while keeping it unwrappable.
type describedError struct {
msg string
err error
}

func (e *describedError) Error() string { return e.msg }
func (e *describedError) Unwrap() error { return e.err }

// describeEmptyAPIError fills in an API error whose response had no "message" field.
// The SDK then leaves the message empty and the CLI would print a bare "Error: ",
// which says nothing about which request failed or how.
//
// Only an error that would print as nothing at all is described; wrappers that add
// their own text already say something about the failure.
func describeEmptyAPIError(err error) error {
if strings.TrimSpace(err.Error()) != "" {
return err
}
apiErr, ok := errors.AsType[*apierr.APIError](err)
if !ok {
return err
}

desc := "API request failed with " + diag.FormatAPIErrorStatus(apiErr)
// The SDK formats a missing error_code as "<nil>".
if code := apiErr.ErrorCode; code != "" && code != "<nil>" {
desc += " (" + code + ")"
}
if endpoint := diag.FormatAPIErrorEndpoint(apiErr); endpoint != "" {
desc += " from " + endpoint
}
return &describedError{msg: desc, err: err}
}
50 changes: 50 additions & 0 deletions cmd/root/api_error_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,50 @@
package root

import (
"errors"
"fmt"
"net/http"
"net/url"
"testing"

"github.com/databricks/databricks-sdk-go/apierr"
"github.com/databricks/databricks-sdk-go/common"
"github.com/stretchr/testify/assert"
)

func newAPIError(message, code string) *apierr.APIError {
req := &http.Request{Method: http.MethodPost, URL: &url.URL{Path: "/api/2.0/x", RawQuery: "q=1"}}
return &apierr.APIError{
Message: message,
ErrorCode: code,
StatusCode: 500,
ResponseWrapper: &common.ResponseWrapper{Response: &http.Response{Request: req}},
}
}

func TestDescribeEmptyAPIError(t *testing.T) {
for _, tc := range []struct {
name string
err error
want string
}{
{"not an API error", errors.New("boom"), "boom"},
{"has a message", newAPIError("no access", "PERMISSION_DENIED"), "no access"},
{"empty message", newAPIError("", "INTERNAL_ERROR"), "API request failed with 500 Internal Server Error (INTERNAL_ERROR) from POST /api/2.0/x"},
{"no error code", newAPIError("", "<nil>"), "API request failed with 500 Internal Server Error from POST /api/2.0/x"},
{"whitespace message", newAPIError("\n", ""), "API request failed with 500 Internal Server Error from POST /api/2.0/x"},
{"wrapped with own text", fmt.Errorf("creating thing: %w", newAPIError("", "")), "creating thing: "},
{"wrapped without own text", fmt.Errorf("%w", newAPIError("", "INTERNAL_ERROR")), "API request failed with 500 Internal Server Error (INTERNAL_ERROR) from POST /api/2.0/x"},
} {
t.Run(tc.name, func(t *testing.T) {
got := describeEmptyAPIError(tc.err)
assert.Equal(t, tc.want, got.Error())
assert.ErrorIs(t, got, tc.err)
})
}
}

func TestDescribeEmptyAPIErrorWithoutRequest(t *testing.T) {
err := &apierr.APIError{StatusCode: 499}
assert.Equal(t, "API request failed with 499", describeEmptyAPIError(err).Error())
}
1 change: 1 addition & 0 deletions cmd/root/root.go
Original file line number Diff line number Diff line change
Expand Up @@ -176,6 +176,7 @@ Stack Trace:
// Run the command
cmd, err = cmd.ExecuteContextC(ctx)
if err != nil && !errors.Is(err, ErrAlreadyPrinted) {
err = describeEmptyAPIError(err)
if cmdctx.HasConfigUsed(cmd.Context()) {
cfg := cmdctx.ConfigUsed(cmd.Context())
err = auth.EnrichAuthError(cmd.Context(), cfg, err)
Expand Down
22 changes: 22 additions & 0 deletions libs/diag/sdk_error.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ package diag
import (
"errors"
"fmt"
"net/http"
"strconv"
"strings"

Expand Down Expand Up @@ -42,3 +43,24 @@ func FormatAPIErrorDetails(e error) string {
}
return fmt.Sprintf("Endpoint: %s\nHTTP Status: %s\nAPI error_code: %s\nAPI message: %s", endpoint, httpStatus, apiErr.ErrorCode, apiErr.Message)
}

// FormatAPIErrorStatus returns the HTTP status of an API error, e.g. "500 Internal Server Error".
func FormatAPIErrorStatus(apiErr *apierr.APIError) string {
status := strconv.Itoa(apiErr.StatusCode)
// Non-standard codes such as 499 have no status text.
if text := http.StatusText(apiErr.StatusCode); text != "" {
status += " " + text
}
return status
}

// FormatAPIErrorEndpoint returns the method and path of the request that failed, e.g.
// "GET /api/2.0/clusters/get", or "" if the error carries no request.
func FormatAPIErrorEndpoint(apiErr *apierr.APIError) string {
rw := apiErr.ResponseWrapper
if rw == nil || rw.Response == nil || rw.Response.Request == nil {
return ""
}
req := rw.Response.Request
return req.Method + " " + req.URL.Path
}
12 changes: 4 additions & 8 deletions libs/retry/retry.go
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ import (
"strings"
"time"

"github.com/databricks/cli/libs/diag"
"github.com/databricks/cli/libs/env"
"github.com/databricks/cli/libs/log"
"github.com/databricks/databricks-sdk-go/apierr"
Expand Down Expand Up @@ -132,15 +133,10 @@ func retryMessage(err error) string {
msg := "retrying after " + err.Error()
if apiErr, ok := errors.AsType[*apierr.APIError](err); ok {
endpoint := ""
if rw := apiErr.ResponseWrapper; rw != nil && rw.Response != nil && rw.Response.Request != nil {
req := rw.Response.Request
endpoint = fmt.Sprintf(" from %s %s", req.Method, req.URL.Path)
}
status := strconv.Itoa(apiErr.StatusCode)
// Non-standard codes such as 499 have no status text.
if text := http.StatusText(apiErr.StatusCode); text != "" {
status += " " + text
if e := diag.FormatAPIErrorEndpoint(apiErr); e != "" {
endpoint = " from " + e
}
status := diag.FormatAPIErrorStatus(apiErr)
msg = fmt.Sprintf("retrying after %s%s", status, endpoint)
// Include the context the caller added to the error, e.g. the path of the file being written.
if err.Error() != apiErr.Error() {
Expand Down
Loading