diff --git a/cmd/pipelines/test.go b/cmd/pipelines/test.go index 3b1fce0f944..9126c3c796b 100644 --- a/cmd/pipelines/test.go +++ b/cmd/pipelines/test.go @@ -459,7 +459,7 @@ func waitForTestUpdate(ctx context.Context, w *databricks.WorkspaceClient, pipel for { response, err := w.Pipelines.GetUpdateByPipelineIdAndUpdateId(ctx, pipelineID, updateID) if err != nil { - return "", err + return "", waitReadError(ctx, err) } if response.Update == nil { return "", fmt.Errorf("no update found with id %s for pipeline %s", updateID, pipelineID) @@ -470,7 +470,7 @@ func waitForTestUpdate(ctx context.Context, w *databricks.WorkspaceClient, pipel // Reading from the last timestamp returns its events again; the run ignores duplicates. events, err := readTestEvents(ctx, w, pipelineID, updateID, lastTimestamp) if err != nil { - return "", err + return "", waitReadError(ctx, err) } if len(events) > 0 { lastTimestamp = events[len(events)-1].Timestamp @@ -482,7 +482,7 @@ func waitForTestUpdate(ctx context.Context, w *databricks.WorkspaceClient, pipel // Events can become readable after later ones, so read all of them once more. events, err := readTestEvents(ctx, w, pipelineID, updateID, "") if err != nil { - return "", err + return "", waitReadError(ctx, err) } addTestEvents(run, printer, events) return state, nil @@ -497,6 +497,18 @@ func waitForTestUpdate(ctx context.Context, w *databricks.WorkspaceClient, pipel } } +// waitReadError adds the cause of ctx, such as errTestWaitTimeout, to a read error that ctx's +// cancellation caused. The SDK rate limiter fails a request on a canceled ctx before sending it, +// with the bare context error that hides the cause. +func waitReadError(ctx context.Context, err error) error { + if errors.Is(err, context.Canceled) { + if cause := context.Cause(ctx); cause != nil && !errors.Is(err, cause) { + return fmt.Errorf("%w: %w", cause, err) + } + } + return err +} + // addTestEvents adds events to the run and prints each case that gets its final outcome. func addTestEvents(run *testRun, printer *testProgressPrinter, events []testPipelineEvent) { for _, e := range events { @@ -756,8 +768,7 @@ of the deployed snapshot, which changes with each deployment.`, defer stopReads() state, err := waitForTestUpdate(readCtx, target.w, target.pipelineID, updateID, run, printer) if err != nil { - // A read that the timeout interrupts fails with the bare context error, so the cause tells. - isTimeout := errors.Is(context.Cause(readCtx), errTestWaitTimeout) + isTimeout := errors.Is(err, errTestWaitTimeout) || errors.Is(context.Cause(readCtx), errTestWaitTimeout) resumeCmd := target.resumeCommand(cmd, updateID) if !isTimeout && waitCtx.Err() == nil { // The SDK client already retries transient errors, so the read failed for good. diff --git a/cmd/pipelines/test_test.go b/cmd/pipelines/test_test.go index 7dc99d66cb6..57cf8844feb 100644 --- a/cmd/pipelines/test_test.go +++ b/cmd/pipelines/test_test.go @@ -5,6 +5,7 @@ import ( "context" "encoding/json" "errors" + "fmt" "net/http" "net/http/httptest" "os" @@ -341,6 +342,34 @@ func TestWaitForTestUpdateTimeout(t *testing.T) { assert.ErrorIs(t, err, errTestWaitTimeout) } +// A timeout that passes between reads fails the next read in the SDK rate limiter, with a bare +// context error; the wait still reports the timeout. +func TestWaitForTestUpdateTimeoutBeforeRead(t *testing.T) { + server := testserver.New(t) + w, err := databricks.NewWorkspaceClient(&databricks.Config{Host: server.URL, Token: "token"}) + require.NoError(t, err) + ctx, cancel := context.WithCancelCause(t.Context()) + cancel(errTestWaitTimeout) + printer := &testProgressPrinter{out: &bytes.Buffer{}, shortID: func(id string) string { return id }} + _, err = waitForTestUpdate(ctx, w, "p1", "u1", newTestRun(), printer) + assert.ErrorIs(t, err, errTestWaitTimeout) +} + +func TestWaitReadError(t *testing.T) { + ctx, cancel := context.WithCancelCause(t.Context()) + readErr := errors.New("403 forbidden") + // A real error is kept even if ctx is canceled by now. + cancel(errTestWaitTimeout) + assert.Equal(t, readErr, waitReadError(ctx, readErr)) + + // A context error gets the cause, and keeps the read's own message. + canceled := fmt.Errorf("failed to fetch pipeline events: failed in rate limiter: %w", context.Canceled) + err := waitReadError(ctx, canceled) + assert.ErrorIs(t, err, errTestWaitTimeout) + assert.ErrorIs(t, err, context.Canceled) + assert.Contains(t, err.Error(), "failed to fetch pipeline events") +} + // The timeout also interrupts a read in flight, so an update that ends after it is not reported. func TestWaitForTestUpdateTimeoutInterruptsRead(t *testing.T) { server := testserver.New(t)