From 5f7e76e0064e8cc1856446e5b044ce4c1ad83a11 Mon Sep 17 00:00:00 2001 From: "denis.bilenko" Date: Thu, 19 Nov 2026 16:42:02 +0000 Subject: [PATCH 1/2] Report the pipelines test wait timeout when it fails a read before it is sent The SDK rate limiter fails a request whose context is already canceled with a bare "context canceled", so a timeout that passed between two reads surfaced as "failed to fetch pipeline events: failed in rate limiter: context canceled" instead of the wait timeout. Return the context's cause for a read that fails after the wait was canceled. Co-authored-by: Isaac --- cmd/pipelines/test.go | 18 +++++++++++++----- cmd/pipelines/test_test.go | 13 +++++++++++++ 2 files changed, 26 insertions(+), 5 deletions(-) diff --git a/cmd/pipelines/test.go b/cmd/pipelines/test.go index 3b1fce0f944..f7772a08246 100644 --- a/cmd/pipelines/test.go +++ b/cmd/pipelines/test.go @@ -459,7 +459,7 @@ func waitForTestUpdate(ctx context.Context, w *databricks.WorkspaceClient, pipel for { response, err := w.Pipelines.GetUpdateByPipelineIdAndUpdateId(ctx, pipelineID, updateID) if err != nil { - return "", err + return "", waitReadError(ctx, err) } if response.Update == nil { return "", fmt.Errorf("no update found with id %s for pipeline %s", updateID, pipelineID) @@ -470,7 +470,7 @@ func waitForTestUpdate(ctx context.Context, w *databricks.WorkspaceClient, pipel // Reading from the last timestamp returns its events again; the run ignores duplicates. events, err := readTestEvents(ctx, w, pipelineID, updateID, lastTimestamp) if err != nil { - return "", err + return "", waitReadError(ctx, err) } if len(events) > 0 { lastTimestamp = events[len(events)-1].Timestamp @@ -482,7 +482,7 @@ func waitForTestUpdate(ctx context.Context, w *databricks.WorkspaceClient, pipel // Events can become readable after later ones, so read all of them once more. events, err := readTestEvents(ctx, w, pipelineID, updateID, "") if err != nil { - return "", err + return "", waitReadError(ctx, err) } addTestEvents(run, printer, events) return state, nil @@ -497,6 +497,15 @@ func waitForTestUpdate(ctx context.Context, w *databricks.WorkspaceClient, pipel } } +// waitReadError returns the cause of ctx, such as errTestWaitTimeout, for a read that failed after +// ctx was canceled: the interrupted read fails with the bare context error, which hides it. +func waitReadError(ctx context.Context, err error) error { + if ctx.Err() != nil { + return context.Cause(ctx) + } + return err +} + // addTestEvents adds events to the run and prints each case that gets its final outcome. func addTestEvents(run *testRun, printer *testProgressPrinter, events []testPipelineEvent) { for _, e := range events { @@ -756,8 +765,7 @@ of the deployed snapshot, which changes with each deployment.`, defer stopReads() state, err := waitForTestUpdate(readCtx, target.w, target.pipelineID, updateID, run, printer) if err != nil { - // A read that the timeout interrupts fails with the bare context error, so the cause tells. - isTimeout := errors.Is(context.Cause(readCtx), errTestWaitTimeout) + isTimeout := errors.Is(err, errTestWaitTimeout) resumeCmd := target.resumeCommand(cmd, updateID) if !isTimeout && waitCtx.Err() == nil { // The SDK client already retries transient errors, so the read failed for good. diff --git a/cmd/pipelines/test_test.go b/cmd/pipelines/test_test.go index 7dc99d66cb6..d643c6f35df 100644 --- a/cmd/pipelines/test_test.go +++ b/cmd/pipelines/test_test.go @@ -341,6 +341,19 @@ func TestWaitForTestUpdateTimeout(t *testing.T) { assert.ErrorIs(t, err, errTestWaitTimeout) } +// A timeout that passes between reads fails the next read in the SDK rate limiter, with a bare +// context error; the wait still reports the timeout. +func TestWaitForTestUpdateTimeoutBeforeRead(t *testing.T) { + server := testserver.New(t) + w, err := databricks.NewWorkspaceClient(&databricks.Config{Host: server.URL, Token: "token"}) + require.NoError(t, err) + ctx, cancel := context.WithCancelCause(t.Context()) + cancel(errTestWaitTimeout) + printer := &testProgressPrinter{out: &bytes.Buffer{}, shortID: func(id string) string { return id }} + _, err = waitForTestUpdate(ctx, w, "p1", "u1", newTestRun(), printer) + assert.ErrorIs(t, err, errTestWaitTimeout) +} + // The timeout also interrupts a read in flight, so an update that ends after it is not reported. func TestWaitForTestUpdateTimeoutInterruptsRead(t *testing.T) { server := testserver.New(t) From 6206a7adca757a087043728d3cc3464262427d6f Mon Sep 17 00:00:00 2001 From: "denis.bilenko" Date: Thu, 19 Nov 2026 16:56:49 +0000 Subject: [PATCH 2/2] Keep real read errors and the read's message; keep the caller's cause check Only add the wait's cause to a context-canceled read error, wrapping it so the read's own message stays, and keep the caller's context.Cause check as a fallback for any read path that does not go through waitReadError. Co-authored-by: Isaac --- cmd/pipelines/test.go | 13 ++++++++----- cmd/pipelines/test_test.go | 16 ++++++++++++++++ 2 files changed, 24 insertions(+), 5 deletions(-) diff --git a/cmd/pipelines/test.go b/cmd/pipelines/test.go index f7772a08246..9126c3c796b 100644 --- a/cmd/pipelines/test.go +++ b/cmd/pipelines/test.go @@ -497,11 +497,14 @@ func waitForTestUpdate(ctx context.Context, w *databricks.WorkspaceClient, pipel } } -// waitReadError returns the cause of ctx, such as errTestWaitTimeout, for a read that failed after -// ctx was canceled: the interrupted read fails with the bare context error, which hides it. +// waitReadError adds the cause of ctx, such as errTestWaitTimeout, to a read error that ctx's +// cancellation caused. The SDK rate limiter fails a request on a canceled ctx before sending it, +// with the bare context error that hides the cause. func waitReadError(ctx context.Context, err error) error { - if ctx.Err() != nil { - return context.Cause(ctx) + if errors.Is(err, context.Canceled) { + if cause := context.Cause(ctx); cause != nil && !errors.Is(err, cause) { + return fmt.Errorf("%w: %w", cause, err) + } } return err } @@ -765,7 +768,7 @@ of the deployed snapshot, which changes with each deployment.`, defer stopReads() state, err := waitForTestUpdate(readCtx, target.w, target.pipelineID, updateID, run, printer) if err != nil { - isTimeout := errors.Is(err, errTestWaitTimeout) + isTimeout := errors.Is(err, errTestWaitTimeout) || errors.Is(context.Cause(readCtx), errTestWaitTimeout) resumeCmd := target.resumeCommand(cmd, updateID) if !isTimeout && waitCtx.Err() == nil { // The SDK client already retries transient errors, so the read failed for good. diff --git a/cmd/pipelines/test_test.go b/cmd/pipelines/test_test.go index d643c6f35df..57cf8844feb 100644 --- a/cmd/pipelines/test_test.go +++ b/cmd/pipelines/test_test.go @@ -5,6 +5,7 @@ import ( "context" "encoding/json" "errors" + "fmt" "net/http" "net/http/httptest" "os" @@ -354,6 +355,21 @@ func TestWaitForTestUpdateTimeoutBeforeRead(t *testing.T) { assert.ErrorIs(t, err, errTestWaitTimeout) } +func TestWaitReadError(t *testing.T) { + ctx, cancel := context.WithCancelCause(t.Context()) + readErr := errors.New("403 forbidden") + // A real error is kept even if ctx is canceled by now. + cancel(errTestWaitTimeout) + assert.Equal(t, readErr, waitReadError(ctx, readErr)) + + // A context error gets the cause, and keeps the read's own message. + canceled := fmt.Errorf("failed to fetch pipeline events: failed in rate limiter: %w", context.Canceled) + err := waitReadError(ctx, canceled) + assert.ErrorIs(t, err, errTestWaitTimeout) + assert.ErrorIs(t, err, context.Canceled) + assert.Contains(t, err.Error(), "failed to fetch pipeline events") +} + // The timeout also interrupts a read in flight, so an update that ends after it is not reported. func TestWaitForTestUpdateTimeoutInterruptsRead(t *testing.T) { server := testserver.New(t)