diff --git a/.nextchanges/bundles/7602.md b/.nextchanges/bundles/7602.md new file mode 100644 index 00000000000..9fe798f6fe3 --- /dev/null +++ b/.nextchanges/bundles/7602.md @@ -0,0 +1 @@ +* Use the Lakebase branch when its create request times out but the branch was created, instead of failing the deploy, in the direct deployment engine. ([#7602](https://github.com/databricks/cli/pull/7602)) diff --git a/acceptance/bundle/resources/postgres_branches/create_504/databricks.yml b/acceptance/bundle/resources/postgres_branches/create_504/databricks.yml new file mode 100644 index 00000000000..3a5d296fbfe --- /dev/null +++ b/acceptance/bundle/resources/postgres_branches/create_504/databricks.yml @@ -0,0 +1,18 @@ +bundle: + name: test-bundle + +sync: + paths: [] + +resources: + postgres_projects: + my_project: + project_id: test-pg-proj + display_name: "Test Postgres Project" + pg_version: 16 + + postgres_branches: + dev: + parent: ${resources.postgres_projects.my_project.id} + branch_id: dev + no_expiry: true diff --git a/acceptance/bundle/resources/postgres_branches/create_504/out.test.toml b/acceptance/bundle/resources/postgres_branches/create_504/out.test.toml new file mode 100644 index 00000000000..15c32f324ac --- /dev/null +++ b/acceptance/bundle/resources/postgres_branches/create_504/out.test.toml @@ -0,0 +1,3 @@ +Cloud = false +EnvMatrix.DMS = ["", "true"] +EnvMatrix.READPLAN = [] diff --git a/acceptance/bundle/resources/postgres_branches/create_504/output.txt b/acceptance/bundle/resources/postgres_branches/create_504/output.txt new file mode 100644 index 00000000000..1caeac8cd64 --- /dev/null +++ b/acceptance/bundle/resources/postgres_branches/create_504/output.txt @@ -0,0 +1,65 @@ + +=== The create request times out but the branch was created: deploy uses the branch + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Created postgres_branches.dev +Created postgres_projects.my_project +Files: 0 uploaded, 0 deleted +Resources: 2 created, 0 changed, 0 deleted, 0 unchanged + +>>> print_requests.py --get //postgres/projects/test-pg-proj/branches +POST /api/2.0/postgres/projects/test-pg-proj/branches +GET /api/2.0/postgres/projects/test-pg-proj/branches/dev + +=== The create request times out and the branch was not created: deploy repeats the create + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Created postgres_projects.my_project +Warn: deploying resources.postgres_branches.dev: retrying after 504 Gateway Timeout from POST /api/2.0/postgres/projects/test-pg-proj/branches +Created postgres_branches.dev +Files: 0 uploaded, 0 deleted +Resources: 2 created, 0 changed, 0 deleted, 0 unchanged + +>>> print_requests.py --get //postgres/projects/test-pg-proj/branches +POST /api/2.0/postgres/projects/test-pg-proj/branches +GET /api/2.0/postgres/projects/test-pg-proj/branches/dev +POST /api/2.0/postgres/projects/test-pg-proj/branches +GET /api/2.0/postgres/projects/test-pg-proj/branches/dev/operations/[UUID][0] + +=== The branch cannot be read after the create request times out: deploy fails with the create error + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Created postgres_projects.my_project +Warn: deploying resources.postgres_branches.dev: checking whether projects/test-pg-proj/branches/dev was created: Fault injected by test. +Error: cannot create resources.postgres_branches.dev: Fault injected by test. (504 DEADLINE_EXCEEDED) + +Endpoint: POST [DATABRICKS_URL]/api/2.0/postgres/projects/test-pg-proj/branches?branch_id=dev +HTTP Status: 504 Gateway Timeout +API error_code: DEADLINE_EXCEEDED +API message: Fault injected by test. + +Files: 0 uploaded, 0 deleted + +Exit code: 1 + +>>> print_requests.py --get //postgres/projects/test-pg-proj/branches +POST /api/2.0/postgres/projects/test-pg-proj/branches +GET /api/2.0/postgres/projects/test-pg-proj/branches/dev + +=== With replace_existing, a timed-out create request is repeated without reading the branch + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Created postgres_projects.my_project +Warn: deploying resources.postgres_branches.dev: retrying after 504 Gateway Timeout from POST /api/2.0/postgres/projects/test-pg-proj/branches +Created postgres_branches.dev +Files: 0 uploaded, 0 deleted +Resources: 2 created, 0 changed, 0 deleted, 0 unchanged + +>>> print_requests.py --get //postgres/projects/test-pg-proj/branches +POST /api/2.0/postgres/projects/test-pg-proj/branches +POST /api/2.0/postgres/projects/test-pg-proj/branches +GET /api/2.0/postgres/projects/test-pg-proj/branches/dev/operations/[UUID][1] diff --git a/acceptance/bundle/resources/postgres_branches/create_504/script b/acceptance/bundle/resources/postgres_branches/create_504/script new file mode 100644 index 00000000000..48ff7fbe56e --- /dev/null +++ b/acceptance/bundle/resources/postgres_branches/create_504/script @@ -0,0 +1,29 @@ +title "The create request times out but the branch was created: deploy uses the branch" +fault.py --after-handler "POST /api/2.0/postgres/projects/*/branches" 504 0 1 DEADLINE_EXCEEDED +trace $CLI bundle deploy +trace print_requests.py --get //postgres/projects/test-pg-proj/branches | jq -r '.method + " " + .path' +$CLI bundle destroy --auto-approve > /dev/null 2>&1 +rm -f "$OUT_REQUESTS" + +title "The create request times out and the branch was not created: deploy repeats the create" +fault.py "POST /api/2.0/postgres/projects/*/branches" 504 0 1 DEADLINE_EXCEEDED +trace $CLI bundle deploy +trace print_requests.py --get //postgres/projects/test-pg-proj/branches | jq -r '.method + " " + .path' +$CLI bundle destroy --auto-approve > /dev/null 2>&1 +rm -f "$OUT_REQUESTS" + +title "The branch cannot be read after the create request times out: deploy fails with the create error" +fault.py "POST /api/2.0/postgres/projects/*/branches" 504 0 1 DEADLINE_EXCEEDED +fault.py "GET /api/2.0/postgres/projects/*/branches/dev" 403 0 1 PERMISSION_DENIED +errcode trace $CLI bundle deploy +trace print_requests.py --get //postgres/projects/test-pg-proj/branches | jq -r '.method + " " + .path' +$CLI bundle destroy --auto-approve > /dev/null 2>&1 +rm -f "$OUT_REQUESTS" + +title "With replace_existing, a timed-out create request is repeated without reading the branch" +update_file.py databricks.yml "no_expiry: true" "no_expiry: true + replace_existing: true" +fault.py --after-handler "POST /api/2.0/postgres/projects/*/branches" 504 0 1 DEADLINE_EXCEEDED +trace $CLI bundle deploy +trace print_requests.py --get //postgres/projects/test-pg-proj/branches | jq -r '.method + " " + .path' +$CLI bundle destroy --auto-approve > /dev/null 2>&1 diff --git a/acceptance/bundle/resources/postgres_branches/create_504/test.toml b/acceptance/bundle/resources/postgres_branches/create_504/test.toml new file mode 100644 index 00000000000..ab3f407bfa7 --- /dev/null +++ b/acceptance/bundle/resources/postgres_branches/create_504/test.toml @@ -0,0 +1,6 @@ +Cloud = false +RecordRequests = true +EnvMatrix.READPLAN = [] +Env.DATABRICKS_BUNDLE_ENGINE = "direct" +Env.DATABRICKS_BUNDLE_RETRY_INTERVAL_MS = "100" +SuppressLines.warn_retry = "" diff --git a/bundle/direct/dresources/postgres_branch.go b/bundle/direct/dresources/postgres_branch.go index e134d5d8876..da755e3ae14 100644 --- a/bundle/direct/dresources/postgres_branch.go +++ b/bundle/direct/dresources/postgres_branch.go @@ -2,10 +2,16 @@ package dresources import ( "context" + "errors" + "fmt" "slices" + "time" "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/cli/libs/log" + "github.com/databricks/cli/libs/retry" "github.com/databricks/databricks-sdk-go" + "github.com/databricks/databricks-sdk-go/apierr" "github.com/databricks/databricks-sdk-go/common/types/fieldmask" sdktime "github.com/databricks/databricks-sdk-go/common/types/time" "github.com/databricks/databricks-sdk-go/marshal" @@ -105,7 +111,7 @@ func (r *ResourcePostgresBranch) DoRead(ctx context.Context, id string) (*Postgr } func (r *ResourcePostgresBranch) DoCreate(ctx context.Context, config *PostgresBranchState) (string, *PostgresBranchRemote, error) { - waiter, err := r.client.Postgres.CreateBranch(ctx, postgres.CreateBranchRequest{ + req := postgres.CreateBranchRequest{ BranchId: config.BranchId, Parent: config.Parent, Branch: postgres.Branch{ @@ -123,21 +129,140 @@ func (r *ResourcePostgresBranch) DoCreate(ctx context.Context, config *PostgresB }, ReplaceExisting: config.ReplaceExisting != nil && *config.ReplaceExisting, ForceSendFields: nil, - }) + } + start := time.Now() + waiter, err := r.client.Postgres.CreateBranch(ctx, req) if err != nil { - return "", nil, err + return r.recoverFailedCreate(ctx, req, start, err) } + return r.waitCreated(ctx, waiter) +} - // Wait for the branch to be ready (long-running operation) +// waitCreated waits for the create operation of a branch that a create request returned. +func (r *ResourcePostgresBranch) waitCreated(ctx context.Context, waiter postgres.CreateBranchOperationInterface) (string, *PostgresBranchRemote, error) { result, err := waitCreateOperation(ctx, waiter.Wait) if err != nil { return "", nil, err } - remote := makePostgresBranchRemote(result) return remote.Name, remote, nil } +// Tuning for recoverFailedCreate, variables so that tests can shorten them. +var ( + // branchReadyTimeout bounds the wait for a recovered branch that is still in INIT. + branchReadyTimeout = 10 * time.Minute + // branchReadyPollInterval is the pause between reads of a recovered branch in INIT. + branchReadyPollInterval = 5 * time.Second + // branchCreateTimeSkew is how much earlier than our first create request a found branch's + // create_time may be, to allow for clock skew, and still count as created by that request. + branchCreateTimeSkew = time.Minute +) + +// recoverFailedCreate handles a create request that failed with a transient error. A slow create +// is answered with 504 DEADLINE_EXCEEDED ("The create request timed out and may still be +// completing"), so the request may create the branch anyway. Look for the branch: use it if our +// request created it, and repeat the request while it is missing. A repeated request that fails +// with ALREADY_EXISTS means an earlier one has landed since, so the next look finds that branch. +func (r *ResourcePostgresBranch) recoverFailedCreate(ctx context.Context, req postgres.CreateBranchRequest, start time.Time, createErr error) (string, *PostgresBranchRemote, error) { + if ctx.Err() != nil || !retry.IsTransient(ctx, createErr) { + return "", nil, createErr + } + // A branch found after a replace may predate it, and repeating a replace is safe. + if req.ReplaceExisting { + return "", nil, retrySafe(createErr) + } + + name := req.Parent + "/branches/" + req.BranchId + lastErr := createErr + for attempt := 1; ; attempt++ { + branch, err := r.readBranch(ctx, name) + if ctx.Err() != nil { + return "", nil, ctx.Err() + } + if err == nil { + return r.useRecoveredBranch(ctx, branch, start, createErr) + } + if !apierr.IsMissing(err) { + log.Warnf(ctx, "checking whether %s was created: %s", name, err) + return "", nil, createErr + } + if attempt > retry.MaxRetries { + return "", nil, lastErr + } + if err := retry.Pause(ctx, lastErr); err != nil { + return "", nil, err + } + waiter, err := r.client.Postgres.CreateBranch(ctx, req) + if err == nil { + return r.waitCreated(ctx, waiter) + } + if !retry.IsTransient(ctx, err) && !errors.Is(err, apierr.ErrAlreadyExists) { + return "", nil, err + } + lastErr = err + } +} + +func (r *ResourcePostgresBranch) readBranch(ctx context.Context, name string) (*postgres.Branch, error) { + return retry.With(ctx, retry.IsTransientRead, func() (*postgres.Branch, error) { + return r.client.Postgres.GetBranch(ctx, postgres.GetBranchRequest{Name: name}) + }) +} + +// useRecoveredBranch returns the branch recoverFailedCreate found, once it is READY. +// +// The wait is inline rather than in WaitAfterCreate, like the wait for the create operation on the +// success path: a WaitAfterCreate would read every created branch once more. Once the branch is +// known to be ours, it is returned even when the wait ends early, so that state records it and the +// next deploy does not fail with ALREADY_EXISTS. +func (r *ResourcePostgresBranch) useRecoveredBranch(ctx context.Context, branch *postgres.Branch, start time.Time, createErr error) (string, *PostgresBranchRemote, error) { + // Without replace_existing, a branch that existed before our request would have made it fail + // with ALREADY_EXISTS. That holds only if the request reached the backend's check, so look at + // when the branch was created rather than adopt a branch the bundle did not create. + if branch.CreateTime == nil || branch.CreateTime.AsTime().Before(start.Add(-branchCreateTimeSkew)) { + return "", nil, fmt.Errorf("%w; branch %s exists but was not created by this deployment", createErr, branch.Name) + } + + deadline := time.Now().Add(branchReadyTimeout) + for branchState(branch) == postgres.BranchStatusStateInit { + if !time.Now().Before(deadline) { + log.Warnf(ctx, "branch %s is still initializing after %s", branch.Name, branchReadyTimeout) + break + } + select { + case <-ctx.Done(): + return "", nil, ctx.Err() + case <-time.After(branchReadyPollInterval): + } + next, err := r.readBranch(ctx, branch.Name) + if err != nil { + if ctx.Err() != nil { + return "", nil, ctx.Err() + } + log.Warnf(ctx, "waiting for branch %s to become ready: %s", branch.Name, err) + break + } + branch = next + } + + // Other states, such as DELETED or ARCHIVED, will not become READY. + if state := branchState(branch); state != postgres.BranchStatusStateReady && state != postgres.BranchStatusStateInit { + return "", nil, fmt.Errorf("%w; branch %s exists in state %q", createErr, branch.Name, state) + } + // Info, not Warn: the deploy got the branch it asked for, so there is nothing to act on. + log.Infof(ctx, "using branch %s, created despite the failed create request: %s", branch.Name, createErr) + remote := makePostgresBranchRemote(branch) + return remote.Name, remote, nil +} + +func branchState(branch *postgres.Branch) postgres.BranchStatusState { + if branch.Status == nil { + return "" + } + return branch.Status.CurrentState +} + // expire_time, no_expiry and ttl are three sides of one oneof, and the API accepts them // in update_mask only under the group name: masking the field itself is answered with // "Unknown field path in update_mask". Probed against a real workspace on 2026-08-31. diff --git a/bundle/direct/dresources/postgres_branch_test.go b/bundle/direct/dresources/postgres_branch_test.go new file mode 100644 index 00000000000..89eb30e4ef8 --- /dev/null +++ b/bundle/direct/dresources/postgres_branch_test.go @@ -0,0 +1,266 @@ +package dresources + +import ( + "context" + "encoding/json" + "errors" + "testing" + "time" + + "github.com/databricks/cli/libs/env" + "github.com/databricks/cli/libs/retry" + "github.com/databricks/cli/libs/testserver" + "github.com/databricks/databricks-sdk-go" + sdktime "github.com/databricks/databricks-sdk-go/common/types/time" + "github.com/databricks/databricks-sdk-go/service/postgres" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +const ( + testBranchParent = "projects/p" + testBranchName = "projects/p/branches/dev" + // The message the backend sends with a slow create; the SDK does not retry it. + testBranchTimeoutMessage = "The create request timed out and may still be completing. Check whether the resource was created before retrying." +) + +// branchCreateScript plays the given responses, in order, to the branch create and get requests. +// A response is a postgres.Branch (for a create: the finished operation that returns it) or a +// testserver.Response for an error. +type branchCreateScript struct { + t *testing.T + creates []any + gets []any + // onGet runs when a get request arrives, before its response is sent. + onGet func() + + createCount int + getCount int + operation *postgres.Operation +} + +func (s *branchCreateScript) client() *databricks.WorkspaceClient { + server := testserver.New(s.t) + server.Handle("POST", "/api/2.0/postgres/projects/{project_id}/branches", func(req testserver.Request) any { + s.createCount++ + require.LessOrEqual(s.t, s.createCount, len(s.creates), "unexpected create request") + resp := s.creates[s.createCount-1] + if branch, ok := resp.(postgres.Branch); ok { + body, err := json.Marshal(branch) + require.NoError(s.t, err) + s.operation = &postgres.Operation{Name: testBranchName + "/operations/op", Done: true, Response: body} + return s.operation + } + return resp + }) + // The SDK polls the operation even when the create response says it is done. + server.Handle("GET", "/api/2.0/postgres/projects/{project_id}/branches/{branch_id}/operations/{operation_id}", func(req testserver.Request) any { + require.NotNil(s.t, s.operation, "unexpected operation request") + return s.operation + }) + server.Handle("GET", "/api/2.0/postgres/projects/{project_id}/branches/{branch_id}", func(req testserver.Request) any { + s.getCount++ + require.LessOrEqual(s.t, s.getCount, len(s.gets), "unexpected get request") + if s.onGet != nil { + s.onGet() + } + return s.gets[s.getCount-1] + }) + testserver.AddDefaultHandlers(server) + + client, err := databricks.NewWorkspaceClient(&databricks.Config{Host: server.URL, Token: "testtoken"}) + require.NoError(s.t, err) + return client +} + +func branchError(status int, code, message string) testserver.Response { + return testserver.Response{StatusCode: status, Body: map[string]string{"error_code": code, "message": message}} +} + +var ( + branchTimeout = branchError(504, "DEADLINE_EXCEEDED", testBranchTimeoutMessage) + branchNotFound = branchError(404, "NOT_FOUND", "branch not found") + branchAlreadyExists = branchError(409, "ALREADY_EXISTS", "branch with such id already exists") +) + +func testBranch(state postgres.BranchStatusState, created time.Time) postgres.Branch { + return postgres.Branch{ + Name: testBranchName, + BranchId: "dev", + Parent: testBranchParent, + CreateTime: sdktime.New(created), + Status: &postgres.BranchStatus{CurrentState: state}, + } +} + +// createTestBranch runs DoCreate with fast retries and polling. +func createTestBranch(t *testing.T, s *branchCreateScript, ctx context.Context, replaceExisting bool) (string, *PostgresBranchRemote, error) { + t.Helper() + ctx = env.Set(ctx, retry.IntervalMsVariable, "1") + setForTest(t, &branchReadyPollInterval, time.Millisecond) + r := (&ResourcePostgresBranch{}).New(s.client()) + return r.DoCreate(ctx, &PostgresBranchState{BranchId: "dev", Parent: testBranchParent, ReplaceExisting: &replaceExisting}) +} + +func setForTest[T any](t *testing.T, v *T, value T) { + old := *v + *v = value + t.Cleanup(func() { *v = old }) +} + +func TestPostgresBranchCreateWaitsForBranchCreatedDespiteTimeout(t *testing.T) { + s := &branchCreateScript{ + t: t, + creates: []any{branchTimeout}, + gets: []any{testBranch(postgres.BranchStatusStateInit, time.Now()), testBranch(postgres.BranchStatusStateReady, time.Now())}, + } + + id, remote, err := createTestBranch(t, s, t.Context(), false) + + require.NoError(t, err) + assert.Equal(t, testBranchName, id) + assert.Equal(t, postgres.BranchStatusStateReady, remote.Status.CurrentState) + assert.Equal(t, 1, s.createCount) + assert.Equal(t, 2, s.getCount) +} + +func TestPostgresBranchCreateReturnsInitBranchAfterReadyTimeout(t *testing.T) { + setForTest(t, &branchReadyTimeout, 0) + s := &branchCreateScript{ + t: t, + creates: []any{branchTimeout}, + gets: []any{testBranch(postgres.BranchStatusStateInit, time.Now())}, + } + + id, remote, err := createTestBranch(t, s, t.Context(), false) + + // Returned so that state records it: the branch exists and is ours. + require.NoError(t, err) + assert.Equal(t, testBranchName, id) + assert.Equal(t, postgres.BranchStatusStateInit, remote.Status.CurrentState) + assert.Equal(t, 1, s.getCount) +} + +func TestPostgresBranchCreateRepeatsCreateWhileBranchIsMissing(t *testing.T) { + s := &branchCreateScript{ + t: t, + creates: []any{branchTimeout, testBranch(postgres.BranchStatusStateReady, time.Now())}, + gets: []any{branchNotFound}, + } + + id, _, err := createTestBranch(t, s, t.Context(), false) + + require.NoError(t, err) + assert.Equal(t, testBranchName, id) + assert.Equal(t, 2, s.createCount) + assert.Equal(t, 1, s.getCount) +} + +func TestPostgresBranchCreateUsesBranchWhenRepeatedCreateFindsIt(t *testing.T) { + // The first request was still completing when the branch was read, and has landed since. + s := &branchCreateScript{ + t: t, + creates: []any{branchTimeout, branchAlreadyExists}, + gets: []any{branchNotFound, testBranch(postgres.BranchStatusStateReady, time.Now())}, + } + + id, _, err := createTestBranch(t, s, t.Context(), false) + + require.NoError(t, err) + assert.Equal(t, testBranchName, id) + assert.Equal(t, 2, s.createCount) + assert.Equal(t, 2, s.getCount) +} + +func TestPostgresBranchCreateGivesUpWhenBranchStaysMissing(t *testing.T) { + s := &branchCreateScript{ + t: t, + creates: []any{branchTimeout, branchTimeout, branchTimeout}, + gets: []any{branchNotFound, branchNotFound, branchNotFound}, + } + + _, _, err := createTestBranch(t, s, t.Context(), false) + + require.ErrorContains(t, err, testBranchTimeoutMessage) + assert.Equal(t, 1+retry.MaxRetries, s.createCount) + assert.Equal(t, 1+retry.MaxRetries, s.getCount) +} + +func TestPostgresBranchCreateDoesNotUseBranchThatPredatesRequest(t *testing.T) { + s := &branchCreateScript{ + t: t, + creates: []any{branchTimeout}, + gets: []any{testBranch(postgres.BranchStatusStateReady, time.Now().Add(-time.Hour))}, + } + + _, _, err := createTestBranch(t, s, t.Context(), false) + + require.ErrorContains(t, err, testBranchTimeoutMessage) + require.ErrorContains(t, err, "branch "+testBranchName+" exists but was not created by this deployment") + assert.Equal(t, 1, s.createCount) +} + +func TestPostgresBranchCreateFailsOnBranchThatWillNotBecomeReady(t *testing.T) { + s := &branchCreateScript{ + t: t, + creates: []any{branchTimeout}, + gets: []any{testBranch(postgres.BranchStatusStateDeleted, time.Now())}, + } + + _, _, err := createTestBranch(t, s, t.Context(), false) + + require.ErrorContains(t, err, testBranchTimeoutMessage) + require.ErrorContains(t, err, `exists in state "DELETED"`) + assert.Equal(t, 1, s.getCount) +} + +func TestPostgresBranchCreateReturnsCreateErrorWhenReadFails(t *testing.T) { + s := &branchCreateScript{ + t: t, + creates: []any{branchTimeout}, + gets: []any{branchError(403, "PERMISSION_DENIED", "no access")}, + } + + _, _, err := createTestBranch(t, s, t.Context(), false) + + require.ErrorContains(t, err, testBranchTimeoutMessage) + assert.NotContains(t, err.Error(), "no access") + assert.Equal(t, 1, s.createCount) +} + +func TestPostgresBranchCreateReturnsCancellationDuringRead(t *testing.T) { + ctx, cancel := context.WithCancel(t.Context()) + s := &branchCreateScript{ + t: t, + creates: []any{branchTimeout}, + gets: []any{branchNotFound}, + onGet: cancel, + } + + _, _, err := createTestBranch(t, s, ctx, false) + + require.ErrorIs(t, err, context.Canceled) + assert.Equal(t, 1, s.createCount) +} + +func TestPostgresBranchCreateLeavesReplaceToCallerRetry(t *testing.T) { + s := &branchCreateScript{t: t, creates: []any{branchTimeout}} + + _, _, err := createTestBranch(t, s, t.Context(), true) + + // Marked safe to retry for DeploymentUnit.Create, without reading the branch. + _, ok := errors.AsType[*RetrySafeError](err) + assert.True(t, ok, "want a RetrySafeError, got %v", err) + assert.Equal(t, 0, s.getCount) +} + +func TestPostgresBranchCreateDoesNotRecoverFromNonTransientError(t *testing.T) { + s := &branchCreateScript{t: t, creates: []any{branchError(400, "INVALID_PARAMETER_VALUE", "bad spec")}} + + _, _, err := createTestBranch(t, s, t.Context(), false) + + require.ErrorContains(t, err, "bad spec") + _, ok := errors.AsType[*RetrySafeError](err) + assert.False(t, ok) + assert.Equal(t, 0, s.getCount) +} diff --git a/libs/retry/retry.go b/libs/retry/retry.go index 95bf5a7250e..db2de0d4524 100644 --- a/libs/retry/retry.go +++ b/libs/retry/retry.go @@ -102,22 +102,30 @@ func IsTransientRead(ctx context.Context, err error) bool { // With retries fn while check returns true for the error, up to MaxRetries times. func With[T any](ctx context.Context, check func(context.Context, error) bool, fn func() (T, error)) (T, error) { - wait := interval(ctx) for attempt := 0; ; attempt++ { result, err := fn() if err == nil || attempt >= MaxRetries || !check(ctx, err) { return result, err } - log.Warnf(ctx, "%s", retryMessage(err)) - select { - case <-ctx.Done(): + if err := Pause(ctx, err); err != nil { var zero T - return zero, ctx.Err() - case <-time.After(wait): + return zero, err } } } +// Pause reports that the call that failed with err is retried and waits out the pause between +// attempts, for callers that run their own retry loop. It returns ctx.Err() if ctx is done first. +func Pause(ctx context.Context, err error) error { + log.Warnf(ctx, "%s", retryMessage(err)) + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(interval(ctx)): + return nil + } +} + // retryMessage describes the error that triggered a retry. func retryMessage(err error) string { // The "retrying after " prefix is what acceptance tests suppress (SuppressLines.warn_retry).